x11: Add helper util to check for xshm support

Cc: mesa-stable
(cherry picked from commit 9e1671dea9)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/40359>

.pick_status.json

@@ -2424,7 +2424,7 @@
         "description": "x11: Add helper util to check for xshm support",
         "nominated": true,
         "nomination_type": 1,
-        "resolution": 0,
+        "resolution": 1,
         "main_sha": null,
         "because_sha": null,
         "notes": null

src/x11/x11_display.c

@@ -23,10 +23,16 @@
 #include "x11_display.h"
 
 #include "util/macros.h"
+#include "util/log.h"
 
 #include <stdio.h>
 #include <X11/Xlibint.h>
 
+#ifdef HAVE_SYS_SHM_H
+#include <xcb/shm.h>
+#include <sys/shm.h>
+#endif
+
 bool
 x11_xlib_display_is_thread_safe(Display *dpy)
 {
@@ -47,3 +53,76 @@ x11_xlib_display_is_thread_safe(Display *dpy)
 
    return false;
 }
+
+#ifdef HAVE_SYS_SHM_H
+bool
+x11_xcb_display_supports_xshm(xcb_connection_t *con)
+{
+   xcb_generic_error_t *error;
+   xcb_query_extension_cookie_t shm_cookie;
+   xcb_query_extension_reply_t *shm_reply;
+   bool has_mit_shm;
+   int shmid;
+   void *shm_addr = (void *) -1;
+   xcb_shm_seg_t shm_seg;
+   bool ret = true;
+
+   shm_cookie = xcb_query_extension(con, 7, "MIT-SHM");
+   shm_reply = xcb_query_extension_reply(con, shm_cookie, NULL);
+
+   has_mit_shm = shm_reply && shm_reply->present;
+   free(shm_reply);
+   if (!has_mit_shm)
+      return false;
+
+   /* Check if we're a remote client by attempting to detach segment 0.
+    * Remote clients will get BadRequest, local clients get BadValue,
+    * since 'info' has an invalid segment name.
+    */
+   if ((error = xcb_request_check(con,
+                                  xcb_shm_detach_checked(con, 0)))) {
+      bool is_remote = error->error_code == BadRequest;
+      free(error);
+      if (is_remote)
+         return false;
+    }
+
+   /* We're a local client. Now verify we can actually attach SHM.
+    * In some container setups, the X server reports MIT-SHM support
+    * but the client lacks permission to attach shared memory.
+    */
+   shmid = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0600);
+   if (shmid < 0) {
+      mesa_logd("shared memory allocation error: %s", strerror(errno));
+      return false;
+   }
+
+   shm_addr = shmat(shmid, NULL, 0);
+   if (shm_addr == (void *) -1) {
+      mesa_logd("shmat failed: %s", strerror(errno));
+      ret = false;
+      goto check_xshm_cleanup;
+   }
+
+   shm_seg = xcb_generate_id(con);
+   error = xcb_request_check(con,
+                             xcb_shm_attach_checked(con,
+                                                    shm_seg, shmid, 0));
+   if (error) {
+      mesa_logd("Failed to attach to x11 shm with error: %u", error->error_code);
+      free(error);
+      ret = false;
+      goto check_xshm_cleanup;
+   }
+
+   /* Only detach if it was attached successfully */
+   xcb_shm_detach(con, shm_seg);
+
+check_xshm_cleanup:
+   if (shm_addr != (void *) -1)
+      shmdt(shm_addr);
+   shmctl(shmid, IPC_RMID, NULL);
+
+   return ret;
+}
+#endif

src/x11/x11_display.h

@@ -26,6 +26,9 @@
 
 #include <stdbool.h>
 #include <X11/Xlib.h>
+#ifdef HAVE_SYS_SHM_H
+#include <xcb/xcb.h>
+#endif
 
 #ifdef __cplusplus
 extern "C" {
@@ -33,6 +36,10 @@ extern "C" {
 
 bool x11_xlib_display_is_thread_safe(Display *dpy);
 
+#ifdef HAVE_SYS_SHM_H
+bool x11_xcb_display_supports_xshm(xcb_connection_t *con);
+#endif
+
 #ifdef __cplusplus
 } /* extern C */
 #endif