wsi/x11: Detect Xwayland

The following commits will introduce different behaviour for Xwayland.

Reviewed-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8197>

src/vulkan/wsi/wsi_common_x11.c

@@ -52,6 +52,7 @@ struct wsi_x11_connection {
    bool has_dri3_modifiers;
    bool has_present;
    bool is_proprietary_x11;
+   bool is_xwayland;
 };
 
 struct wsi_x11 {
@@ -115,12 +116,57 @@ wsi_x11_check_dri3_compatible(const struct wsi_device *wsi_dev,
    return match;
 }
 
+static bool
+wsi_x11_detect_xwayland(xcb_connection_t *conn)
+{
+   xcb_randr_query_version_cookie_t ver_cookie =
+      xcb_randr_query_version_unchecked(conn, 1, 2);
+   xcb_randr_query_version_reply_t *ver_reply =
+      xcb_randr_query_version_reply(conn, ver_cookie, NULL);
+   bool has_randr_v1_2 = ver_reply && (ver_reply->major_version > 1 ||
+                                       ver_reply->minor_version >= 2);
+   free(ver_reply);
+
+   if (!has_randr_v1_2)
+      return false;
+
+   const xcb_setup_t *setup = xcb_get_setup(conn);
+   xcb_screen_iterator_t iter = xcb_setup_roots_iterator(setup);
+
+   xcb_randr_get_screen_resources_cookie_t gsr_cookie =
+      xcb_randr_get_screen_resources_unchecked(conn, iter.data->root);
+   xcb_randr_get_screen_resources_reply_t *gsr_reply =
+      xcb_randr_get_screen_resources_reply(conn, gsr_cookie, NULL);
+
+   if (!gsr_reply || gsr_reply->num_outputs == 0) {
+      free(gsr_reply);
+      return false;
+   }
+
+   xcb_randr_output_t *randr_outputs = xcb_randr_get_screen_resources_outputs(gsr_reply);
+   xcb_randr_get_output_info_cookie_t goi_cookie =
+      xcb_randr_get_output_info(conn, randr_outputs[0], gsr_reply->config_timestamp);
+   free(gsr_reply);
+
+   xcb_randr_get_output_info_reply_t *goi_reply =
+      xcb_randr_get_output_info_reply(conn, goi_cookie, NULL);
+   if (!goi_reply) {
+      return false;
+   }
+
+   char *output_name = (char*)xcb_randr_get_output_info_name(goi_reply);
+   bool is_xwayland = output_name && strncmp(output_name, "XWAYLAND", 8) == 0;
+   free(goi_reply);
+
+   return is_xwayland;
+}
+
 static struct wsi_x11_connection *
 wsi_x11_connection_create(struct wsi_device *wsi_dev,
                           xcb_connection_t *conn)
 {
-   xcb_query_extension_cookie_t dri3_cookie, pres_cookie, amd_cookie, nv_cookie;
+   xcb_query_extension_cookie_t dri3_cookie, pres_cookie, randr_cookie, amd_cookie, nv_cookie;
-   xcb_query_extension_reply_t *dri3_reply, *pres_reply, *amd_reply, *nv_reply;
+   xcb_query_extension_reply_t *dri3_reply, *pres_reply, *randr_reply, *amd_reply, *nv_reply;
    bool has_dri3_v1_2 = false;
    bool has_present_v1_2 = false;
 
@@ -132,6 +178,7 @@ wsi_x11_connection_create(struct wsi_device *wsi_dev,
 
    dri3_cookie = xcb_query_extension(conn, 4, "DRI3");
    pres_cookie = xcb_query_extension(conn, 7, "Present");
+   randr_cookie = xcb_query_extension(conn, 5, "RANDR");
 
    /* We try to be nice to users and emit a warning if they try to use a
     * Vulkan application on a system without DRI3 enabled.  However, this ends
@@ -147,11 +194,13 @@ wsi_x11_connection_create(struct wsi_device *wsi_dev,
 
    dri3_reply = xcb_query_extension_reply(conn, dri3_cookie, NULL);
    pres_reply = xcb_query_extension_reply(conn, pres_cookie, NULL);
+   randr_reply = xcb_query_extension_reply(conn, randr_cookie, NULL);
    amd_reply = xcb_query_extension_reply(conn, amd_cookie, NULL);
    nv_reply = xcb_query_extension_reply(conn, nv_cookie, NULL);
    if (!dri3_reply || !pres_reply) {
       free(dri3_reply);
       free(pres_reply);
+      free(randr_reply);
       free(amd_reply);
       free(nv_reply);
       vk_free(&wsi_dev->instance_alloc, wsi_conn);
@@ -186,6 +235,11 @@ wsi_x11_connection_create(struct wsi_device *wsi_dev,
    }
 #endif
 
+   if (randr_reply && randr_reply->present != 0)
+   {
+      wsi_conn->is_xwayland = wsi_x11_detect_xwayland(conn);
+   }
+
    wsi_conn->has_dri3_modifiers = has_dri3_v1_2 && has_present_v1_2;
    wsi_conn->is_proprietary_x11 = false;
    if (amd_reply && amd_reply->present)