From 945564e49866f19af267c9d634fc8a4a807ec604 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Roberto=20de=20Souza?= <jose.souza@intel.com>
Date: Fri, 12 Jul 2024 07:36:47 -0700
Subject: [PATCH] anv: Wait for Xe exec queue to be idle before destroying it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Paulo reported that Vulkan is also affected by the drop of permanent
exec queues in Xe KMD, Iris already have this handling.

So here using the special DRM_IOCTL_XE_EXEC with num_batch_buffer == 0
to get a syncobj signaled when the last DRM_IOCTL_XE_EXEC is completed.

Reviewed-by: Paulo Zanoni <paulo.r.zanoni@intel.com>
Signed-off-by: José Roberto de Souza <jose.souza@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30156>
---
 src/intel/vulkan/xe/anv_queue.c | 59 +++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/src/intel/vulkan/xe/anv_queue.c b/src/intel/vulkan/xe/anv_queue.c
index ac043a40758..fc6a6dc8bf7 100644
--- a/src/intel/vulkan/xe/anv_queue.c
+++ b/src/intel/vulkan/xe/anv_queue.c
@@ -145,12 +145,71 @@ anv_xe_create_engine(struct anv_device *device,
    return result;
 }
 
+/*
+ * Wait for all previous DRM_IOCTL_XE_EXEC calls over the
+ * drm_xe_exec_queue to complete.
+ **/
+static void
+anv_xe_wait_exec_queue_idle(struct anv_device *device, uint32_t exec_queue_id)
+{
+   struct drm_syncobj_create syncobj_create = {};
+   struct drm_xe_sync xe_sync = {
+      .type = DRM_XE_SYNC_TYPE_SYNCOBJ,
+      .flags = DRM_XE_SYNC_FLAG_SIGNAL,
+   };
+   struct drm_xe_exec exec = {
+      .exec_queue_id = exec_queue_id,
+      .num_syncs = 1,
+      .syncs = (uintptr_t)&xe_sync,
+      .num_batch_buffer = 0,
+   };
+   struct drm_syncobj_destroy syncobj_destroy = {};
+   struct drm_syncobj_wait syncobj_wait = {
+      .count_handles = 1,
+      .timeout_nsec = INT64_MAX,
+   };
+   int fd = device->fd;
+   int ret;
+
+   ret = intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &syncobj_create);
+   assert(ret == 0);
+   if (ret)
+      return;
+
+   xe_sync.handle = syncobj_create.handle;
+   /* Using the special exec.num_batch_buffer == 0 handling to get syncobj
+    * signaled when the last DRM_IOCTL_XE_EXEC is completed.
+    */
+   ret = intel_ioctl(fd, DRM_IOCTL_XE_EXEC, &exec);
+   if (ret) {
+      /* exec_queue could have been banned, that is why it is being destroyed
+       * so no assert() here
+       */
+      goto error_exec;
+   }
+
+   syncobj_wait.handles = (uintptr_t)&syncobj_create.handle;
+   ret = intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_WAIT, &syncobj_wait);
+   assert(ret == 0);
+
+error_exec:
+   syncobj_destroy.handle = syncobj_create.handle;
+   ret = intel_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &syncobj_destroy);
+   assert(ret == 0);
+}
+
 static void
 destroy_engine(struct anv_device *device, uint32_t exec_queue_id)
 {
    struct drm_xe_exec_queue_destroy destroy = {
       .exec_queue_id = exec_queue_id,
    };
+
+   /* Application could submit a workload and before it is done, destroy the
+    * queue causing job timeouts in Xe KMD as it don't have permanent
+    * exec queues.
+    */
+   anv_xe_wait_exec_queue_idle(device, exec_queue_id);
    intel_ioctl(device->fd, DRM_IOCTL_XE_EXEC_QUEUE_DESTROY, &destroy);
 }