venus: roundtrip now belongs to ring

A roundtrip is to ensure a cmd via virtqueue happens on the renderer
side before a ring relies on it. Since venus is now with multi-ring, the
roundtrip submit and wait should belong to ring instead of instance, and
each ring owns its own roundtrip seqno to synchronize with virtqueue.

No behavior change.

Signed-off-by: Yiwei Zhang <zzyiwei@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/28147>

src/virtio/vulkan/vn_device_memory.c

@@ -17,8 +17,9 @@
 #include "vn_buffer.h"
 #include "vn_device.h"
 #include "vn_image.h"
-#include "vn_instance.h"
 #include "vn_physical_device.h"
+#include "vn_renderer.h"
+#include "vn_renderer_util.h"
 
 /* device memory commands */
 
@@ -131,7 +132,7 @@ vn_device_memory_pool_grow_alloc(struct vn_device *dev,
       goto mem_free;
 
    result =
-      vn_instance_submit_roundtrip(dev->instance, &mem->bo_roundtrip_seqno);
+      vn_ring_submit_roundtrip(dev->primary_ring, &mem->bo_roundtrip_seqno);
    if (result != VK_SUCCESS)
       goto bo_unref;
 
@@ -171,7 +172,7 @@ vn_device_memory_pool_unref(struct vn_device *dev,
 
    /* wait on valid bo_roundtrip_seqno before vkFreeMemory */
    if (pool_mem->bo_roundtrip_seqno_valid)
-      vn_instance_wait_roundtrip(dev->instance, pool_mem->bo_roundtrip_seqno);
+      vn_ring_wait_roundtrip(dev->primary_ring, pool_mem->bo_roundtrip_seqno);
 
    vn_device_memory_free_simple(dev, pool_mem);
    vk_device_memory_destroy(&dev->base.base, NULL, &pool_mem->base.base);
@@ -320,7 +321,7 @@ vn_device_memory_import_dma_buf(struct vn_device *dev,
    if (result != VK_SUCCESS)
       return result;
 
-   vn_instance_roundtrip(dev->instance);
+   vn_ring_roundtrip(dev->primary_ring);
 
    const VkImportMemoryResourceInfoMESA import_memory_resource_info = {
       .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_RESOURCE_INFO_MESA,
@@ -384,7 +385,7 @@ vn_device_memory_alloc_guest_vram(struct vn_device *dev,
       .memoryTypeIndex = alloc_info->memoryTypeIndex,
    };
 
-   vn_instance_roundtrip(dev->instance);
+   vn_ring_roundtrip(dev->primary_ring);
 
    result = vn_device_memory_alloc_simple(dev, mem, &memory_allocate_info);
    if (result != VK_SUCCESS) {
@@ -411,7 +412,7 @@ vn_device_memory_alloc_export(struct vn_device *dev,
    }
 
    result =
-      vn_instance_submit_roundtrip(dev->instance, &mem->bo_roundtrip_seqno);
+      vn_ring_submit_roundtrip(dev->primary_ring, &mem->bo_roundtrip_seqno);
    if (result != VK_SUCCESS) {
       vn_renderer_bo_unref(dev->renderer, mem->base_bo);
       vn_device_memory_free_simple(dev, mem);
@@ -629,7 +630,7 @@ vn_FreeMemory(VkDevice device,
       vn_device_memory_bo_fini(dev, mem);
 
       if (mem->bo_roundtrip_seqno_valid)
-         vn_instance_wait_roundtrip(dev->instance, mem->bo_roundtrip_seqno);
+         vn_ring_wait_roundtrip(dev->primary_ring, mem->bo_roundtrip_seqno);
 
       vn_device_memory_free_simple(dev, mem);
    }
@@ -688,8 +689,8 @@ vn_MapMemory(VkDevice device,
    if (!ptr) {
       /* vn_renderer_bo_map implies a roundtrip on success, but not here. */
       if (need_bo) {
-         result = vn_instance_submit_roundtrip(dev->instance,
-                                               &mem->bo_roundtrip_seqno);
+         result = vn_ring_submit_roundtrip(dev->primary_ring,
+                                           &mem->bo_roundtrip_seqno);
          if (result != VK_SUCCESS)
             return vn_error(dev->instance, result);
 
@@ -805,7 +806,7 @@ vn_get_memory_dma_buf_properties(struct vn_device *dev,
    if (result != VK_SUCCESS)
       return result;
 
-   vn_instance_roundtrip(dev->instance);
+   vn_ring_roundtrip(dev->primary_ring);
 
    VkMemoryResourceAllocationSizePropertiesMESA alloc_size_props = {
       .sType =

src/virtio/vulkan/vn_instance.c

@@ -115,8 +115,6 @@ vn_instance_init_renderer_versions(struct vn_instance *instance)
 static inline void
 vn_instance_fini_ring(struct vn_instance *instance)
 {
-   mtx_destroy(&instance->ring.roundtrip_mutex);
-
    vn_watchdog_fini(&instance->ring.watchdog);
 
    list_for_each_entry_safe(struct vn_tls_ring, tls_ring,
@@ -149,9 +147,6 @@ vn_instance_init_ring(struct vn_instance *instance)
 
    vn_watchdog_init(&instance->ring.watchdog);
 
-   mtx_init(&instance->ring.roundtrip_mutex, mtx_plain);
-   instance->ring.roundtrip_next = 1;
-
    return VK_SUCCESS;
 }
 
@@ -218,33 +213,6 @@ vn_instance_init_renderer(struct vn_instance *instance)
    return VK_SUCCESS;
 }
 
-VkResult
-vn_instance_submit_roundtrip(struct vn_instance *instance,
-                             uint64_t *roundtrip_seqno)
-{
-   const uint64_t ring_id = vn_ring_get_id(instance->ring.ring);
-   uint32_t local_data[8];
-   struct vn_cs_encoder local_enc =
-      VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
-
-   mtx_lock(&instance->ring.roundtrip_mutex);
-   const uint64_t seqno = instance->ring.roundtrip_next++;
-   vn_encode_vkSubmitVirtqueueSeqnoMESA(&local_enc, 0, ring_id, seqno);
-   VkResult result = vn_renderer_submit_simple(
-      instance->renderer, local_data, vn_cs_encoder_get_len(&local_enc));
-   mtx_unlock(&instance->ring.roundtrip_mutex);
-
-   *roundtrip_seqno = seqno;
-   return result;
-}
-
-void
-vn_instance_wait_roundtrip(struct vn_instance *instance,
-                           uint64_t roundtrip_seqno)
-{
-   vn_async_vkWaitVirtqueueSeqnoMESA(instance->ring.ring, roundtrip_seqno);
-}
-
 /* instance commands */
 
 VkResult

src/virtio/vulkan/vn_instance.h

@@ -13,10 +13,6 @@
 
 #include "vn_common.h"
 
-#include "venus-protocol/vn_protocol_driver_defines.h"
-
-#include "vn_cs.h"
-#include "vn_renderer.h"
 #include "vn_renderer_util.h"
 
 /* require and request at least Vulkan 1.1 at both instance and device levels
@@ -51,10 +47,6 @@ struct vn_instance {
       struct vn_ring *ring;
       struct list_head tls_rings;
 
-      /* to synchronize renderer/ring */
-      mtx_t roundtrip_mutex;
-      uint64_t roundtrip_next;
-
       struct vn_watchdog watchdog;
    } ring;
 
@@ -86,22 +78,6 @@ VK_DEFINE_HANDLE_CASTS(vn_instance,
                        VkInstance,
                        VK_OBJECT_TYPE_INSTANCE)
 
-VkResult
-vn_instance_submit_roundtrip(struct vn_instance *instance,
-                             uint64_t *roundtrip_seqno);
-
-void
-vn_instance_wait_roundtrip(struct vn_instance *instance,
-                           uint64_t roundtrip_seqno);
-
-static inline void
-vn_instance_roundtrip(struct vn_instance *instance)
-{
-   uint64_t roundtrip_seqno;
-   if (vn_instance_submit_roundtrip(instance, &roundtrip_seqno) == VK_SUCCESS)
-      vn_instance_wait_roundtrip(instance, roundtrip_seqno);
-}
-
 static inline struct vn_renderer_shmem *
 vn_instance_cs_shmem_alloc(struct vn_instance *instance,
                            size_t size,

src/virtio/vulkan/vn_ring.c

@@ -50,6 +50,10 @@ struct vn_ring {
 
    struct list_head submits;
    struct list_head free_submits;
+
+   /* to synchronize renderer/ring */
+   mtx_t roundtrip_mutex;
+   uint64_t roundtrip_next;
 };
 
 struct vn_ring_submit {
@@ -304,6 +308,9 @@ vn_ring_create(struct vn_instance *instance,
    list_inithead(&ring->submits);
    list_inithead(&ring->free_submits);
 
+   mtx_init(&ring->roundtrip_mutex, mtx_plain);
+   ring->roundtrip_next = 1;
+
    const struct VkRingMonitorInfoMESA monitor_info = {
       .sType = VK_STRUCTURE_TYPE_RING_MONITOR_INFO_MESA,
       .maxReportingPeriodMicroseconds = VN_WATCHDOG_REPORT_PERIOD_US,
@@ -347,6 +354,8 @@ vn_ring_destroy(struct vn_ring *ring)
    vn_renderer_submit_simple(ring->instance->renderer, destroy_ring_data,
                              vn_cs_encoder_get_len(&local_enc));
 
+   mtx_destroy(&ring->roundtrip_mutex);
+
    vn_ring_retire_submits(ring, ring->cur);
    assert(list_is_empty(&ring->submits));
 
@@ -682,3 +691,28 @@ vn_ring_free_command_reply(struct vn_ring *ring,
    assert(submit->reply_shmem);
    vn_renderer_shmem_unref(ring->instance->renderer, submit->reply_shmem);
 }
+
+VkResult
+vn_ring_submit_roundtrip(struct vn_ring *ring, uint64_t *roundtrip_seqno)
+{
+   uint32_t local_data[8];
+   struct vn_cs_encoder local_enc =
+      VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
+
+   mtx_lock(&ring->roundtrip_mutex);
+   const uint64_t seqno = ring->roundtrip_next++;
+   vn_encode_vkSubmitVirtqueueSeqnoMESA(&local_enc, 0, ring->id, seqno);
+   VkResult result =
+      vn_renderer_submit_simple(ring->instance->renderer, local_data,
+                                vn_cs_encoder_get_len(&local_enc));
+   mtx_unlock(&ring->roundtrip_mutex);
+
+   *roundtrip_seqno = seqno;
+   return result;
+}
+
+void
+vn_ring_wait_roundtrip(struct vn_ring *ring, uint64_t roundtrip_seqno)
+{
+   vn_async_vkWaitVirtqueueSeqnoMESA(ring, roundtrip_seqno);
+}

src/virtio/vulkan/vn_ring.h

@@ -120,4 +120,18 @@ VkResult
 vn_ring_submit_command_simple(struct vn_ring *ring,
                               const struct vn_cs_encoder *cs);
 
+VkResult
+vn_ring_submit_roundtrip(struct vn_ring *ring, uint64_t *roundtrip_seqno);
+
+void
+vn_ring_wait_roundtrip(struct vn_ring *ring, uint64_t roundtrip_seqno);
+
+static inline void
+vn_ring_roundtrip(struct vn_ring *ring)
+{
+   uint64_t roundtrip_seqno;
+   if (vn_ring_submit_roundtrip(ring, &roundtrip_seqno) == VK_SUCCESS)
+      vn_ring_wait_roundtrip(ring, roundtrip_seqno);
+}
+
 #endif /* VN_RING_H */