diff --git a/src/virtio/vulkan/vn_common.c b/src/virtio/vulkan/vn_common.c index 95da1b3d2cf..73434071d47 100644 --- a/src/virtio/vulkan/vn_common.c +++ b/src/virtio/vulkan/vn_common.c @@ -241,9 +241,81 @@ vn_relax(struct vn_relax_state *state) os_time_sleep(base_sleep_us << shift); } +struct vn_ring * +vn_tls_get_ring(struct vn_instance *instance) +{ + if (VN_PERF(NO_MULTI_RING)) + return instance->ring.ring; + + struct vn_tls *tls = vn_tls_get(); + if (unlikely(!tls)) { + /* only allow to fallback on missing tls */ + return instance->ring.ring; + } + + /* look up tls_ring owned by instance */ + list_for_each_entry(struct vn_tls_ring, tls_ring, &tls->tls_rings, + tls_head) { + mtx_lock(&tls_ring->mutex); + if (tls_ring->instance == instance) { + mtx_unlock(&tls_ring->mutex); + assert(tls_ring->ring); + return tls_ring->ring; + } + mtx_unlock(&tls_ring->mutex); + } + + struct vn_tls_ring *tls_ring = calloc(1, sizeof(*tls_ring)); + if (!tls_ring) + return NULL; + + /* keep the extra for potential roundtrip sync on tls ring */ + static const size_t extra_size = sizeof(uint32_t); + + /* only need a small ring for synchronous cmds on tls ring */ + static const size_t buf_size = 16 * 1024; + + struct vn_ring_layout layout; + vn_ring_get_layout(buf_size, extra_size, &layout); + + tls_ring->ring = vn_ring_create(instance, &layout); + if (!tls_ring->ring) { + free(tls_ring); + return NULL; + } + + mtx_init(&tls_ring->mutex, mtx_plain); + tls_ring->instance = instance; + list_add(&tls_ring->tls_head, &tls->tls_rings); + list_add(&tls_ring->vk_head, &instance->ring.tls_rings); + + return tls_ring->ring; +} + +void +vn_tls_destroy_ring(struct vn_tls_ring *tls_ring) +{ + mtx_lock(&tls_ring->mutex); + if (tls_ring->ring) { + vn_ring_destroy(tls_ring->ring); + tls_ring->ring = NULL; + tls_ring->instance = NULL; + mtx_unlock(&tls_ring->mutex); + } else { + mtx_unlock(&tls_ring->mutex); + mtx_destroy(&tls_ring->mutex); + free(tls_ring); + } +} + static void vn_tls_free(void *tls) { + if (tls) { + list_for_each_entry_safe(struct vn_tls_ring, tls_ring, + &((struct vn_tls *)tls)->tls_rings, tls_head) + vn_tls_destroy_ring(tls_ring); + } free(tls); } @@ -271,9 +343,17 @@ vn_tls_get(void) return tls; tls = calloc(1, sizeof(*tls)); - if (tls && tss_set(vn_tls_key, tls) == thrd_success) - return tls; + if (!tls) + return NULL; - free(tls); - return NULL; + /* initialize tls */ + tls->async_pipeline_create = false; + list_inithead(&tls->tls_rings); + + if (tss_set(vn_tls_key, tls) != thrd_success) { + free(tls); + return NULL; + } + + return tls; } diff --git a/src/virtio/vulkan/vn_common.h b/src/virtio/vulkan/vn_common.h index c802b28493e..215a7f97940 100644 --- a/src/virtio/vulkan/vn_common.h +++ b/src/virtio/vulkan/vn_common.h @@ -210,11 +210,29 @@ struct vn_relax_state { const char *reason; }; +/* TLS ring + * - co-owned by TLS and VkInstance + * - initialized in TLS upon requested + * - teardown happens upon thread exit or instance destroy + * - teardown is split into 2 stages: + * 1. one owner locks and destroys the ring and mark destroyed + * 2. the other owner locks and frees up the tls ring storage + */ +struct vn_tls_ring { + mtx_t mutex; + struct vn_ring *ring; + struct vn_instance *instance; + struct list_head tls_head; + struct list_head vk_head; +}; + struct vn_tls { /* Track the threads on which swapchain and command pool creations occur. * Pipeline create on those threads are forced async via the primary ring. */ bool async_pipeline_create; + /* Track TLS rings owned across instances. */ + struct list_head tls_rings; }; void @@ -501,4 +519,10 @@ vn_tls_get_async_pipeline_create(void) return true; } +struct vn_ring * +vn_tls_get_ring(struct vn_instance *instance); + +void +vn_tls_destroy_ring(struct vn_tls_ring *tls_ring); + #endif /* VN_COMMON_H */ diff --git a/src/virtio/vulkan/vn_instance.c b/src/virtio/vulkan/vn_instance.c index 4870b05a4e0..06627aeeb5b 100644 --- a/src/virtio/vulkan/vn_instance.c +++ b/src/virtio/vulkan/vn_instance.c @@ -118,6 +118,10 @@ vn_instance_fini_ring(struct vn_instance *instance) vn_watchdog_fini(&instance->ring.watchdog); + list_for_each_entry_safe(struct vn_tls_ring, tls_ring, + &instance->ring.tls_rings, vk_head) + vn_tls_destroy_ring(tls_ring); + vn_ring_destroy(instance->ring.ring); } @@ -133,6 +137,8 @@ vn_instance_init_ring(struct vn_instance *instance) if (!instance->ring.ring) return VK_ERROR_OUT_OF_HOST_MEMORY; + list_inithead(&instance->ring.tls_rings); + vn_watchdog_init(&instance->ring.watchdog); mtx_init(&instance->ring.roundtrip_mutex, mtx_plain); diff --git a/src/virtio/vulkan/vn_instance.h b/src/virtio/vulkan/vn_instance.h index 57095e07f04..9b611d20e43 100644 --- a/src/virtio/vulkan/vn_instance.h +++ b/src/virtio/vulkan/vn_instance.h @@ -49,6 +49,7 @@ struct vn_instance { struct { struct vn_ring *ring; + struct list_head tls_rings; /* to synchronize renderer/ring */ mtx_t roundtrip_mutex; diff --git a/src/virtio/vulkan/vn_pipeline.c b/src/virtio/vulkan/vn_pipeline.c index 375b47ef0f1..dd97c49a6cd 100644 --- a/src/virtio/vulkan/vn_pipeline.c +++ b/src/virtio/vulkan/vn_pipeline.c @@ -449,29 +449,25 @@ vn_DestroyPipelineCache(VkDevice device, static struct vn_ring * vn_get_target_ring(struct vn_device *dev) { - if (VN_PERF(NO_MULTI_RING)) - return dev->primary_ring; - if (vn_tls_get_async_pipeline_create()) return dev->primary_ring; - if (!dev->secondary_ring) { - if (!vn_device_secondary_ring_init_once(dev)) { - /* fallback to primary ring submission */ - return dev->primary_ring; - } - } + struct vn_ring *ring = vn_tls_get_ring(dev->instance); + if (!ring) + return NULL; - /* Ensure pipeline cache and pipeline deps are ready in the renderer. - * - * TODO: - * - For cache retrieval, track ring seqno of cache obj and only wait - * for that seqno once. - * - For pipeline creation, track ring seqnos of pipeline layout and - * renderpass objs it depends on, and only wait for those seqnos once. - */ - vn_ring_wait_all(dev->primary_ring); - return dev->secondary_ring; + if (ring != dev->primary_ring) { + /* Ensure pipeline create dependencies are ready on the renderer side. + * + * TODO: + * - For pipeline objects, avoid object id re-use between async pipeline + * destroy on the primary ring and sync pipeline create on TLS ring. + * - For pipeline create, track ring seqnos of layout and renderpass + * objects it depends on, and only wait for those seqnos once. + */ + vn_ring_wait_all(dev->primary_ring); + } + return ring; } VkResult @@ -1546,8 +1542,14 @@ vn_CreateGraphicsPipelines(VkDevice device, } struct vn_ring *target_ring = vn_get_target_ring(dev); - assert(target_ring); - if (want_sync || target_ring == dev->secondary_ring) { + if (!target_ring) { + vk_free(alloc, fix_tmp); + vn_destroy_failed_pipelines(dev, createInfoCount, pPipelines, alloc); + STACK_ARRAY_FINISH(fix_descs); + return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + } + + if (want_sync || target_ring != dev->primary_ring) { result = vn_call_vkCreateGraphicsPipelines( target_ring, device, pipelineCache, createInfoCount, pCreateInfos, NULL, pPipelines); @@ -1602,8 +1604,12 @@ vn_CreateComputePipelines(VkDevice device, } struct vn_ring *target_ring = vn_get_target_ring(dev); - assert(target_ring); - if (want_sync || target_ring == dev->secondary_ring) { + if (!target_ring) { + vn_destroy_failed_pipelines(dev, createInfoCount, pPipelines, alloc); + return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY); + } + + if (want_sync || target_ring != dev->primary_ring) { result = vn_call_vkCreateComputePipelines( target_ring, device, pipelineCache, createInfoCount, pCreateInfos, NULL, pPipelines);