From 23409aea73d7d750d1f45874bd0e8037048e7920 Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Mon, 30 Jan 2023 20:12:01 -0600 Subject: [PATCH] nvk: Events Part-of: --- src/nouveau/vulkan/meson.build | 2 + src/nouveau/vulkan/nvk_device.c | 9 ++ src/nouveau/vulkan/nvk_device.h | 1 + src/nouveau/vulkan/nvk_event.c | 229 ++++++++++++++++++++++++++++ src/nouveau/vulkan/nvk_event.h | 20 +++ src/nouveau/vulkan/nvk_query_pool.c | 68 +-------- 6 files changed, 262 insertions(+), 67 deletions(-) create mode 100644 src/nouveau/vulkan/nvk_event.c create mode 100644 src/nouveau/vulkan/nvk_event.h diff --git a/src/nouveau/vulkan/meson.build b/src/nouveau/vulkan/meson.build index 148334c22e6..c04cc1e882a 100644 --- a/src/nouveau/vulkan/meson.build +++ b/src/nouveau/vulkan/meson.build @@ -26,6 +26,8 @@ nvk_files = files( 'nvk_device.h', 'nvk_device_memory.c', 'nvk_device_memory.h', + 'nvk_event.c', + 'nvk_event.h', 'nvk_format.c', 'nvk_format.h', 'nvk_graphics_pipeline.c', diff --git a/src/nouveau/vulkan/nvk_device.c b/src/nouveau/vulkan/nvk_device.c index 8db3aefa996..e1680ff3ccd 100644 --- a/src/nouveau/vulkan/nvk_device.c +++ b/src/nouveau/vulkan/nvk_device.c @@ -177,6 +177,12 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice, if (result != VK_SUCCESS) goto fail_samplers; + result = nvk_heap_init(device, &device->event_heap, + NOUVEAU_WS_BO_LOCAL, NOUVEAU_WS_BO_WR, + 0 /* overalloc */); + if (result != VK_SUCCESS) + goto fail_shader_heap; + nvk_slm_area_init(&device->slm); if (pthread_mutex_init(&device->mutex, NULL) != 0) { @@ -236,6 +242,8 @@ fail_mutex: pthread_mutex_destroy(&device->mutex); fail_slm: nvk_slm_area_finish(&device->slm); + nvk_heap_finish(device, &device->event_heap); +fail_shader_heap: nvk_heap_finish(device, &device->shader_heap); fail_samplers: nvk_descriptor_table_finish(device, &device->samplers); @@ -267,6 +275,7 @@ nvk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator) nouveau_ws_bo_destroy(device->zero_page); vk_device_finish(&device->vk); nvk_slm_area_finish(&device->slm); + nvk_heap_finish(device, &device->event_heap); nvk_heap_finish(device, &device->shader_heap); nvk_descriptor_table_finish(device, &device->samplers); nvk_descriptor_table_finish(device, &device->images); diff --git a/src/nouveau/vulkan/nvk_device.h b/src/nouveau/vulkan/nvk_device.h index 3a820020f10..e86bbbb9c98 100644 --- a/src/nouveau/vulkan/nvk_device.h +++ b/src/nouveau/vulkan/nvk_device.h @@ -37,6 +37,7 @@ struct nvk_device { struct nvk_descriptor_table images; struct nvk_descriptor_table samplers; struct nvk_heap shader_heap; + struct nvk_heap event_heap; struct nvk_slm_area slm; struct nouveau_ws_bo *zero_page; diff --git a/src/nouveau/vulkan/nvk_event.c b/src/nouveau/vulkan/nvk_event.c new file mode 100644 index 00000000000..9eda0753b44 --- /dev/null +++ b/src/nouveau/vulkan/nvk_event.c @@ -0,0 +1,229 @@ +#include "nvk_event.h" + +#include "nvk_cmd_buffer.h" +#include "nvk_device.h" +#include "nvk_mme.h" + +#include "nvk_cl906f.h" +#include "nvk_cl9097.h" + +#define NVK_EVENT_MEM_SIZE sizeof(VkResult) + +VKAPI_ATTR VkResult VKAPI_CALL +nvk_CreateEvent(VkDevice device, + const VkEventCreateInfo *pCreateInfo, + const VkAllocationCallbacks *pAllocator, + VkEvent *pEvent) +{ + VK_FROM_HANDLE(nvk_device, dev, device); + struct nvk_event *event; + VkResult result; + + event = vk_object_zalloc(&dev->vk, pAllocator, sizeof(*event), + VK_OBJECT_TYPE_EVENT); + if (!event) + return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY); + + result = nvk_heap_alloc(dev, &dev->event_heap, + NVK_EVENT_MEM_SIZE, NVK_EVENT_MEM_SIZE, + &event->addr, (void **)&event->status); + if (result != VK_SUCCESS) { + vk_object_free(&dev->vk, pAllocator, event); + return result; + } + + *event->status = VK_EVENT_RESET; + + *pEvent = nvk_event_to_handle(event); + + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL +nvk_DestroyEvent(VkDevice device, + VkEvent _event, + const VkAllocationCallbacks *pAllocator) +{ + VK_FROM_HANDLE(nvk_device, dev, device); + VK_FROM_HANDLE(nvk_event, event, _event); + + if (!event) + return; + + nvk_heap_free(dev, &dev->event_heap, event->addr, NVK_EVENT_MEM_SIZE); + + vk_object_free(&dev->vk, pAllocator, event); +} + +VKAPI_ATTR VkResult VKAPI_CALL +nvk_GetEventStatus(VkDevice device, + VkEvent _event) +{ + VK_FROM_HANDLE(nvk_event, event, _event); + + return *event->status; +} + +VKAPI_ATTR VkResult VKAPI_CALL +nvk_SetEvent(VkDevice device, + VkEvent _event) +{ + VK_FROM_HANDLE(nvk_event, event, _event); + + *event->status = VK_EVENT_SET; + + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL +nvk_ResetEvent(VkDevice device, + VkEvent _event) +{ + VK_FROM_HANDLE(nvk_event, event, _event); + + *event->status = VK_EVENT_RESET; + + return VK_SUCCESS; +} + +static bool +clear_bits64(uint64_t *bitfield, uint64_t bits) +{ + bool has_bits = (*bitfield & bits) != 0; + *bitfield &= ~bits; + return has_bits; +} + +uint32_t +vk_stage_flags_to_nv9097_pipeline_location(VkPipelineStageFlags2 flags) +{ + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT_KHR | + VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT | + VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT | + VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT | + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT| + VK_PIPELINE_STAGE_2_COPY_BIT| + VK_PIPELINE_STAGE_2_RESOLVE_BIT| + VK_PIPELINE_STAGE_2_BLIT_BIT| + VK_PIPELINE_STAGE_2_CLEAR_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_ALL; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_DEPTH_TEST; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_PIXEL_SHADER; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_ZCULL; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TRANSFORM_FEEDBACK_BIT_EXT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_STREAMING_OUTPUT; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_GEOMETRY_SHADER_BIT | + VK_PIPELINE_STAGE_2_PRE_RASTERIZATION_SHADERS_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_GEOMETRY_SHADER; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TESSELLATION_EVALUATION_SHADER_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_TESSELATION_SHADER; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TESSELLATION_CONTROL_SHADER_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_TESSELATION_INIT_SHADER; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_VERTEX_SHADER; + + if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT | + VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT | + VK_PIPELINE_STAGE_2_VERTEX_ATTRIBUTE_INPUT_BIT)) + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_DATA_ASSEMBLER; + + clear_bits64(&flags, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT | + VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | + VK_PIPELINE_STAGE_2_CONDITIONAL_RENDERING_BIT_EXT); + + /* TODO: Doing this on 3D will likely cause a WFI which is probably ok but, + * if we tracked which subchannel we've used most recently, we can probably + * do better than that. + */ + clear_bits64(&flags, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + + assert(flags == 0); + + return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_NONE; +} + +VKAPI_ATTR void VKAPI_CALL +nvk_CmdSetEvent2(VkCommandBuffer commandBuffer, + VkEvent _event, + const VkDependencyInfo *pDependencyInfo) +{ + VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer); + VK_FROM_HANDLE(nvk_event, event, _event); + + VkPipelineStageFlags2 stages; + for (uint32_t i = 0; i < pDependencyInfo->memoryBarrierCount; i++) + stages |= pDependencyInfo->pMemoryBarriers[i].srcStageMask; + for (uint32_t i = 0; i < pDependencyInfo->bufferMemoryBarrierCount; i++) + stages |= pDependencyInfo->pBufferMemoryBarriers[i].srcStageMask; + for (uint32_t i = 0; i < pDependencyInfo->imageMemoryBarrierCount; i++) + stages |= pDependencyInfo->pImageMemoryBarriers[i].srcStageMask; + + struct nv_push *p = nvk_cmd_buffer_push(cmd, 5); + P_MTHD(p, NV9097, SET_REPORT_SEMAPHORE_A); + P_NV9097_SET_REPORT_SEMAPHORE_A(p, event->addr >> 32); + P_NV9097_SET_REPORT_SEMAPHORE_B(p, event->addr); + P_NV9097_SET_REPORT_SEMAPHORE_C(p, VK_EVENT_SET); + P_NV9097_SET_REPORT_SEMAPHORE_D(p, { + .operation = OPERATION_RELEASE, + .release = RELEASE_AFTER_ALL_PRECEEDING_WRITES_COMPLETE, + .pipeline_location = vk_stage_flags_to_nv9097_pipeline_location(stages), + .structure_size = STRUCTURE_SIZE_ONE_WORD, + }); +} + +VKAPI_ATTR void VKAPI_CALL +nvk_CmdResetEvent2(VkCommandBuffer commandBuffer, + VkEvent _event, + VkPipelineStageFlags2 stageMask) +{ + VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer); + VK_FROM_HANDLE(nvk_event, event, _event); + + struct nv_push *p = nvk_cmd_buffer_push(cmd, 5); + P_MTHD(p, NV9097, SET_REPORT_SEMAPHORE_A); + P_NV9097_SET_REPORT_SEMAPHORE_A(p, event->addr >> 32); + P_NV9097_SET_REPORT_SEMAPHORE_B(p, event->addr); + P_NV9097_SET_REPORT_SEMAPHORE_C(p, VK_EVENT_RESET); + P_NV9097_SET_REPORT_SEMAPHORE_D(p, { + .operation = OPERATION_RELEASE, + .release = RELEASE_AFTER_ALL_PRECEEDING_WRITES_COMPLETE, + .pipeline_location = + vk_stage_flags_to_nv9097_pipeline_location(stageMask), + .structure_size = STRUCTURE_SIZE_ONE_WORD, + }); +} + +VKAPI_ATTR void VKAPI_CALL +nvk_CmdWaitEvents2(VkCommandBuffer commandBuffer, + uint32_t eventCount, + const VkEvent *pEvents, + const VkDependencyInfo *pDependencyInfos) +{ + VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer); + + for (uint32_t i = 0; i < eventCount; i++) { + VK_FROM_HANDLE(nvk_event, event, pEvents[i]); + + struct nv_push *p = nvk_cmd_buffer_push(cmd, 5); + __push_mthd(p, SUBC_NV9097, NV906F_SEMAPHOREA); + P_NV906F_SEMAPHOREA(p, event->addr >> 32); + P_NV906F_SEMAPHOREB(p, (event->addr & UINT32_MAX) >> 2); + P_NV906F_SEMAPHOREC(p, VK_EVENT_SET); + P_NV906F_SEMAPHORED(p, { + .operation = OPERATION_ACQUIRE, + .acquire_switch = ACQUIRE_SWITCH_ENABLED, + .release_size = RELEASE_SIZE_4BYTE, + }); + } +} diff --git a/src/nouveau/vulkan/nvk_event.h b/src/nouveau/vulkan/nvk_event.h new file mode 100644 index 00000000000..4f68f5dde87 --- /dev/null +++ b/src/nouveau/vulkan/nvk_event.h @@ -0,0 +1,20 @@ +#ifndef NVK_EVENT_H +#define NVK_EVENT_H 1 + +#include "nvk_private.h" + +#include "vulkan/runtime/vk_object.h" + +struct nvk_event { + struct vk_object_base base; + + uint64_t addr; + VkResult *status; +}; + +VK_DEFINE_HANDLE_CASTS(nvk_event, base, VkEvent, VK_OBJECT_TYPE_EVENT) + +uint32_t +vk_stage_flags_to_nv9097_pipeline_location(VkPipelineStageFlags2 flags); + +#endif /* define NVK_EVENT_H */ diff --git a/src/nouveau/vulkan/nvk_query_pool.c b/src/nouveau/vulkan/nvk_query_pool.c index 3b8699f1ef6..a35da9af70c 100644 --- a/src/nouveau/vulkan/nvk_query_pool.c +++ b/src/nouveau/vulkan/nvk_query_pool.c @@ -3,6 +3,7 @@ #include "nvk_buffer.h" #include "nvk_cmd_buffer.h" #include "nvk_device.h" +#include "nvk_event.h" #include "nvk_mme.h" #include "nvk_physical_device.h" #include "util/os_time.h" @@ -185,73 +186,6 @@ nvk_CmdResetQueryPool(VkCommandBuffer commandBuffer, } } -static bool -clear_bits64(uint64_t *bitfield, uint64_t bits) -{ - bool has_bits = (*bitfield & bits) != 0; - *bitfield &= ~bits; - return has_bits; -} - -static uint32_t -vk_stage_flags_to_nv9097_pipeline_location(VkPipelineStageFlags2 flags) -{ - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT_KHR | - VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT | - VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT | - VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT | - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT| - VK_PIPELINE_STAGE_2_COPY_BIT| - VK_PIPELINE_STAGE_2_RESOLVE_BIT| - VK_PIPELINE_STAGE_2_BLIT_BIT| - VK_PIPELINE_STAGE_2_CLEAR_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_ALL; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_DEPTH_TEST; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_PIXEL_SHADER; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_ZCULL; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TRANSFORM_FEEDBACK_BIT_EXT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_STREAMING_OUTPUT; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_GEOMETRY_SHADER_BIT | - VK_PIPELINE_STAGE_2_PRE_RASTERIZATION_SHADERS_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_GEOMETRY_SHADER; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TESSELLATION_EVALUATION_SHADER_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_TESSELATION_SHADER; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TESSELLATION_CONTROL_SHADER_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_TESSELATION_INIT_SHADER; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_VERTEX_SHADER; - - if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT | - VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT | - VK_PIPELINE_STAGE_2_VERTEX_ATTRIBUTE_INPUT_BIT)) - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_DATA_ASSEMBLER; - - clear_bits64(&flags, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT | - VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | - VK_PIPELINE_STAGE_2_CONDITIONAL_RENDERING_BIT_EXT); - - /* TODO: Doing this on 3D will likely cause a WFI which is probably ok but, - * if we tracked which subchannel we've used most recently, we can probably - * do better than that. - */ - clear_bits64(&flags, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); - - assert(flags == 0); - - return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_NONE; -} - VKAPI_ATTR void VKAPI_CALL nvk_CmdWriteTimestamp2(VkCommandBuffer commandBuffer, VkPipelineStageFlags2 stage,