From d05b92d72003e6f2331dc05fc5e97afad3472e02 Mon Sep 17 00:00:00 2001 From: Karmjit Mahil Date: Mon, 5 May 2025 13:29:16 +0100 Subject: [PATCH] tu: Add "check_cmd_buffer_status" debug option Add a debug option which checks the status of a command buffer, if it has completed execution on the GPU, before it's reset or destroyed. This works by getting the GPU to write to a specific memory address on vkBeginCommandBuffer and vkEndCommandBuffer, so the CPU can check that the GPU has written the TU_CMD_BUFFER_STATUS_IDLE to the slot before actually resetting or freeing the command buffer. This can to help in debugging sync issues within the driver. Signed-off-by: Karmjit Mahil Part-of: --- src/freedreno/vulkan/tu_cmd_buffer.cc | 99 ++++++++++++++++++++++++++- src/freedreno/vulkan/tu_cmd_buffer.h | 3 + src/freedreno/vulkan/tu_util.cc | 1 + src/freedreno/vulkan/tu_util.h | 1 + 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/freedreno/vulkan/tu_cmd_buffer.cc b/src/freedreno/vulkan/tu_cmd_buffer.cc index 7d546c5c4a8..e6e8a5baf4e 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.cc +++ b/src/freedreno/vulkan/tu_cmd_buffer.cc @@ -10,6 +10,7 @@ #include "tu_cmd_buffer.h" #include "vk_common_entrypoints.h" +#include "vk_log.h" #include "vk_render_pass.h" #include "vk_util.h" @@ -18,11 +19,79 @@ #include "tu_cs.h" #include "tu_event.h" #include "tu_image.h" +#include "tu_knl.h" #include "tu_tracepoints.h" #include "common/freedreno_gpu_event.h" #include "common/freedreno_lrz.h" +enum tu_cmd_buffer_status { + TU_CMD_BUFFER_STATUS_IDLE = 0, + TU_CMD_BUFFER_STATUS_ACTIVE = 1, +}; + +static struct tu_bo * +tu_cmd_buffer_setup_status_tracking(struct tu_device *device) +{ + struct tu_bo *status_bo; + VkResult result; + + result = tu_bo_init_new_explicit_iova( + device, NULL, &status_bo, sizeof(enum tu_cmd_buffer_status), 0, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + TU_BO_ALLOC_INTERNAL_RESOURCE, "cmd_buffer_status"); + if (result != VK_SUCCESS) + return NULL; + + result = tu_bo_map(device, status_bo, NULL); + if (result != VK_SUCCESS) + return NULL; + + return status_bo; +} + +static VkResult +tu_cmd_buffer_status_check_idle(struct tu_cmd_buffer *cmd_buffer) +{ + if (cmd_buffer->status_bo == NULL) + return VK_SUCCESS; + + const enum tu_cmd_buffer_status status = + *(enum tu_cmd_buffer_status *)cmd_buffer->status_bo->map; + + switch (status) { + case TU_CMD_BUFFER_STATUS_IDLE: + return VK_SUCCESS; + + case TU_CMD_BUFFER_STATUS_ACTIVE: + mesa_loge("Trying to reset or destroy cmd_buffer %p while in use", + cmd_buffer); + return vk_errorf(cmd_buffer, VK_ERROR_UNKNOWN, + "Trying to reset or destroy while being used"); + default: + mesa_loge("Something went wrong with cmd_buffer status tracking"); + return vk_error(cmd_buffer, VK_ERROR_UNKNOWN); + } +} + +static inline void +tu_cmd_buffer_status_gpu_write(struct tu_cmd_buffer *cmd_buffer, + enum tu_cmd_buffer_status status) +{ + struct tu_cs *cs = &cmd_buffer->cs; + + if (cmd_buffer->status_bo == NULL) + return; + + static_assert(sizeof(uint32_t) == sizeof(status), + "Code below needs adjusting"); + tu_cs_emit_pkt7(cs, CP_MEM_WRITE, 3); + tu_cs_emit_qw(cs, cmd_buffer->status_bo->iova); + tu_cs_emit(cs, (uint32_t)status); +} + static void tu_clone_trace_range(struct tu_cmd_buffer *cmd, struct tu_cs *cs, struct u_trace_iterator begin, struct u_trace_iterator end) @@ -3000,6 +3069,14 @@ tu_create_cmd_buffer(struct vk_command_pool *pool, u_trace_init(&cmd_buffer->trace, &device->trace_context); list_inithead(&cmd_buffer->renderpass_autotune_results); + if (TU_DEBUG_ENV(CHECK_CMD_BUFFER_STATUS)) { + cmd_buffer->status_bo = tu_cmd_buffer_setup_status_tracking(device); + if (cmd_buffer->status_bo == NULL) { + mesa_logw("Failed creating cmd_buffer status_bo. " + "Won't track status for this cmd_buffer."); + } + } + tu_cs_init(&cmd_buffer->cs, device, TU_CS_MODE_GROW, 4096, "cmd cs"); tu_cs_init(&cmd_buffer->draw_cs, device, TU_CS_MODE_GROW, 4096, "draw cs"); tu_cs_init(&cmd_buffer->tile_store_cs, device, TU_CS_MODE_GROW, 2048, "tile store cs"); @@ -3030,6 +3107,12 @@ tu_cmd_buffer_destroy(struct vk_command_buffer *vk_cmd_buffer) tu_cs_finish(&cmd_buffer->pre_chain.draw_cs); tu_cs_finish(&cmd_buffer->pre_chain.draw_epilogue_cs); + if (TU_DEBUG_ENV(CHECK_CMD_BUFFER_STATUS)) { + tu_cmd_buffer_status_check_idle(cmd_buffer); + tu_bo_unmap(cmd_buffer->device, cmd_buffer->status_bo, false); + tu_bo_finish(cmd_buffer->device, cmd_buffer->status_bo); + } + u_trace_fini(&cmd_buffer->trace); tu_autotune_free_results(cmd_buffer->device, &cmd_buffer->renderpass_autotune_results); @@ -3059,7 +3142,16 @@ tu_reset_cmd_buffer(struct vk_command_buffer *vk_cmd_buffer, struct tu_cmd_buffer *cmd_buffer = container_of(vk_cmd_buffer, struct tu_cmd_buffer, vk); - vk_command_buffer_reset(&cmd_buffer->vk); + VkResult status_check_result = VK_SUCCESS; + if (TU_DEBUG_ENV(CHECK_CMD_BUFFER_STATUS)) + status_check_result = tu_cmd_buffer_status_check_idle(cmd_buffer); + + vk_command_buffer_reset(&cmd_buffer->vk); + + if (TU_DEBUG_ENV(CHECK_CMD_BUFFER_STATUS) && + status_check_result != VK_SUCCESS) { + cmd_buffer->vk.record_result = status_check_result; + } tu_cs_reset(&cmd_buffer->cs); tu_cs_reset(&cmd_buffer->draw_cs); @@ -3141,6 +3233,8 @@ tu_cmd_buffer_begin(struct tu_cmd_buffer *cmd_buffer, tu_cs_begin(&cmd_buffer->draw_cs); tu_cs_begin(&cmd_buffer->draw_epilogue_cs); + tu_cmd_buffer_status_gpu_write(cmd_buffer, TU_CMD_BUFFER_STATUS_ACTIVE); + return VK_SUCCESS; } @@ -4080,6 +4174,9 @@ tu_EndCommandBuffer(VkCommandBuffer commandBuffer) cmd_buffer->state.pass ? &cmd_buffer->draw_cs : &cmd_buffer->cs); } + if (TU_DEBUG_ENV(CHECK_CMD_BUFFER_STATUS)) + tu_cmd_buffer_status_gpu_write(cmd_buffer, TU_CMD_BUFFER_STATUS_IDLE); + tu_cs_end(&cmd_buffer->cs); tu_cs_end(&cmd_buffer->draw_cs); tu_cs_end(&cmd_buffer->draw_epilogue_cs); diff --git a/src/freedreno/vulkan/tu_cmd_buffer.h b/src/freedreno/vulkan/tu_cmd_buffer.h index d010c8991f4..032a5e84e25 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.h +++ b/src/freedreno/vulkan/tu_cmd_buffer.h @@ -585,6 +585,9 @@ struct tu_cmd_buffer struct tu_cmd_state state; uint32_t queue_family_index; + /* For TU_DEBUG_ENV(CHECK_CMD_BUFFER_STATUS) functionality. */ + struct tu_bo *status_bo; + uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4]; VkShaderStageFlags push_constant_stages; struct tu_descriptor_set meta_push_descriptors; diff --git a/src/freedreno/vulkan/tu_util.cc b/src/freedreno/vulkan/tu_util.cc index 36e6c6d561a..a5dc4e84e38 100644 --- a/src/freedreno/vulkan/tu_util.cc +++ b/src/freedreno/vulkan/tu_util.cc @@ -50,6 +50,7 @@ static const struct debug_control tu_debug_options[] = { { "nobinmerging", TU_DEBUG_NO_BIN_MERGING }, { "perfcraw", TU_DEBUG_PERFCRAW }, { "fdmoffset", TU_DEBUG_FDM_OFFSET }, + { "check_cmd_buffer_status", TU_DEBUG_CHECK_CMD_BUFFER_STATUS }, { NULL, 0 } }; diff --git a/src/freedreno/vulkan/tu_util.h b/src/freedreno/vulkan/tu_util.h index 5ebdcd26a9d..208b2631985 100644 --- a/src/freedreno/vulkan/tu_util.h +++ b/src/freedreno/vulkan/tu_util.h @@ -70,6 +70,7 @@ enum tu_debug_flags : uint64_t TU_DEBUG_NO_BIN_MERGING = BITFIELD64_BIT(29), TU_DEBUG_PERFCRAW = BITFIELD64_BIT(30), TU_DEBUG_FDM_OFFSET = BITFIELD64_BIT(31), + TU_DEBUG_CHECK_CMD_BUFFER_STATUS = BITFIELD64_BIT(32), }; struct tu_env {