From c8fd0298e42d00cfe0251ee5ed79de9c50b5caae Mon Sep 17 00:00:00 2001
From: Konstantin Seurer <konstantin.seurer@gmail.com>
Date: Sat, 21 Sep 2024 13:33:09 +0200
Subject: [PATCH] vulkan: Add utilities for triggering renderdoc captures

This can be useful for adding custom triggers for renderdoc.

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31300>
---
 src/vulkan/runtime/vk_instance.c | 44 ++++++++++++++++++++++++++++++++
 src/vulkan/runtime/vk_instance.h | 11 ++++++++
 2 files changed, 55 insertions(+)

diff --git a/src/vulkan/runtime/vk_instance.c b/src/vulkan/runtime/vk_instance.c
index ff72f595ec3..1f21fb08509 100644
--- a/src/vulkan/runtime/vk_instance.c
+++ b/src/vulkan/runtime/vk_instance.c
@@ -38,6 +38,10 @@
 #include "compiler/glsl_types.h"
 #endif
 
+#ifndef _WIN32
+#include "dlfcn.h"
+#endif
+
 #define VERSION_IS_1_0(version) \
    (VK_API_VERSION_MAJOR(version) == 1 && VK_API_VERSION_MINOR(version) == 0)
 
@@ -206,6 +210,8 @@ vk_instance_init(struct vk_instance *instance,
       instance->trace_trigger_file = secure_getenv("MESA_VK_TRACE_TRIGGER");
    }
 
+   simple_mtx_init(&instance->renderdoc_mtx, mtx_plain);
+
 #if !VK_LITE_RUNTIME_INSTANCE
    glsl_type_singleton_init_or_ref();
 #endif
@@ -232,6 +238,8 @@ vk_instance_finish(struct vk_instance *instance)
    glsl_type_singleton_decref();
 #endif
 
+   simple_mtx_destroy(&instance->renderdoc_mtx);
+
    if (unlikely(!list_is_empty(&instance->debug_utils.callbacks))) {
       list_for_each_entry_safe(struct vk_debug_utils_messenger, messenger,
                                &instance->debug_utils.callbacks, link) {
@@ -645,3 +653,39 @@ vk_icdNegotiateLoaderICDInterfaceVersion(uint32_t *pSupportedVersion)
    *pSupportedVersion = vk_icd_version;
    return VK_SUCCESS;
 }
+
+void
+vk_instance_start_renderdoc_capture(struct vk_instance *instance)
+{
+#ifndef _WIN32
+   simple_mtx_lock(&instance->renderdoc_mtx);
+
+   if (!instance->renderdoc_api) {
+      void *renderdoc = dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
+      pRENDERDOC_GetAPI get_api = dlsym(renderdoc, "RENDERDOC_GetAPI");
+      get_api(eRENDERDOC_API_Version_1_0_0, (void *)&instance->renderdoc_api);
+
+      instance->renderdoc_api->SetActiveWindow(
+         RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(vk_instance_to_handle(instance)), NULL);
+   }
+
+   if (!instance->renderdoc_api->IsFrameCapturing())
+      instance->renderdoc_api->StartFrameCapture(NULL, NULL);
+
+   simple_mtx_unlock(&instance->renderdoc_mtx);
+#endif
+}
+
+void
+vk_instance_end_renderdoc_capture(struct vk_instance *instance)
+{
+   if (!instance->renderdoc_api)
+      return;
+
+   simple_mtx_lock(&instance->renderdoc_mtx);
+
+   if (instance->renderdoc_api->IsFrameCapturing())
+      instance->renderdoc_api->EndFrameCapture(NULL, NULL);
+
+   simple_mtx_unlock(&instance->renderdoc_mtx);
+}
diff --git a/src/vulkan/runtime/vk_instance.h b/src/vulkan/runtime/vk_instance.h
index 1ec1fea768e..9cb5fa81e9a 100644
--- a/src/vulkan/runtime/vk_instance.h
+++ b/src/vulkan/runtime/vk_instance.h
@@ -29,8 +29,11 @@
 
 #include "c11/threads.h"
 #include "util/list.h"
+#include "util/simple_mtx.h"
 #include "util/u_debug.h"
 
+#include "renderdoc_app.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -176,6 +179,10 @@ struct vk_instance {
 
    /** Whether the capture mode is per-submit. */
    bool trace_per_submit;
+
+   /** For triggering renderdoc captures from inside the driver. */
+   simple_mtx_t renderdoc_mtx;
+   RENDERDOC_API_1_0_0 *renderdoc_api;
 };
 
 VK_DEFINE_HANDLE_CASTS(vk_instance, base, VkInstance,
@@ -249,6 +256,10 @@ vk_instance_add_driver_trace_modes(struct vk_instance *instance,
 uint32_t
 vk_get_negotiated_icd_version(void);
 
+void vk_instance_start_renderdoc_capture(struct vk_instance *instance);
+
+void vk_instance_end_renderdoc_capture(struct vk_instance *instance);
+
 #ifdef __cplusplus
 }
 #endif