gfxstream: add VulkanMapper

This adds logic for a singleton VulkanMapper to be used with
Kumquat.  The main goal is remove the Vulkano dependency on
virtgpu_kumquat, so it's more portable.

Reviewed-by: Marcin Radomski <dextero@google.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34884>

src/gfxstream/guest/meson.build

@@ -45,6 +45,7 @@ inc_vulkan_enc = include_directories('vulkan_enc')
 # Subdirectories #
 #================#
 subdir('iostream')
+subdir('vulkan-mapper')
 subdir('platform')
 subdir('GoldfishAddressSpace')
 subdir('connection-manager')

src/gfxstream/guest/platform/kumquat/VirtGpuKumquat.h

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include "GfxStreamVulkanMapper.h"
 #include "VirtGpu.h"
 
 // Blueprint and Meson builds place things differently
@@ -47,6 +48,10 @@ class VirtGpuKumquatResourceMapping : public VirtGpuResourceMapping {
    public:
     VirtGpuKumquatResourceMapping(VirtGpuResourcePtr blob, struct virtgpu_kumquat* virtGpu,
                                   uint8_t* ptr, uint64_t size);
+
+    VirtGpuKumquatResourceMapping(VirtGpuResourcePtr blob, struct virtgpu_kumquat* virtGpu,
+                                  struct VulkanMapperData& data, uint64_t size);
+
     ~VirtGpuKumquatResourceMapping(void);
 
     uint8_t* asRawPtr(void) override;
@@ -54,8 +59,9 @@ class VirtGpuKumquatResourceMapping : public VirtGpuResourceMapping {
    private:
     VirtGpuResourcePtr mBlob;
     struct virtgpu_kumquat* mVirtGpu = nullptr;
-    uint8_t* mPtr;
-    uint64_t mSize;
+    struct VulkanMapperData mVulkanData;
+    uint8_t* mPtr = nullptr;
+    uint64_t mSize = 0;
 };
 
 class VirtGpuKumquatDevice : public VirtGpuDevice {

src/gfxstream/guest/platform/kumquat/VirtGpuKumquatBlob.cpp

@@ -10,6 +10,7 @@
 #include <cerrno>
 #include <cstring>
 
+#include "GfxStreamVulkanMapper.h"
 #include "VirtGpuKumquat.h"
 #include "util/log.h"
 
@@ -37,19 +38,57 @@ uint64_t VirtGpuKumquatResource::getSize() const { return mSize; }
 
 VirtGpuResourceMappingPtr VirtGpuKumquatResource::createMapping() {
     int ret;
+    struct drm_kumquat_resource_info info = {};
+
     struct drm_kumquat_map map {
         .bo_handle = mBlobHandle, .ptr = NULL, .size = mSize,
     };
 
-    ret = virtgpu_kumquat_resource_map(mVirtGpu, &map);
-    if (ret < 0) {
-        mesa_loge("Mapping failed with %s for resource %u blob %u", strerror(errno),
-                  mResourceHandle, mBlobHandle);
-        return nullptr;
-    }
+    info.bo_handle = mBlobHandle;
+    ret = virtgpu_kumquat_resource_info(mVirtGpu, &info);
 
-    return std::make_shared<VirtGpuKumquatResourceMapping>(shared_from_this(), mVirtGpu,
-                                                           (uint8_t*)map.ptr, mSize);
+    if (info.vulkan_info.device_id.device_uuid[0]) {
+        struct drm_kumquat_resource_export resource_export = {};
+        struct DeviceId deviceId = {};
+
+        resource_export.bo_handle = mBlobHandle;
+
+        ret = virtgpu_kumquat_resource_export(mVirtGpu, &resource_export);
+        if (ret) {
+            mesa_loge("External memory export from kumquat failed: %s", strerror(errno));
+            return nullptr;
+        }
+
+        memcpy(&deviceId, &info.vulkan_info.device_id, sizeof(struct DeviceId));
+        std::optional<DeviceId> deviceIdOpt{deviceId};
+        auto mapper = GfxStreamVulkanMapper::getInstance(deviceIdOpt);
+        struct VulkanMapperData mapData = {0};
+
+        mapData.handle = resource_export.os_handle;
+        mapData.handleType = resource_export.handle_type;
+        mapData.memoryIdx = info.vulkan_info.memory_idx;
+        mapData.size = mSize;
+
+        ret = mapper->map(&mapData);
+        if (ret < 0) {
+            mesa_loge("Mapping failed with %s for resource %u blob %u", strerror(errno),
+                      mResourceHandle, mBlobHandle);
+            return nullptr;
+        }
+
+        return std::make_shared<VirtGpuKumquatResourceMapping>(shared_from_this(), mVirtGpu,
+                                                               mapData, mSize);
+    } else {
+        ret = virtgpu_kumquat_resource_map(mVirtGpu, &map);
+        if (ret < 0) {
+            mesa_loge("Mapping failed with %s for resource %u blob %u", strerror(errno),
+                      mResourceHandle, mBlobHandle);
+            return nullptr;
+        }
+
+        return std::make_shared<VirtGpuKumquatResourceMapping>(shared_from_this(), mVirtGpu,
+                                                               (uint8_t*)map.ptr, mSize);
+    }
 }
 
 int VirtGpuKumquatResource::exportBlob(struct VirtGpuExternalHandle& handle) {

src/gfxstream/guest/platform/kumquat/VirtGpuKumquatBlobMapping.cpp

@@ -11,10 +11,20 @@ VirtGpuKumquatResourceMapping::VirtGpuKumquatResourceMapping(VirtGpuResourcePtr
                                                              uint8_t* ptr, uint64_t size)
     : mBlob(blob), mVirtGpu(virtGpu), mPtr(ptr), mSize(size) {}
 
+VirtGpuKumquatResourceMapping::VirtGpuKumquatResourceMapping(VirtGpuResourcePtr blob,
+                                                             struct virtgpu_kumquat* virtGpu,
+                                                             VulkanMapperData& data, uint64_t size)
+    : mBlob(blob), mVirtGpu(virtGpu), mVulkanData(data), mPtr(data.ptr), mSize(size) {}
+
 VirtGpuKumquatResourceMapping::~VirtGpuKumquatResourceMapping(void) {
-    int32_t ret = virtgpu_kumquat_resource_unmap(mVirtGpu, mBlob->getBlobHandle());
-    if (ret) {
-        mesa_loge("failed to unmap buffer");
+    if (mVulkanData.ptr && mVulkanData.memory != VK_NULL_HANDLE) {
+        auto mapper = GfxStreamVulkanMapper::getInstance();
+        mapper->unmap(&mVulkanData);
+    } else {
+        int32_t ret = virtgpu_kumquat_resource_unmap(mVirtGpu, mBlob->getBlobHandle());
+        if (ret) {
+            mesa_loge("failed to unmap buffer");
+        }
     }
 }
 

src/gfxstream/guest/platform/kumquat/VirtGpuKumquatDevice.cpp

@@ -12,6 +12,7 @@
 #include <fstream>
 #include <string>
 
+#include "GfxStreamVulkanMapper.h"
 #include "VirtGpuKumquat.h"
 #include "util/log.h"
 

src/gfxstream/guest/platform/kumquat/meson.build

@@ -16,6 +16,8 @@ libplatform_virtgpu_kumquat = static_library(
    'platform_virtgpu_kumquat',
    files_libplatform_virtgpu_kumquat,
    cpp_args: gfxstream_guest_args,
-   include_directories: [inc_platform_virtgpu, inc_src],
+   include_directories: [inc_platform_virtgpu, inc_src, inc_gfxstream_vulkan_mapper,
+                         inc_vulkan_util, inc_include],
+   link_with: [libgfxstream_vulkan_mapper],
    dependencies: virtgpu_kumquat_dep,
 )

src/gfxstream/guest/vulkan-mapper/GfxStreamVulkanMapper.cpp

@@ -0,0 +1,268 @@
+/*
+ * Copyright 2025 Mesa3D authors
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "GfxStreamVulkanMapper.h"
+
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+#include <vector>
+
+#include "util/detect_os.h"
+#include "util/log.h"
+
+#if DETECT_OS_WINDOWS
+#include <io.h>
+#define VK_LIBNAME "vulkan-1.dll"
+#else
+#include <unistd.h>
+#if DETECT_OS_APPLE
+#define VK_LIBNAME "libvulkan.1.dylib"
+#elif DETECT_OS_ANDROID
+#define VK_LIBNAME "libvulkan.so"
+#else
+#define VK_LIBNAME "libvulkan.so.1"
+#endif
+#endif
+
+const char* VK_ICD_FILENAMES = "VK_ICD_FILENAMES";
+
+#define GET_PROC_ADDR_INSTANCE_LOCAL(x) \
+    PFN_vk##x vk_##x = (PFN_vk##x)vk_GetInstanceProcAddr(nullptr, "vk" #x)
+
+std::unique_ptr<GfxStreamVulkanMapper> sVkMapper;
+
+GfxStreamVulkanMapper::GfxStreamVulkanMapper() {}
+GfxStreamVulkanMapper::~GfxStreamVulkanMapper() {}
+
+uint32_t chooseGfxQueueFamily(vk_uncompacted_dispatch_table* vk, VkPhysicalDevice phys_dev) {
+    uint32_t family_idx = UINT32_MAX;
+    uint32_t nProps = 0;
+
+    vk->GetPhysicalDeviceQueueFamilyProperties(phys_dev, &nProps, NULL);
+    std::vector<VkQueueFamilyProperties> props(nProps, VkQueueFamilyProperties{});
+    vk->GetPhysicalDeviceQueueFamilyProperties(phys_dev, &nProps, props.data());
+    // Choose the first graphics queue.
+    for (uint32_t i = 0; i < nProps; ++i) {
+        if ((props[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) && props[i].queueCount > 0) {
+            family_idx = i;
+            break;
+        }
+    }
+
+    return family_idx;
+}
+
+bool GfxStreamVulkanMapper::initialize(DeviceId& deviceId) {
+    mLoaderLib = util_dl_open(VK_LIBNAME);
+    if (!mLoaderLib) {
+        mesa_loge("failed to get loader library");
+        return false;
+    }
+
+    VkResult res;
+
+    auto vk_GetInstanceProcAddr =
+        (PFN_vkGetInstanceProcAddr)util_dl_get_proc_address(mLoaderLib, "vkGetInstanceProcAddr");
+    auto vk_GetDeviceProcAddr =
+        (PFN_vkGetDeviceProcAddr)util_dl_get_proc_address(mLoaderLib, "vkGetDeviceProcAddr");
+
+    if (!vk_GetInstanceProcAddr || !vk_GetDeviceProcAddr) {
+        mesa_loge("failed to get devuce or instance ProcAddr");
+        return false;
+    }
+
+    std::vector<const char*> externalMemoryInstanceExtNames = {
+        VK_KHR_EXTERNAL_MEMORY_CAPABILITIES_EXTENSION_NAME,
+    };
+
+    std::vector<const char*> externalMemoryDeviceExtNames = {
+        VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME,
+#if DETECT_OS_WINDOWS
+        VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME,
+#elif DETECT_OS_LINUX
+        VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME,
+        VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME,
+#endif
+    };
+
+    VkInstanceCreateInfo instCi = {
+        VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, 0, 0, nullptr, 0, nullptr, 0, nullptr,
+    };
+
+    VkApplicationInfo appInfo = {
+        VK_STRUCTURE_TYPE_APPLICATION_INFO,
+        0,
+        "gfxstream_vk_mapper",
+        1,
+        "gfxstream_vk_mapper",
+        1,
+        VK_API_VERSION_1_1,
+    };
+
+    instCi.pApplicationInfo = &appInfo;
+    instCi.enabledExtensionCount = static_cast<uint32_t>(externalMemoryInstanceExtNames.size());
+    instCi.ppEnabledExtensionNames = externalMemoryInstanceExtNames.data();
+
+    GET_PROC_ADDR_INSTANCE_LOCAL(CreateInstance);
+    res = vk_CreateInstance(&instCi, nullptr, &mInstance);
+    if (res != VK_SUCCESS) {
+        mesa_loge("failed to create VkMapper Instance");
+        return false;
+    }
+
+    vk_instance_uncompacted_dispatch_table_load(&mVk.instance, vk_GetInstanceProcAddr, mInstance);
+    vk_physical_device_uncompacted_dispatch_table_load(&mVk.physical_device, vk_GetInstanceProcAddr,
+                                                       mInstance);
+
+    uint32_t physicalDeviceCount = 0;
+    std::vector<VkPhysicalDevice> physicalDevices;
+    res = mVk.EnumeratePhysicalDevices(mInstance, &physicalDeviceCount, nullptr);
+    if (res != VK_SUCCESS) {
+        mesa_loge("failed to get physical device count");
+        return false;
+    }
+
+    physicalDevices.resize(physicalDeviceCount);
+    res = mVk.EnumeratePhysicalDevices(mInstance, &physicalDeviceCount, physicalDevices.data());
+    if (res != VK_SUCCESS) {
+        mesa_loge("failed to enumerate physical devices");
+        return false;
+    }
+
+    for (uint32_t i = 0; i < physicalDeviceCount; i++) {
+        VkPhysicalDeviceIDProperties idProps = {
+            VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES_KHR,
+        };
+
+        VkPhysicalDeviceProperties2 deviceProps = {
+            .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2_KHR,
+            .pNext = reinterpret_cast<void*>(&idProps),
+        };
+
+        mVk.GetPhysicalDeviceProperties2(physicalDevices[i], &deviceProps);
+        if (memcmp(&idProps.deviceUUID[0], &deviceId.deviceUUID[0], VK_UUID_SIZE)) {
+            continue;
+        }
+
+        if (memcmp(&idProps.driverUUID[0], &deviceId.driverUUID[0], VK_UUID_SIZE)) {
+            continue;
+        }
+
+        VkPhysicalDevice physDev = physicalDevices[i];
+        uint32_t gfxQueueFamilyIdx = chooseGfxQueueFamily(&mVk, physDev);
+        if (gfxQueueFamilyIdx == UINT32_MAX) {
+            mesa_loge("failed to get gfx queue family idx");
+            return false;
+        }
+
+        float priority = 1.0f;
+        VkDeviceQueueCreateInfo dqCi = {
+            VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, 0, 0, gfxQueueFamilyIdx, 1, &priority,
+        };
+
+        VkDeviceCreateInfo dCi = {};
+        dCi.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
+        dCi.queueCreateInfoCount = 1;
+        dCi.pQueueCreateInfos = &dqCi;
+        dCi.enabledExtensionCount = static_cast<uint32_t>(externalMemoryDeviceExtNames.size());
+        dCi.ppEnabledExtensionNames = externalMemoryDeviceExtNames.data();
+        res = mVk.CreateDevice(physDev, &dCi, nullptr, &mDevice);
+        if (res != VK_SUCCESS) {
+            mesa_loge("failed to create device");
+            return false;
+        }
+    }
+
+    vk_device_uncompacted_dispatch_table_load(&mVk.device, vk_GetDeviceProcAddr, mDevice);
+
+    return true;
+}
+
+// The Tesla V-100 driver seems to enter a power management mode and stops being available to
+// the Vulkan loader if more than a certain number of VK instances are created in the same
+// process.
+//
+// This behavior is reproducible via:
+//
+// GfxstreamEnd2EndTests --gtest_filter="*MultiThreadedVkMapMemory*"
+//
+// Workaround this by having a singleton mapper per-process.
+GfxStreamVulkanMapper* GfxStreamVulkanMapper::getInstance(std::optional<DeviceId> deviceIdOpt) {
+    if (sVkMapper == nullptr && deviceIdOpt) {
+        // The idea is to make sure the gfxstream ICD isn't loaded when the mapper starts
+        // up. The Nvidia ICD should be loaded.
+        //
+        // This is mostly useful for developers.  For AOSP hermetic gfxstream end2end
+        // testing, VK_ICD_FILENAMES shouldn't be defined.  For deqp-vk, this is
+        // useful, but not safe for multi-threaded tests.  For now, since this is only
+        // used for end2end tests, we should be good.
+        const char* driver = getenv(VK_ICD_FILENAMES);
+        unsetenv(VK_ICD_FILENAMES);
+        sVkMapper = std::make_unique<GfxStreamVulkanMapper>();
+        if (!sVkMapper->initialize(*deviceIdOpt)) {
+            sVkMapper = nullptr;
+            return nullptr;
+        }
+
+        setenv(VK_ICD_FILENAMES, driver, 1);
+    }
+
+    return sVkMapper.get();
+}
+
+int32_t GfxStreamVulkanMapper::map(struct VulkanMapperData* mapData) {
+    VkMemoryAllocateInfo mai = {};
+
+    mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+    mai.allocationSize = mapData->size;
+    mai.memoryTypeIndex = mapData->memoryIdx;
+
+#if DETECT_OS_WINDOWS
+    VkImportMemoryWin32HandleInfoKHR importInfo{
+        VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR,
+        0,
+        VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT,
+        static_cast<HANDLE>(mapData->handle),
+        L"",
+    };
+
+#elif DETECT_OS_LINUX
+    // check for VIRTGPU_KUMQUAT_HANDLE_TYPE_MEM_DMABUF
+    VkExternalMemoryHandleTypeFlagBits flagBits = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
+    if (mapData->handleType == 0x1) {
+        flagBits = (enum VkExternalMemoryHandleTypeFlagBits)(
+            uint32_t(flagBits) | uint32_t(VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT));
+    }
+
+    VkImportMemoryFdInfoKHR importInfo{
+        VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
+        0,
+        flagBits,
+        static_cast<int>(mapData->handle),
+    };
+#endif
+
+    mai.pNext = reinterpret_cast<void*>(&importInfo);
+
+    VkResult result = mVk.AllocateMemory(mDevice, &mai, nullptr, &mapData->memory);
+    if (result != VK_SUCCESS) {
+        mesa_loge("failed to import memory");
+        return -EINVAL;
+    }
+
+    result = mVk.MapMemory(mDevice, mapData->memory, 0, mapData->size, 0, (void**)&mapData->ptr);
+    if (result != VK_SUCCESS) {
+        mesa_loge("failed to map memory");
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+void GfxStreamVulkanMapper::unmap(struct VulkanMapperData* mapData) {
+    mVk.UnmapMemory(mDevice, mapData->memory);
+    mVk.FreeMemory(mDevice, mapData->memory, nullptr);
+}

src/gfxstream/guest/vulkan-mapper/GfxStreamVulkanMapper.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef GFXSTREAM_VULKAN_MAPPER_H
+#define GFXSTREAM_VULKAN_MAPPER_H
+
+#include <memory>
+#include <optional>
+#include <unordered_map>
+
+#include "util/u_dl.h"
+#include "vk_dispatch_table.h"
+#include "vulkan/vulkan_core.h"
+
+struct VulkanMapperData {
+    // in
+    int64_t handle;
+    int32_t handleType;
+    uint32_t memoryIdx;
+    uint64_t size;
+
+    // out
+    VkDeviceMemory memory = VK_NULL_HANDLE;
+    uint8_t* ptr = nullptr;
+};
+
+struct DeviceId {
+    uint8_t deviceUUID[16];
+    uint8_t driverUUID[16];
+};
+
+class GfxStreamVulkanMapper {
+   public:
+    ~GfxStreamVulkanMapper();
+    GfxStreamVulkanMapper();
+
+    static GfxStreamVulkanMapper* getInstance(std::optional<DeviceId> deviceId = std::nullopt);
+    int32_t map(struct VulkanMapperData* mapData);
+    void unmap(struct VulkanMapperData* mapData);
+
+   private:
+    bool initialize(DeviceId& deviceId);
+
+    struct util_dl_library* mLoaderLib = nullptr;
+    struct vk_uncompacted_dispatch_table mVk;
+    VkInstance mInstance;
+    VkDevice mDevice;
+};
+
+#endif

src/gfxstream/guest/vulkan-mapper/meson.build

@@ -0,0 +1,18 @@
+# Copyright 2025 Android Open Source Project
+# SPDX-License-Identifier: MIT
+
+inc_gfxstream_vulkan_mapper = include_directories('.')
+
+files_gfxstream_vulkan_mapper = files(
+  'GfxStreamVulkanMapper.cpp',
+)
+
+libgfxstream_vulkan_mapper = static_library(
+   'gfxstream_vulkan_mapper',
+   files_gfxstream_vulkan_mapper,
+   cpp_args: gfxstream_guest_args,
+   include_directories: [inc_src, inc_vulkan_util, inc_gfxstream_vulkan_mapper,
+                         inc_include],
+   dependencies: [idep_vulkan_util_headers, idep_vulkan_util,
+                  idep_mesautil],
+)