v3dv: implement VK_EXT_memory_budget

This is mostly based on Turnip's implementation.

Reviewed-by: Alejandro Piñeiro <apinheiro@igalia.com>
Reviewed-by: Eric Engestrom <eric@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/18483>

docs/features.txt

@@ -561,7 +561,7 @@ Khronos extensions that are not part of any Vulkan version:
   VK_EXT_image_view_min_lod                             DONE (anv, radv, tu, vn)
   VK_EXT_index_type_uint8                               DONE (anv, lvp, panvk, radv/gfx8+, v3dv, tu, vn)
   VK_EXT_line_rasterization                             DONE (anv, lvp, radv, tu, v3dv, vn)
-  VK_EXT_memory_budget                                  DONE (anv, radv, tu)
+  VK_EXT_memory_budget                                  DONE (anv, radv, tu, v3dv)
   VK_EXT_memory_priority                                DONE (radv)
   VK_EXT_multi_draw                                     DONE (anv, lvp, radv)
   VK_EXT_multisampled_render_to_single_sampled          DONE (lvp)

src/broadcom/vulkan/v3dv_device.c

@@ -177,6 +177,7 @@ get_device_extensions(const struct v3dv_physical_device *device,
       .EXT_image_drm_format_modifier        = true,
       .EXT_index_type_uint8                 = true,
       .EXT_line_rasterization               = true,
+      .EXT_memory_budget                    = true,
       .EXT_physical_device_drm              = true,
       .EXT_pipeline_creation_cache_control  = true,
       .EXT_pipeline_creation_feedback       = true,
@@ -369,6 +370,27 @@ compute_heap_size()
    return available;
 }
 
+static uint64_t
+compute_memory_budget(struct v3dv_physical_device *device)
+{
+   uint64_t heap_size = device->memory.memoryHeaps[0].size;
+   uint64_t heap_used = device->heap_used;
+   uint64_t sys_available;
+#if !using_v3d_simulator
+   ASSERTED bool has_available_memory =
+      os_get_available_system_memory(&sys_available);
+   assert(has_available_memory);
+#else
+   sys_available = (uint64_t) v3d_simulator_get_mem_free();
+#endif
+
+   /* Let's not incite the app to starve the system: report at most 90% of
+    * available system memory.
+    */
+   uint64_t heap_available = sys_available * 9 / 10;
+   return MIN2(heap_size, heap_used + heap_available);
+}
+
 #if !using_v3d_simulator
 #ifdef VK_USE_PLATFORM_XCB_KHR
 static int
@@ -1786,11 +1808,28 @@ VKAPI_ATTR void VKAPI_CALL
 v3dv_GetPhysicalDeviceMemoryProperties2(VkPhysicalDevice physicalDevice,
                                         VkPhysicalDeviceMemoryProperties2 *pMemoryProperties)
 {
+   V3DV_FROM_HANDLE(v3dv_physical_device, device, physicalDevice);
+
    v3dv_GetPhysicalDeviceMemoryProperties(physicalDevice,
                                           &pMemoryProperties->memoryProperties);
 
    vk_foreach_struct(ext, pMemoryProperties->pNext) {
       switch (ext->sType) {
+      case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT: {
+         VkPhysicalDeviceMemoryBudgetPropertiesEXT *p =
+            (VkPhysicalDeviceMemoryBudgetPropertiesEXT *) ext;
+         p->heapUsage[0] = device->heap_used;
+         p->heapBudget[0] = compute_memory_budget(device);
+
+         /* The heapBudget and heapUsage values must be zero for array elements
+          * greater than or equal to VkPhysicalDeviceMemoryProperties::memoryHeapCount
+          */
+         for (unsigned i = 1; i < VK_MAX_MEMORY_HEAPS; i++) {
+            p->heapBudget[i] = 0u;
+            p->heapUsage[i] = 0u;
+         }
+         break;
+      }
       default:
          v3dv_debug_ignored_stype(ext->sType);
          break;
@@ -2118,6 +2157,8 @@ device_free(struct v3dv_device *device, struct v3dv_device_memory *mem)
       device_free_wsi_dumb(device->pdevice->display_fd, mem->bo->dumb_handle);
    }
 
+   p_atomic_add(&device->pdevice->heap_used, -((int64_t)mem->bo->size));
+
    v3dv_bo_free(device, mem->bo);
 }
 
@@ -2282,6 +2323,35 @@ device_remove_device_address_bo(struct v3dv_device *device,
                                   bo);
 }
 
+static void
+free_memory(struct v3dv_device *device,
+            struct v3dv_device_memory *mem,
+            const VkAllocationCallbacks *pAllocator)
+{
+   if (mem == NULL)
+      return;
+
+   if (mem->bo->map)
+      device_unmap(device, mem);
+
+   if (mem->is_for_device_address)
+      device_remove_device_address_bo(device, mem->bo);
+
+   device_free(device, mem);
+
+   vk_object_free(&device->vk, pAllocator, mem);
+}
+
+VKAPI_ATTR void VKAPI_CALL
+v3dv_FreeMemory(VkDevice _device,
+                VkDeviceMemory _mem,
+                const VkAllocationCallbacks *pAllocator)
+{
+   V3DV_FROM_HANDLE(v3dv_device, device, _device);
+   V3DV_FROM_HANDLE(v3dv_device_memory, mem, _mem);
+   free_memory(device, mem, pAllocator);
+}
+
 VKAPI_ATTR VkResult VKAPI_CALL
 v3dv_AllocateMemory(VkDevice _device,
                     const VkMemoryAllocateInfo *pAllocateInfo,
@@ -2297,6 +2367,18 @@ v3dv_AllocateMemory(VkDevice _device,
    /* The Vulkan 1.0.33 spec says "allocationSize must be greater than 0". */
    assert(pAllocateInfo->allocationSize > 0);
 
+   /* We always allocate device memory in multiples of a page, so round up
+    * requested size to that.
+    */
+   const VkDeviceSize alloc_size = ALIGN(pAllocateInfo->allocationSize, 4096);
+
+   if (unlikely(alloc_size > MAX_MEMORY_ALLOCATION_SIZE))
+      return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+
+   uint64_t heap_used = p_atomic_read(&pdevice->heap_used);
+   if (unlikely(heap_used + alloc_size > pdevice->memory.memoryHeaps[0].size))
+      return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+
    mem = vk_object_zalloc(&device->vk, pAllocator, sizeof(*mem),
                           VK_OBJECT_TYPE_DEVICE_MEMORY);
    if (mem == NULL)
@@ -2337,28 +2419,18 @@ v3dv_AllocateMemory(VkDevice _device,
       }
    }
 
-   VkResult result = VK_SUCCESS;
+   VkResult result;
-
+   if (wsi_info) {
-   /* We always allocate device memory in multiples of a page, so round up
+      result = device_alloc_for_wsi(device, pAllocator, mem, alloc_size);
-    * requested size to that.
+   } else if (fd_info && fd_info->handleType) {
-    */
+      assert(fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
-   VkDeviceSize alloc_size = ALIGN(pAllocateInfo->allocationSize, 4096);
+             fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
-
+      result = device_import_bo(device, pAllocator,
-   if (unlikely(alloc_size > MAX_MEMORY_ALLOCATION_SIZE)) {
+                                fd_info->fd, alloc_size, &mem->bo);
-      result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
+      if (result == VK_SUCCESS)
+         close(fd_info->fd);
    } else {
-      if (wsi_info) {
+      result = device_alloc(device, mem, alloc_size);
-         result = device_alloc_for_wsi(device, pAllocator, mem, alloc_size);
-      } else if (fd_info && fd_info->handleType) {
-         assert(fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
-                fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
-         result = device_import_bo(device, pAllocator,
-                                   fd_info->fd, alloc_size, &mem->bo);
-         if (result == VK_SUCCESS)
-            close(fd_info->fd);
-      } else {
-         result = device_alloc(device, mem, alloc_size);
-      }
    }
 
    if (result != VK_SUCCESS) {
@@ -2366,6 +2438,12 @@ v3dv_AllocateMemory(VkDevice _device,
       return vk_error(device, result);
    }
 
+   heap_used = p_atomic_add_return(&pdevice->heap_used, mem->bo->size);
+   if (heap_used > pdevice->memory.memoryHeaps[0].size) {
+      free_memory(device, mem, pAllocator);
+      return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
+   }
+
    /* If this memory can be used via VK_KHR_buffer_device_address then we
     * will need to manually add the BO to any job submit that makes use of
     * VK_KHR_buffer_device_address, since such jobs may produde buffer
@@ -2384,28 +2462,6 @@ v3dv_AllocateMemory(VkDevice _device,
    return result;
 }
 
-VKAPI_ATTR void VKAPI_CALL
-v3dv_FreeMemory(VkDevice _device,
-                VkDeviceMemory _mem,
-                const VkAllocationCallbacks *pAllocator)
-{
-   V3DV_FROM_HANDLE(v3dv_device, device, _device);
-   V3DV_FROM_HANDLE(v3dv_device_memory, mem, _mem);
-
-   if (mem == NULL)
-      return;
-
-   if (mem->bo->map)
-      v3dv_UnmapMemory(_device, _mem);
-
-   if (mem->is_for_device_address)
-      device_remove_device_address_bo(device, mem->bo);
-
-   device_free(device, mem);
-
-   vk_object_free(&device->vk, pAllocator, mem);
-}
-
 VKAPI_ATTR VkResult VKAPI_CALL
 v3dv_MapMemory(VkDevice _device,
                VkDeviceMemory _memory,

src/broadcom/vulkan/v3dv_private.h

@@ -165,6 +165,8 @@ struct v3dv_physical_device {
    const struct v3d_compiler *compiler;
    uint32_t next_program_id;
 
+   uint64_t heap_used;
+
    /* This array holds all our 'struct v3dv_bo' allocations. We use this
     * so we can add a refcount to our BOs and check if a particular BO
     * was already allocated in this device using its GEM handle. This is