radv: Fix budget calculations with large BAR.

If we don't have a non-visible VRAM heap, we should be counting
our non-visible VRAM allocations to the visible-VRAM heap.

CC: mesa-stable
Reviewed-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6827>

src/amd/vulkan/radv_device.c

@@ -139,6 +139,13 @@ radv_get_vram_size(struct radv_physical_device *device)
 	return device->rad_info.vram_size - radv_get_visible_vram_size(device);
 }
 
+enum radv_heap {
+	RADV_HEAP_VRAM     = 1 << 0,
+	RADV_HEAP_GTT      = 1 << 1,
+	RADV_HEAP_VRAM_VIS = 1 << 2,
+	RADV_HEAP_MAX      = 1 << 3,
+};
+
 static void
 radv_physical_device_init_mem_types(struct radv_physical_device *device)
 {
@@ -146,11 +153,13 @@ radv_physical_device_init_mem_types(struct radv_physical_device *device)
 	uint64_t vram_size = radv_get_vram_size(device);
 	int vram_index = -1, visible_vram_index = -1, gart_index = -1;
 	device->memory_properties.memoryHeapCount = 0;
+	device->heaps = 0;
 
 	/* Only get a VRAM heap if it is significant, not if it is a 16 MiB
 	 * remainder above visible VRAM. */
 	if (vram_size > 0 && vram_size * 9 >= visible_vram_size) {
 		vram_index = device->memory_properties.memoryHeapCount++;
+		device->heaps |= RADV_HEAP_VRAM;
 		device->memory_properties.memoryHeaps[vram_index] = (VkMemoryHeap) {
 			.size = vram_size,
 			.flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
@@ -159,6 +168,7 @@ radv_physical_device_init_mem_types(struct radv_physical_device *device)
 
 	if (device->rad_info.gart_size > 0) {
 		gart_index = device->memory_properties.memoryHeapCount++;
+		device->heaps |= RADV_HEAP_GTT;
 		device->memory_properties.memoryHeaps[gart_index] = (VkMemoryHeap) {
 			.size = device->rad_info.gart_size,
 			.flags = 0,
@@ -167,6 +177,7 @@ radv_physical_device_init_mem_types(struct radv_physical_device *device)
 
 	if (visible_vram_size) {
 		visible_vram_index = device->memory_properties.memoryHeapCount++;
+		device->heaps |= RADV_HEAP_VRAM_VIS;
 		device->memory_properties.memoryHeaps[visible_vram_index] = (VkMemoryHeap) {
 			.size = visible_vram_size,
 			.flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
@@ -2176,10 +2187,6 @@ radv_get_memory_budget_properties(VkPhysicalDevice physicalDevice,
 {
 	RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
 	VkPhysicalDeviceMemoryProperties *memory_properties = &device->memory_properties;
-	uint64_t visible_vram_size = radv_get_visible_vram_size(device);
-	uint64_t vram_size = radv_get_vram_size(device);
-	uint64_t gtt_size = device->rad_info.gart_size;
-	uint64_t heap_budget, heap_usage;
 
 	/* For all memory heaps, the computation of budget is as follow:
 	 *	heap_budget = heap_size - global_heap_usage + app_heap_usage
@@ -2190,44 +2197,39 @@ radv_get_memory_budget_properties(VkPhysicalDevice physicalDevice,
 	 * Note that the application heap usages are not really accurate (eg.
 	 * in presence of shared buffers).
 	 */
-	for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
+	unsigned mask = device->heaps;
-		uint32_t heap_index = device->memory_properties.memoryTypes[i].heapIndex;
+	unsigned heap = 0;
+	while (mask) {
+		uint64_t internal_usage = 0, total_usage = 0;
+		unsigned type = 1u << u_bit_scan(&mask);
 
-		if ((device->memory_domains[i] & RADEON_DOMAIN_VRAM) && (device->memory_flags[i] & RADEON_FLAG_NO_CPU_ACCESS)) {
+		switch(type) {
-			heap_usage = device->ws->query_value(device->ws,
+		case RADV_HEAP_VRAM:
-							     RADEON_ALLOCATED_VRAM);
+			internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM);
-
+			total_usage = device->ws->query_value(device->ws, RADEON_VRAM_USAGE);
-			heap_budget = vram_size -
+			break;
-				device->ws->query_value(device->ws, RADEON_VRAM_USAGE) +
+		case RADV_HEAP_VRAM_VIS:
-				heap_usage;
+			internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM_VIS);
-
+			if (!(device->heaps & RADV_HEAP_VRAM))
-			memoryBudget->heapBudget[heap_index] = heap_budget;
+				internal_usage += device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM);
-			memoryBudget->heapUsage[heap_index] = heap_usage;
+			total_usage = device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE);
-		} else if (device->memory_domains[i] & RADEON_DOMAIN_VRAM) {
+			break;
-			heap_usage = device->ws->query_value(device->ws,
+		case RADV_HEAP_GTT:
-							     RADEON_ALLOCATED_VRAM_VIS);
+			internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_GTT);
-
+			total_usage = device->ws->query_value(device->ws, RADEON_GTT_USAGE);
-			heap_budget = visible_vram_size -
+			break;
-				device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE) +
-				heap_usage;
-
-			memoryBudget->heapBudget[heap_index] = heap_budget;
-			memoryBudget->heapUsage[heap_index] = heap_usage;
-		} else {
-			assert(device->memory_domains[i] & RADEON_DOMAIN_GTT);
-
-			heap_usage = device->ws->query_value(device->ws,
-							     RADEON_ALLOCATED_GTT);
-
-			heap_budget = gtt_size -
-				device->ws->query_value(device->ws, RADEON_GTT_USAGE) +
-				heap_usage;
-
-			memoryBudget->heapBudget[heap_index] = heap_budget;
-			memoryBudget->heapUsage[heap_index] = heap_usage;
 		}
+
+		uint64_t free_space = device->memory_properties.memoryHeaps[heap].size -
+			MIN2(device->memory_properties.memoryHeaps[heap].size,
+			     total_usage);
+		memoryBudget->heapBudget[heap] = free_space + internal_usage;
+		memoryBudget->heapUsage[heap] = internal_usage;
+		++heap;
 	}
 
+	assert(heap == memory_properties->memoryHeapCount);
+
 	/* The heapBudget and heapUsage values must be zero for array elements
 	 * greater than or equal to
 	 * VkPhysicalDeviceMemoryProperties::memoryHeapCount.

src/amd/vulkan/radv_private.h

@@ -307,6 +307,7 @@ struct radv_physical_device {
 	VkPhysicalDeviceMemoryProperties memory_properties;
 	enum radeon_bo_domain memory_domains[VK_MAX_MEMORY_TYPES];
 	enum radeon_bo_flag memory_flags[VK_MAX_MEMORY_TYPES];
+	unsigned heaps;
 
 	drmPciBusInfo bus_info;