diff --git a/src/panfrost/vulkan/panvk_buffer.c b/src/panfrost/vulkan/panvk_buffer.c index 14779c72be8..ed5d7570a1e 100644 --- a/src/panfrost/vulkan/panvk_buffer.c +++ b/src/panfrost/vulkan/panvk_buffer.c @@ -23,12 +23,30 @@ panvk_GetBufferOpaqueCaptureAddress(VkDevice _device, return buffer->vk.device_address; } +static uint64_t +panvk_buffer_get_sparse_size(const struct panvk_buffer *buffer) +{ + struct panvk_device *device = to_panvk_device(buffer->vk.base.device); + uint64_t buffer_size = buffer->vk.size; + uint64_t page_size = panvk_get_gpu_page_size(device); + return ALIGN_POT(buffer_size, page_size); +} + VKAPI_ATTR void VKAPI_CALL -panvk_GetDeviceBufferMemoryRequirements(VkDevice device, +panvk_GetDeviceBufferMemoryRequirements(VkDevice _device, const VkDeviceBufferMemoryRequirements *pInfo, VkMemoryRequirements2 *pMemoryRequirements) { - const uint64_t align = 64; + VK_FROM_HANDLE(panvk_device, device, _device); + + /* For sparse resources alignment specifies binding granularity, rather than + * the alignment requirement. It's up to us to satisfy the alignment + * requirement when allocating the VA range. + */ + const uint64_t align = + pInfo->pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT + ? panvk_get_gpu_page_size(device) + : 64; const uint64_t size = align64(pInfo->pCreateInfo->size, align); pMemoryRequirements->memoryRequirements.memoryTypeBits = 1; @@ -60,12 +78,13 @@ panvk_BindBufferMemory2(VkDevice _device, uint32_t bindInfoCount, const VkBindMemoryStatus *bind_status = vk_find_struct_const(&pBindInfos[i], BIND_MEMORY_STATUS); + assert(!(buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)); + assert(buffer->vk.device_address == 0); + assert(mem != NULL); + if (bind_status) *bind_status->pResult = VK_SUCCESS; - assert(mem != NULL); - assert(buffer->vk.device_address == 0); - buffer->vk.device_address = mem->addr.dev + pBindInfos[i].memoryOffset; } return VK_SUCCESS; @@ -76,21 +95,58 @@ panvk_CreateBuffer(VkDevice _device, const VkBufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer) { VK_FROM_HANDLE(panvk_device, device, _device); + struct panvk_instance *instance = + to_panvk_instance(device->vk.physical->instance); struct panvk_buffer *buffer; + VkResult result; assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO); - if (pCreateInfo->size > PANVK_MAX_BUFFER_SIZE) - return panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); - buffer = vk_buffer_create(&device->vk, pCreateInfo, pAllocator, sizeof(*buffer)); if (buffer == NULL) return panvk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + if (buffer->vk.size > PANVK_MAX_BUFFER_SIZE) { + result = panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + goto err_destroy_buffer; + } + + if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) { + uint64_t va_range = panvk_buffer_get_sparse_size(buffer); + + buffer->vk.device_address = panvk_as_alloc(device, va_range, + pan_choose_gpu_va_alignment(device->kmod.vm, va_range)); + if (!buffer->vk.device_address) { + result = panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + goto err_destroy_buffer; + } + + if ((buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT) || + (instance->debug_flags & PANVK_DEBUG_FORCE_BLACKHOLE)) { + /* Map last so that we don't have a possibility of getting any more + * errors, in which case we'd have to unmap. + */ + result = panvk_map_to_blackhole(device, buffer->vk.device_address, + va_range); + if (result != VK_SUCCESS) { + result = panvk_error(device, result); + goto err_free_va; + } + } + } + *pBuffer = panvk_buffer_to_handle(buffer); return VK_SUCCESS; + +err_free_va: + if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) + panvk_as_free(device, buffer->vk.device_address, panvk_buffer_get_sparse_size(buffer)); + +err_destroy_buffer: + vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk); + return result; } VKAPI_ATTR void VKAPI_CALL @@ -103,5 +159,22 @@ panvk_DestroyBuffer(VkDevice _device, VkBuffer _buffer, if (!buffer) return; + if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) { + uint64_t va_range = panvk_buffer_get_sparse_size(buffer); + + struct pan_kmod_vm_op unmap = { + .type = PAN_KMOD_VM_OP_TYPE_UNMAP, + .va = { + .start = buffer->vk.device_address, + .size = va_range, + }, + }; + ASSERTED int ret = + pan_kmod_vm_bind(device->kmod.vm, PAN_KMOD_VM_OP_TYPE_UNMAP, &unmap, 1); + assert(!ret); + + panvk_as_free(device, buffer->vk.device_address, va_range); + } + vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk); } diff --git a/src/panfrost/vulkan/panvk_image.c b/src/panfrost/vulkan/panvk_image.c index db628852eaa..5bd885484fb 100644 --- a/src/panfrost/vulkan/panvk_image.c +++ b/src/panfrost/vulkan/panvk_image.c @@ -67,7 +67,8 @@ panvk_image_can_use_afbc( * - this is a 1D image * - this is a 3D image on a pre-v7 GPU * - this is a mutable format image on v7- (format re-interpretation is - * not possible on Bifrost hardware). + * not possible on Bifrost hardware) + * - this is a sparse image * * Some of these checks are redundant with tests provided by the AFBC mod * handler when pan_image_test_props() is called, but we need them because @@ -82,7 +83,8 @@ panvk_image_can_use_afbc( pan_afbc_supports_format(arch, pfmt) && tiling == VK_IMAGE_TILING_OPTIMAL && type != VK_IMAGE_TYPE_1D && (type != VK_IMAGE_TYPE_3D || arch >= 7) && - (!(flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) || arch >= 9); + (!(flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) || arch >= 9) && + (!(flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)); } static enum mali_texture_dimension @@ -519,6 +521,15 @@ panvk_image_get_total_size(const struct panvk_image *image) return size; } +static uint64_t +panvk_image_get_sparse_size(const struct panvk_image *image) +{ + struct panvk_device *device = to_panvk_device(image->vk.base.device); + uint64_t image_size = panvk_image_get_total_size(image); + uint64_t page_size = panvk_get_gpu_page_size(device); + return ALIGN_POT(image_size, page_size); +} + VkResult panvk_image_init(struct panvk_image *image, const VkImageCreateInfo *pCreateInfo) @@ -538,15 +549,23 @@ panvk_image_init(struct panvk_image *image, } static void -panvk_image_plane_bind(struct panvk_device *dev, - struct panvk_image_plane *plane, - struct panvk_device_memory *mem, uint64_t offset) +panvk_image_plane_bind_mem(struct panvk_device *dev, + struct panvk_image_plane *plane, + struct panvk_device_memory *mem, uint64_t offset) { plane->plane.base = mem->addr.dev + offset; plane->mem = mem; plane->mem_offset = offset; } +static void +panvk_image_plane_bind_addr(struct panvk_device *dev, + struct panvk_image_plane *plane, + uint64_t addr) +{ + plane->plane.base = addr; +} + VKAPI_ATTR VkResult VKAPI_CALL panvk_CreateImage(VkDevice device, const VkImageCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkImage *pImage) @@ -554,6 +573,9 @@ panvk_CreateImage(VkDevice device, const VkImageCreateInfo *pCreateInfo, VK_FROM_HANDLE(panvk_device, dev, device); struct panvk_physical_device *phys_dev = to_panvk_physical_device(dev->vk.physical); + struct panvk_instance *instance = + to_panvk_instance(phys_dev->vk.instance); + VkResult result; if (panvk_android_is_gralloc_image(pCreateInfo)) { return panvk_android_create_gralloc_image(device, pCreateInfo, pAllocator, @@ -574,25 +596,65 @@ panvk_CreateImage(VkDevice device, const VkImageCreateInfo *pCreateInfo, if (!image) return panvk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - VkResult result = panvk_image_init(image, pCreateInfo); + result = panvk_image_init(image, pCreateInfo); if (result != VK_SUCCESS) { vk_image_destroy(&dev->vk, pAllocator, &image->vk); return result; } + uint64_t size = panvk_image_get_total_size(image); + /* * From the Vulkan spec: * * If the size of the resultant image would exceed maxResourceSize, then * vkCreateImage must fail and return VK_ERROR_OUT_OF_DEVICE_MEMORY. */ - if (panvk_image_get_total_size(image) > UINT32_MAX) { - vk_image_destroy(&dev->vk, pAllocator, &image->vk); - return panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + if (size > UINT32_MAX) { + result = panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + goto err_destroy_image; + } + + if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) { + uint64_t va_range = panvk_image_get_sparse_size(image); + + image->sparse.device_address = panvk_as_alloc(dev, va_range, + pan_choose_gpu_va_alignment(dev->kmod.vm, va_range)); + if (!image->sparse.device_address) { + result = panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); + goto err_destroy_image; + } + + for (unsigned plane = 0; plane < image->plane_count; plane++) { + panvk_image_plane_bind_addr(dev, &image->planes[plane], + image->sparse.device_address); + } + + if ((image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) || + (instance->debug_flags & PANVK_DEBUG_FORCE_BLACKHOLE)) { + /* Map last so that we don't have a possibility of getting any more + * errors, in which case we'd have to unmap. + */ + result = panvk_map_to_blackhole(dev, image->sparse.device_address, + va_range); + if (result != VK_SUCCESS) { + result = panvk_error(dev, result); + goto err_free_va; + } + } } *pImage = panvk_image_to_handle(image); return VK_SUCCESS; + +err_free_va: + if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) + panvk_as_free(dev, image->sparse.device_address, + panvk_image_get_sparse_size(image)); + +err_destroy_image: + vk_image_destroy(&dev->vk, pAllocator, &image->vk); + return result; } VKAPI_ATTR void VKAPI_CALL @@ -605,6 +667,23 @@ panvk_DestroyImage(VkDevice _device, VkImage _image, if (!image) return; + if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) { + uint64_t va_range = panvk_image_get_sparse_size(image); + + struct pan_kmod_vm_op unmap = { + .type = PAN_KMOD_VM_OP_TYPE_UNMAP, + .va = { + .start = image->sparse.device_address, + .size = va_range, + }, + }; + ASSERTED int ret = + pan_kmod_vm_bind(device->kmod.vm, PAN_KMOD_VM_OP_TYPE_UNMAP, &unmap, 1); + assert(!ret); + + panvk_as_free(device, image->sparse.device_address, va_range); + } + vk_image_destroy(&device->vk, pAllocator, &image->vk); } @@ -692,18 +771,30 @@ panvk_GetImageMemoryRequirements2(VkDevice device, const VkImageMemoryRequirementsInfo2 *pInfo, VkMemoryRequirements2 *pMemoryRequirements) { + VK_FROM_HANDLE(panvk_device, dev, device); VK_FROM_HANDLE(panvk_image, image, pInfo->image); - const uint64_t alignment = 4096; + /* For sparse resources alignment specifies binding granularity, rather than + * the alignment requirement. It's up to us to satisfy the alignment + * requirement when allocating the VA range. + */ + const uint64_t alignment = + image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT + ? panvk_get_gpu_page_size(dev) + : 4096; const VkImagePlaneMemoryRequirementsInfo *plane_info = vk_find_struct_const(pInfo->pNext, IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO); const bool disjoint = is_disjoint(image); const VkImageAspectFlags aspects = plane_info ? plane_info->planeAspect : image->vk.aspects; uint8_t plane = panvk_plane_index(image, aspects); - const uint64_t size = + const uint64_t size_non_sparse = disjoint ? image->planes[plane].plane.layout.data_size_B : panvk_image_get_total_size(image); + const uint64_t size = + image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT + ? align64(size_non_sparse, panvk_get_gpu_page_size(dev)) + : size_non_sparse; pMemoryRequirements->memoryRequirements.memoryTypeBits = 1; pMemoryRequirements->memoryRequirements.alignment = alignment; @@ -772,6 +863,8 @@ panvk_image_bind(struct panvk_device *dev, VK_FROM_HANDLE(panvk_device_memory, mem, bind_info->memory); uint64_t offset = bind_info->memoryOffset; + assert(!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)); + if (!mem) { VkDeviceMemory mem_handle; #ifdef VK_USE_PLATFORM_ANDROID_KHR @@ -797,10 +890,10 @@ panvk_image_bind(struct panvk_device *dev, vk_find_struct_const(bind_info->pNext, BIND_IMAGE_PLANE_MEMORY_INFO); const uint8_t plane = panvk_plane_index(image, plane_info->planeAspect); - panvk_image_plane_bind(dev, &image->planes[plane], mem, offset); + panvk_image_plane_bind_mem(dev, &image->planes[plane], mem, offset); } else { for (unsigned plane = 0; plane < image->plane_count; plane++) - panvk_image_plane_bind(dev, &image->planes[plane], mem, offset); + panvk_image_plane_bind_mem(dev, &image->planes[plane], mem, offset); } return VK_SUCCESS; diff --git a/src/panfrost/vulkan/panvk_image.h b/src/panfrost/vulkan/panvk_image.h index 06bc60fff48..15d6a4b3ba6 100644 --- a/src/panfrost/vulkan/panvk_image.h +++ b/src/panfrost/vulkan/panvk_image.h @@ -34,6 +34,10 @@ struct panvk_image_plane { struct panvk_image { struct vk_image vk; + struct { + VkDeviceAddress device_address; + } sparse; + uint8_t plane_count; struct panvk_image_plane planes[PANVK_MAX_PLANES]; };