From bd6940a68d40aaa23ce904478cfbeacccc751db6 Mon Sep 17 00:00:00 2001 From: Mohamed Ahmed Date: Wed, 28 Feb 2024 02:47:09 +0200 Subject: [PATCH] nvk: add sparse queries Part-of: --- src/nouveau/vulkan/nvk_image.c | 255 +++++++++++++++++++++++++++++---- src/nouveau/vulkan/nvk_image.h | 23 +++ 2 files changed, 253 insertions(+), 25 deletions(-) diff --git a/src/nouveau/vulkan/nvk_image.c b/src/nouveau/vulkan/nvk_image.c index 1678a879766..3260c6410ae 100644 --- a/src/nouveau/vulkan/nvk_image.c +++ b/src/nouveau/vulkan/nvk_image.c @@ -227,6 +227,38 @@ nvk_GetPhysicalDeviceImageFormatProperties2( if (ycbcr_info && pImageFormatInfo->type != VK_IMAGE_TYPE_2D) return VK_ERROR_FORMAT_NOT_SUPPORTED; + /* From the Vulkan 1.3.279 spec: + * + * VUID-VkImageCreateInfo-tiling-04121 + * + * "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain + * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT" + * + * VUID-VkImageCreateInfo-imageType-00970 + * + * "If imageType is VK_IMAGE_TYPE_1D, flags must not contain + * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT" + */ + if (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT && + (pImageFormatInfo->type == VK_IMAGE_TYPE_1D || + pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)) + return VK_ERROR_FORMAT_NOT_SUPPORTED; + + /* From the Vulkan 1.3.279 spec: + * + * VUID-VkImageCreateInfo-flags-09403 + * + * "If flags contains VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT, flags + * must not include VK_IMAGE_CREATE_SPARSE_ALIASED_BIT, + * VK_IMAGE_CREATE_SPARSE_BINDING_BIT, or + * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT" + */ + if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT) && + (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT | + VK_IMAGE_CREATE_SPARSE_BINDING_BIT | + VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))) + return VK_ERROR_FORMAT_NOT_SUPPORTED; + const uint32_t max_dim = nvk_image_max_dimension(&pdev->info, VK_IMAGE_TYPE_1D); VkExtent3D maxExtent; @@ -371,6 +403,11 @@ nvk_GetPhysicalDeviceImageFormatProperties2( (pImageFormatInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT)) return VK_ERROR_FORMAT_NOT_SUPPORTED; + if (ycbcr_info && + ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) || + (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))) + return VK_ERROR_FORMAT_NOT_SUPPORTED; + pImageFormatProperties->imageFormatProperties = (VkImageFormatProperties) { .maxExtent = maxExtent, .maxMipLevels = maxMipLevels, @@ -410,17 +447,6 @@ nvk_GetPhysicalDeviceImageFormatProperties2( return VK_SUCCESS; } -VKAPI_ATTR void VKAPI_CALL -nvk_GetPhysicalDeviceSparseImageFormatProperties2( - VkPhysicalDevice physicalDevice, - const VkPhysicalDeviceSparseImageFormatInfo2* pFormatInfo, - uint32_t *pPropertyCount, - VkSparseImageFormatProperties2 *pProperties) -{ - /* Sparse images are not yet supported. */ - *pPropertyCount = 0; -} - static enum nil_image_dim vk_image_type_to_nil_dim(VkImageType type) { @@ -433,6 +459,88 @@ vk_image_type_to_nil_dim(VkImageType type) } } +static VkSparseImageFormatProperties +nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects, + const enum pipe_format format, + const enum nil_image_dim dim, + const enum nil_sample_layout sample_layout) +{ + struct nil_extent4d sparse_block_extent_px = + nil_sparse_block_extent_px(format, dim, sample_layout); + + assert(sparse_block_extent_px.a == 1); + + VkSparseImageFormatProperties sparse_format_props = { + .aspectMask = aspects, + .flags = VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT, + .imageGranularity = { + .width = sparse_block_extent_px.w, + .height = sparse_block_extent_px.h, + .depth = sparse_block_extent_px.d, + }, + }; + + return sparse_format_props; +} + +VKAPI_ATTR void VKAPI_CALL +nvk_GetPhysicalDeviceSparseImageFormatProperties2( + VkPhysicalDevice physicalDevice, + const VkPhysicalDeviceSparseImageFormatInfo2* pFormatInfo, + uint32_t *pPropertyCount, + VkSparseImageFormatProperties2 *pProperties) +{ + VkResult result; + + /* Check if the given format info is valid first before returning sparse + * props. The easiest way to do this is to just call + * nvk_GetPhysicalDeviceImageFormatProperties2() + */ + const VkPhysicalDeviceImageFormatInfo2 img_fmt_info = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, + .format = pFormatInfo->format, + .type = pFormatInfo->type, + .tiling = pFormatInfo->tiling, + .usage = pFormatInfo->usage, + .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT | + VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT, + }; + + VkImageFormatProperties2 img_fmt_props2 = { + .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2, + .pNext = NULL, + }; + + result = nvk_GetPhysicalDeviceImageFormatProperties2(physicalDevice, + &img_fmt_info, + &img_fmt_props2); + if (result != VK_SUCCESS) { + *pPropertyCount = 0; + return; + } + + const VkImageFormatProperties *props = &img_fmt_props2.imageFormatProperties; + if (!(pFormatInfo->samples & props->sampleCounts)) { + *pPropertyCount = 0; + return; + } + + VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out, + pProperties, pPropertyCount); + + VkImageAspectFlags aspects = vk_format_aspects(pFormatInfo->format); + const enum pipe_format pipe_format = + vk_format_to_pipe_format(pFormatInfo->format); + const enum nil_image_dim dim = vk_image_type_to_nil_dim(pFormatInfo->type); + const enum nil_sample_layout sample_layout = + nil_choose_sample_layout(pFormatInfo->samples); + + vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props) { + props->properties = nvk_fill_sparse_image_fmt_props(aspects, pipe_format, + dim, sample_layout); + } +} + static VkResult nvk_image_init(struct nvk_device *dev, struct nvk_image *image, @@ -472,6 +580,12 @@ nvk_image_init(struct nvk_device *dev, image->disjoint = image->plane_count > 1 && (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT); + if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) { + /* Sparse multiplane is not supported */ + assert(image->plane_count == 1); + usage |= NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT; + } + const struct vk_format_ycbcr_info *ycbcr_info = vk_format_get_ycbcr_info(pCreateInfo->format); for (uint8_t plane = 0; plane < image->plane_count; plane++) { @@ -715,7 +829,7 @@ nvk_GetImageMemoryRequirements2(VkDevice device, pMemoryRequirements); } -VKAPI_ATTR void VKAPI_CALL +VKAPI_ATTR void VKAPI_CALL nvk_GetDeviceImageMemoryRequirements(VkDevice device, const VkDeviceImageMemoryRequirements *pInfo, VkMemoryRequirements2 *pMemoryRequirements) @@ -736,24 +850,115 @@ nvk_GetDeviceImageMemoryRequirements(VkDevice device, nvk_image_finish(dev, &image, NULL); } -VKAPI_ATTR void VKAPI_CALL -nvk_GetImageSparseMemoryRequirements2(VkDevice device, - const VkImageSparseMemoryRequirementsInfo2* pInfo, - uint32_t* pSparseMemoryRequirementCount, - VkSparseImageMemoryRequirements2* pSparseMemoryRequirements) +static VkSparseImageMemoryRequirements +nvk_fill_sparse_image_memory_reqs(const struct nil_image *nil, + const struct nil_image *stencil_tmp, + VkImageAspectFlags aspects) { - /* We dont support sparse images yet, this is a stub to get KHR_get_memory_requirements2 */ - *pSparseMemoryRequirementCount = 0; + VkSparseImageFormatProperties sparse_format_props = + nvk_fill_sparse_image_fmt_props(aspects, nil->format, + nil->dim, nil->sample_layout); + + assert(nil->mip_tail_first_lod <= nil->num_levels); + VkSparseImageMemoryRequirements sparse_memory_reqs = { + .formatProperties = sparse_format_props, + .imageMipTailFirstLod = nil->mip_tail_first_lod, + .imageMipTailStride = 0, + }; + + if (nil->mip_tail_first_lod == 0) { + sparse_memory_reqs.imageMipTailSize = nil->size_B; + sparse_memory_reqs.imageMipTailOffset = 0; + } else if (nil->mip_tail_first_lod < nil->num_levels) { + sparse_memory_reqs.imageMipTailSize = + nil_image_mip_tail_size_B(nil) * nil->extent_px.a; + sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET; + } else { + sparse_memory_reqs.imageMipTailSize = 0; + sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET; + } + + if (stencil_tmp != NULL) + sparse_memory_reqs.imageMipTailSize += stencil_tmp->size_B; + + return sparse_memory_reqs; +} + +static void +nvk_get_image_sparse_memory_requirements( + struct nvk_device *dev, + struct nvk_image *image, + VkImageAspectFlags aspects, + uint32_t *pSparseMemoryRequirementCount, + VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements) +{ + VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out, + pSparseMemoryRequirements, + pSparseMemoryRequirementCount); + + /* From the Vulkan 1.3.279 spec: + * + * "The sparse image must have been created using the + * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse + * image memory requirements." + */ + if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)) + return; + + /* We don't support multiplane sparse for now */ + if (image->plane_count > 1) + return; + + const struct nil_image *stencil_tmp = NULL; + if (image->stencil_copy_temp.nil.size_B > 0) + stencil_tmp = &image->stencil_copy_temp.nil; + + vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs) { + reqs->memoryRequirements = + nvk_fill_sparse_image_memory_reqs(&image->planes[0].nil, + stencil_tmp, aspects); + }; } VKAPI_ATTR void VKAPI_CALL -nvk_GetDeviceImageSparseMemoryRequirements(VkDevice device, - const VkDeviceImageMemoryRequirements* pInfo, - uint32_t *pSparseMemoryRequirementCount, - VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements) +nvk_GetImageSparseMemoryRequirements2( + VkDevice device, + const VkImageSparseMemoryRequirementsInfo2* pInfo, + uint32_t* pSparseMemoryRequirementCount, + VkSparseImageMemoryRequirements2* pSparseMemoryRequirements) { - /* Sparse images are not supported so this is just a stub for now. */ - *pSparseMemoryRequirementCount = 0; + VK_FROM_HANDLE(nvk_device, dev, device); + VK_FROM_HANDLE(nvk_image, image, pInfo->image); + + const VkImageAspectFlags aspects = image->vk.aspects; + + nvk_get_image_sparse_memory_requirements(dev, image, aspects, + pSparseMemoryRequirementCount, + pSparseMemoryRequirements); +} + +VKAPI_ATTR void VKAPI_CALL +nvk_GetDeviceImageSparseMemoryRequirements( + VkDevice device, + const VkDeviceImageMemoryRequirements* pInfo, + uint32_t *pSparseMemoryRequirementCount, + VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements) +{ + VK_FROM_HANDLE(nvk_device, dev, device); + ASSERTED VkResult result; + struct nvk_image image = {0}; + + result = nvk_image_init(dev, &image, pInfo->pCreateInfo); + assert(result == VK_SUCCESS); + + const VkImageAspectFlags aspects = + image.disjoint ? pInfo->planeAspect : image.vk.aspects; + + nvk_get_image_sparse_memory_requirements(dev, &image, aspects, + pSparseMemoryRequirementCount, + pSparseMemoryRequirements); + + nvk_image_finish(dev, &image, NULL); } static void diff --git a/src/nouveau/vulkan/nvk_image.h b/src/nouveau/vulkan/nvk_image.h index 6305fe7c92c..a0f7af0276b 100644 --- a/src/nouveau/vulkan/nvk_image.h +++ b/src/nouveau/vulkan/nvk_image.h @@ -11,6 +11,29 @@ #include "nil_image.h" +/* Because small images can end up with an array_stride_B that is less than + * the sparse block size (in bytes), we have to set SINGLE_MIPTAIL_BIT when + * advertising sparse properties to the client. This means that we get one + * single memory range for the miptail of the image. For large images with + * mipTailStartLod > 0, we have to deal with the array stride ourselves. + * + * We do this by returning NVK_MIP_TAIL_START_OFFSET as the image's + * imageMipTailOffset. We can then detect anything with that address as + * being part of the miptail and re-map it accordingly. The Vulkan spec + * explicitly allows for this. + * + * From the Vulkan 1.3.279 spec: + * + * "When VK_SPARSE_MEMORY_BIND_METADATA_BIT is present, the resourceOffset + * must have been derived explicitly from the imageMipTailOffset in the + * sparse resource properties returned for the metadata aspect. By + * manipulating the value returned for imageMipTailOffset, the + * resourceOffset does not have to correlate directly to a device virtual + * address offset, and may instead be whatever value makes it easiest for + * the implementation to derive the correct device virtual address." + */ +#define NVK_MIP_TAIL_START_OFFSET 0x6d74000000000000UL + struct nvk_device_memory; struct nvk_physical_device;