tu: Support sparse residency for images

The tricky thing here is that we have to emulate the 64k "standard"
tile sizes in terms of the native 4k macrotiles. We do this by
manipulating which 4k pages get mapped, dividing the 64k tile into 4k
macrotiles and mapping each tile in such a way that, when viewed in
terms of the final swizzled image coordinates, the 4k tiles linearly
tile the image region that's supposed to be mapped to the 64k "tile".
Supporting the standard block sizes allows emulation layers to claim D3D
Tiled Resources Tier 2, which is required for the 12.0 feature level.
It's also required for ARB_sparse_texture2.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32671>
This commit is contained in:
Connor Abbott 2024-11-22 14:08:28 -05:00 committed by Marge Bot
parent ae53234414
commit 918e25e158
5 changed files with 399 additions and 29 deletions

View file

@ -408,6 +408,11 @@ tu_get_features(struct tu_physical_device *pdevice,
features->shaderInt16 = true;
features->sparseBinding = pdevice->has_sparse;
features->sparseResidencyBuffer = pdevice->has_sparse_prr;
features->sparseResidencyImage2D = pdevice->has_sparse_prr &&
pdevice->info->a7xx.ubwc_all_formats_compatible;
features->sparseResidency2Samples = features->sparseResidencyImage2D;
features->sparseResidency4Samples = features->sparseResidencyImage2D;
features->sparseResidency8Samples = features->sparseResidencyImage2D;
features->sparseResidencyAliased = pdevice->has_sparse_prr;
features->variableMultisampleRate = true;
features->inheritedQueries = true;
@ -1155,10 +1160,10 @@ tu_get_properties(struct tu_physical_device *pdevice,
props->dynamicRenderingLocalReadMultisampledAttachments = true;
/* sparse properties */
props->sparseResidencyStandard2DBlockShape = { 0 };
props->sparseResidencyStandard2DMultisampleBlockShape = { 0 };
props->sparseResidencyStandard3DBlockShape = { 0 };
props->sparseResidencyAlignedMipSize = { 0 };
props->sparseResidencyStandard2DBlockShape = true;
props->sparseResidencyStandard2DMultisampleBlockShape = true;
props->sparseResidencyStandard3DBlockShape = false;
props->sparseResidencyAlignedMipSize = false;
props->sparseResidencyNonResidentStrict = true;
strcpy(props->deviceName, pdevice->name);

View file

@ -344,7 +344,7 @@ tu_GetPhysicalDeviceFormatProperties2(
/* note: ubwc_possible() argument values to be ignored except for format */
if (pFormatProperties->formatProperties.optimalTilingFeatures &&
tiling_possible(format) &&
ubwc_possible(NULL, format, VK_IMAGE_TYPE_2D, 0, 0,
ubwc_possible(NULL, format, VK_IMAGE_TYPE_2D, 0, 0, 0,
physical_device->info, VK_SAMPLE_COUNT_1_BIT, 1,
false)) {
vk_outarray_append_typed(VkDrmFormatModifierPropertiesEXT, &out, mod_props) {
@ -403,6 +403,10 @@ tu_get_image_format_properties(
if (info->flags & VK_IMAGE_CREATE_SUBSAMPLED_BIT_EXT)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
/* Don't allow modifiers with sparse */
if (info->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
return VK_ERROR_FORMAT_NOT_SUPPORTED;
switch (drm_info->drmFormatModifier) {
case DRM_FORMAT_MOD_QCOM_COMPRESSED:
/* falling back to linear/non-UBWC isn't possible with explicit modifier */
@ -421,9 +425,9 @@ tu_get_image_format_properties(
return VK_ERROR_FORMAT_NOT_SUPPORTED;
}
if (!ubwc_possible(NULL, info->format, info->type, info->usage,
info->usage, physical_device->info, sampleCounts,
1, false)) {
if (!ubwc_possible(NULL, info->format, info->type, info->flags,
info->usage, info->usage, physical_device->info,
sampleCounts, 1, false)) {
return VK_ERROR_FORMAT_NOT_SUPPORTED;
}
@ -446,6 +450,31 @@ tu_get_image_format_properties(
if (format_feature_flags == 0)
return tu_image_unsupported_format(pImageFormatProperties);
if (info->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) {
if (!physical_device->has_sparse)
return tu_image_unsupported_format(pImageFormatProperties);
}
if (info->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
/* Don't support multi-planar formats with sparse yet */
if (vk_format_get_plane_count(info->format) > 1)
return tu_image_unsupported_format(pImageFormatProperties);
/* Sparse isn't compatible with HIC */
if (info->usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT)
return tu_image_unsupported_format(pImageFormatProperties);
/* We can't support sparse when we force linear tiling, so disable
* sparse with formats or usages which could cause us to fall back to
* linear. We also currently don't support sparse for 3D images.
*/
if (info->type != VK_IMAGE_TYPE_2D ||
info->tiling != VK_IMAGE_TILING_OPTIMAL ||
!tiling_possible(info->format) ||
(info->usage & VK_IMAGE_USAGE_FRAGMENT_DENSITY_MAP_BIT_EXT))
return tu_image_unsupported_format(pImageFormatProperties);
}
if (info->type != VK_IMAGE_TYPE_2D &&
vk_format_is_depth_or_stencil(info->format))
return tu_image_unsupported_format(pImageFormatProperties);
@ -765,6 +794,7 @@ tu_GetPhysicalDeviceImageFormatProperties2(
(fd6_color_swap(vk_format_to_pipe_format(base_info->format),
TILE6_LINEAR, false) == WZYX &&
!ubwc_possible(NULL, base_info->format, base_info->type,
base_info->flags,
(base_info->usage & ~VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT),
(base_info->usage & ~VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT),
physical_device->info, VK_SAMPLE_COUNT_1_BIT, 1,
@ -787,14 +817,3 @@ fail:
return result;
}
VKAPI_ATTR void VKAPI_CALL
tu_GetPhysicalDeviceSparseImageFormatProperties2(
VkPhysicalDevice physicalDevice,
const VkPhysicalDeviceSparseImageFormatInfo2 *pFormatInfo,
uint32_t *pPropertyCount,
VkSparseImageFormatProperties2 *pProperties)
{
/* Sparse images are not yet supported. */
*pPropertyCount = 0;
}

View file

@ -20,6 +20,8 @@
#include "drm-uapi/drm_fourcc.h"
#include "vulkan/vulkan_core.h"
#include "fdl/freedreno_layout.h"
#include "tu_buffer.h"
#include "tu_cs.h"
#include "tu_descriptor_set.h"
@ -323,6 +325,7 @@ bool
ubwc_possible(struct tu_device *device,
VkFormat format,
VkImageType type,
VkImageCreateFlags flags,
VkImageUsageFlags usage,
VkImageUsageFlags stencil_usage,
const struct fd_dev_info *info,
@ -334,6 +337,13 @@ ubwc_possible(struct tu_device *device,
if (info->a6xx.is_a702)
return false;
/* UBWC isn't possible with sparse residency, because unbound blocks may
* have leftover fast-clear data and therefore may show up as non-zero.
* TODO: Enable UBWC if nonResidentStrict isn't enabled.
*/
if (flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)
return false;
/* no UBWC with compressed formats, E5B9G9R9, S8_UINT
* (S8_UINT because separate stencil doesn't have UBWC-enable bit)
*/
@ -508,6 +518,12 @@ tu_image_update_layout(struct tu_device *device, struct tu_image *image,
tile_mode = TILE6_LINEAR;
}
/* We cannot support sparse residency with linear images, it should've been
* rejected.
*/
assert(!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) ||
tile_mode == TILE6_3);
for (uint32_t i = 0; i < tu6_plane_count(image->vk.format); i++) {
struct fdl_layout *layout = &image->layout[i];
enum pipe_format format = tu6_plane_format(image->vk.format, i);
@ -625,10 +641,10 @@ format_list_ubwc_possible(struct tu_device *dev,
for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
if (!ubwc_possible(dev, fmt_list->pViewFormats[i],
create_info->imageType, create_info->usage,
create_info->usage, dev->physical_device->info,
create_info->samples, create_info->mipLevels,
dev->use_z24uint_s8uint))
create_info->imageType, create_info->flags,
create_info->usage, create_info->usage,
dev->physical_device->info, create_info->samples,
create_info->mipLevels, dev->use_z24uint_s8uint))
return false;
}
@ -683,7 +699,8 @@ tu_image_init(struct tu_device *device, struct tu_image *image,
if (image->force_linear_tile ||
!ubwc_possible(device, image->vk.format, pCreateInfo->imageType,
pCreateInfo->usage, image->vk.stencil_usage,
pCreateInfo->flags, pCreateInfo->usage,
image->vk.stencil_usage,
device->physical_device->info, pCreateInfo->samples,
pCreateInfo->mipLevels, device->use_z24uint_s8uint))
image->ubwc_enabled = false;
@ -1043,9 +1060,15 @@ tu_get_image_memory_requirements(struct tu_device *dev, struct tu_image *image,
uint32_t alignment = image->layout[0].base_align;
if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
alignment = MAX2(alignment, os_page_size);
if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)
alignment = 65536;
pMemoryRequirements->memoryRequirements = (VkMemoryRequirements) {
.size = image->total_size,
/* Due to how we fake the sparse tile size, the real size may not be
* aligned. CTS doesn't like this, and real apps may also be surprised,
* so we align it.
*/
.size = align64(image->total_size, alignment),
.alignment = alignment,
.memoryTypeBits = (1 << dev->physical_device->memory.type_count) - 1,
};
@ -1066,6 +1089,155 @@ tu_get_image_memory_requirements(struct tu_device *dev, struct tu_image *image,
}
}
static VkSparseImageFormatProperties
tu_fill_sparse_image_fmt_props(VkImageAspectFlags aspects,
const enum pipe_format format,
VkSampleCountFlags samples)
{
uint32_t width, height;
fdl_get_sparse_block_size(format, samples, &width, &height);
VkSparseImageFormatProperties sparse_format_props = {
.aspectMask = aspects,
.imageGranularity = {
.width = width * util_format_get_blockwidth(format),
.height = height * util_format_get_blockheight(format),
.depth = 1,
},
.flags = 0,
};
return sparse_format_props;
}
VKAPI_ATTR void VKAPI_CALL
tu_GetPhysicalDeviceSparseImageFormatProperties2(
VkPhysicalDevice physicalDevice,
const VkPhysicalDeviceSparseImageFormatInfo2* pFormatInfo,
uint32_t *pPropertyCount,
VkSparseImageFormatProperties2 *pProperties)
{
VkResult result;
/* Check if the given format info is valid first before returning sparse
* props. The easiest way to do this is to just call
* tu_GetPhysicalDeviceImageFormatProperties2()
*/
const VkPhysicalDeviceImageFormatInfo2 img_fmt_info = {
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
.format = pFormatInfo->format,
.type = pFormatInfo->type,
.tiling = pFormatInfo->tiling,
.usage = pFormatInfo->usage,
.flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT,
};
VkImageFormatProperties2 img_fmt_props2 = {
.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
.pNext = NULL,
};
result = tu_GetPhysicalDeviceImageFormatProperties2(physicalDevice,
&img_fmt_info,
&img_fmt_props2);
if (result != VK_SUCCESS) {
*pPropertyCount = 0;
return;
}
const VkImageFormatProperties *props = &img_fmt_props2.imageFormatProperties;
if (!(pFormatInfo->samples & props->sampleCounts)) {
*pPropertyCount = 0;
return;
}
/* We should already reject non-2D images */
assert(pFormatInfo->type == VK_IMAGE_TYPE_2D);
VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out,
pProperties, pPropertyCount);
VkImageAspectFlags aspects = vk_format_aspects(pFormatInfo->format);
const enum pipe_format pipe_format =
vk_format_to_pipe_format(pFormatInfo->format);
if (pFormatInfo->format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
u_foreach_bit (aspect, aspects) {
enum pipe_format aspect_format =
tu6_plane_format(pFormatInfo->format, aspect);
vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props) {
props->properties =
tu_fill_sparse_image_fmt_props(aspect, aspect_format,
pFormatInfo->samples);
}
}
} else {
vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props) {
props->properties = tu_fill_sparse_image_fmt_props(aspects, pipe_format,
pFormatInfo->samples);
}
}
}
static VkSparseImageMemoryRequirements
tu_fill_sparse_image_memory_reqs(const struct fdl_layout *layout,
VkImageAspectFlags aspects)
{
VkSparseImageFormatProperties sparse_format_props =
tu_fill_sparse_image_fmt_props(aspects,
layout->format,
layout->nr_samples);
VkSparseImageMemoryRequirements sparse_memory_reqs = {
.formatProperties = sparse_format_props,
.imageMipTailFirstLod = layout->mip_tail_first_lod,
.imageMipTailSize = fdl_sparse_miptail_size(layout),
.imageMipTailOffset = fdl_sparse_miptail_offset(layout),
.imageMipTailStride = layout->layer_size,
};
return sparse_memory_reqs;
}
static void
tu_get_image_sparse_memory_requirements(
struct tu_device *dev,
struct tu_image *image,
uint32_t *pSparseMemoryRequirementCount,
VkSparseImageMemoryRequirements2 *pMemoryRequirements)
{
VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out,
pMemoryRequirements, pSparseMemoryRequirementCount);
/* From the Vulkan 1.3.279 spec:
*
* "The sparse image must have been created using the
* VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
* image memory requirements."
*/
if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
return;
if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
u_foreach_bit (aspect, image->vk.aspects) {
const struct fdl_layout *layout =
&image->layout[tu6_plane_index(image->vk.format, aspect)];
vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs) {
reqs->memoryRequirements =
tu_fill_sparse_image_memory_reqs(layout, aspect);
};
}
} else {
vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs) {
reqs->memoryRequirements =
tu_fill_sparse_image_memory_reqs(&image->layout[0],
image->vk.aspects);
};
}
}
VKAPI_ATTR void VKAPI_CALL
tu_GetImageMemoryRequirements2(VkDevice _device,
const VkImageMemoryRequirementsInfo2 *pInfo,
@ -1079,12 +1251,17 @@ tu_GetImageMemoryRequirements2(VkDevice _device,
VKAPI_ATTR void VKAPI_CALL
tu_GetImageSparseMemoryRequirements2(
VkDevice device,
VkDevice _device,
const VkImageSparseMemoryRequirementsInfo2 *pInfo,
uint32_t *pSparseMemoryRequirementCount,
VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
{
tu_stub();
VK_FROM_HANDLE(tu_device, device, _device);
VK_FROM_HANDLE(tu_image, image, pInfo->image);
tu_get_image_sparse_memory_requirements(device, image,
pSparseMemoryRequirementCount,
pSparseMemoryRequirements);
}
VKAPI_ATTR void VKAPI_CALL
@ -1106,12 +1283,22 @@ tu_GetDeviceImageMemoryRequirements(
VKAPI_ATTR void VKAPI_CALL
tu_GetDeviceImageSparseMemoryRequirements(
VkDevice device,
VkDevice _device,
const VkDeviceImageMemoryRequirements *pInfo,
uint32_t *pSparseMemoryRequirementCount,
VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
{
tu_stub();
VK_FROM_HANDLE(tu_device, device, _device);
struct tu_image image = {0};
vk_image_init(&device->vk, &image.vk, pInfo->pCreateInfo);
tu_image_init(device, &image, pInfo->pCreateInfo);
TU_CALLX(device, tu_image_update_layout)(device, &image, DRM_FORMAT_MOD_INVALID, NULL);
tu_get_image_sparse_memory_requirements(device, &image,
pSparseMemoryRequirementCount,
pSparseMemoryRequirements);
}
static void
@ -1258,3 +1445,148 @@ tu_GetImageOpaqueCaptureDescriptorDataEXT(VkDevice device,
*(uint64_t *)pData = image->iova;
return VK_SUCCESS;
}
/* The native macrotile size is 4K, and the page size is also 4K, so the
* most natural thing would be to expose 4K tiles. But that isn't compatible
* with D3D requirements, so we have to emulate 64K "sparse tiles" on the
* native 4K macrotiles.
*
* Each "sparse tile" contains macrotiles in the natural linear order when
* viewed in terms of the (bank swizzled) image coordinates. We have to do
* this in order to guarantee that aliasing tiles in different images works.
* For example, if cpp=16, then the UBWC 4K macrotile is 16x16 pixels, while
* the sparse 64K tile is 64x64 pixels. That means each sparse tile contains
* 4x4 macrotiles. Then the 64K tile at (0, 0) is mapped like this to the
* image:
*
* |--16px---|
* - -----------------------------------------------
* | | | | | |
* 16px | Tile 0 | Tile 1 | Tile 2 | Tile 3 | . . .
* | | | | | |
* - -----------------------------------------------
* | | | | |
* | Tile 4 | Tile 5 | Tile 6 | Tile 7 | . . .
* | | | | |
* -----------------------------------------------
* | | | | |
* | Tile 8 | Tile 9 | Tile 10 | Tile 11 | . . .
* | | | | |
* -----------------------------------------------
* | | | | |
* | Tile 12 | Tile 13 | Tile 14 | Tile 15 | . . .
* | | | | |
* -----------------------------------------------
* | . | . | . | . | .
* | . | . | . | . | .
* | . | . | . | . | .
*
* One tricky case is when the stride isn't aligned to the sparse tile width,
* or the height isn't aligned to the sparse height: at the bottom or left
* edges there may be macrotiles inside the sparse tile that overhang the
* image and don't have any corresponding backing memory, and we have to skip
* mapping/unmapping those.
*
* When doing the mapping, we have to be aware of bank swizzling. It may
* reorder macrotiles inside a sparse tile, or it may reorder sparse tiles, or
* both, depending on the highest bank bit, bank swizzling levels, and
* alignment. We cannot ignore the bank swizzling even in the first case,
* where it only reorders macrotiles inside the sparse tile, to ensure that
* aliasing (i.e. remapping the same sparse tile into a different image) works
* because different images have different alignments and therefore different
* bank swizzling.
*/
void
tu_bind_sparse_image(struct tu_device *device, void *submit,
struct tu_image *image,
const VkSparseImageMemoryBind *bind)
{
VK_FROM_HANDLE(tu_device_memory, mem, bind->memory);
const struct fdl_layout *layout =
&image->layout[tu6_plane_index(image->vk.format,
bind->subresource.aspectMask)];
struct tu_bo *bo = mem ? mem->bo : NULL;
uint64_t bo_offset = mem ? bind->memoryOffset : 0;
uint32_t sparse_width, sparse_height;
uint32_t macrotile_width, macrotile_height;
fdl_get_sparse_block_size(layout->format,
layout->nr_samples,
&sparse_width, &sparse_height);
fdl6_get_ubwc_macrotile_size(layout,
&macrotile_width, &macrotile_height);
assert(sparse_width % macrotile_width == 0);
assert(sparse_height % macrotile_height == 0);
uint32_t blockwidth = util_format_get_blockwidth(layout->format);
uint32_t blockheight = util_format_get_blockheight(layout->format);
uint32_t x_start = bind->offset.x / blockwidth;
uint32_t x_end = DIV_ROUND_UP(bind->offset.x + bind->extent.width,
blockwidth);
uint32_t y_start = bind->offset.y / blockheight;
uint32_t y_end = DIV_ROUND_UP(bind->offset.y + bind->extent.height,
blockheight);
uint32_t cpp = layout->cpp;
uint32_t pitch = fdl_pitch(layout, bind->subresource.mipLevel);
uint64_t image_offset = fdl_surface_offset(layout,
bind->subresource.mipLevel,
bind->subresource.arrayLayer);
uint32_t bank_mask = fdl6_get_bank_mask(layout,
bind->subresource.mipLevel,
&device->physical_device->ubwc_config);
uint32_t bank_shift =
fdl6_get_bank_shift(&device->physical_device->ubwc_config);
/* Our y offset is in pixels */
bank_mask *= macrotile_height;
bank_shift -= util_logbase2(macrotile_height);
uint64_t prev_image_offset = 0;
uint64_t prev_bo_offset = 0;
uint64_t bind_range = 0;
for (unsigned sy = y_start; sy < y_end; sy += sparse_height) {
for (unsigned sx = x_start; sx < x_end; sx += sparse_width,
bo_offset += 65536) {
uint64_t row_bo_offset = bo_offset;
for (unsigned ty = sy; ty < MIN2(sy + sparse_height, y_end);
ty += macrotile_height,
row_bo_offset += sparse_width * macrotile_height * cpp) {
uint64_t row_image_offset = image_offset + pitch * ty;
uint32_t x_swizzle = (ty & bank_mask) << bank_shift;
uint64_t column_bo_offset = row_bo_offset;
for (unsigned tx = sx; tx < MIN2(sx + sparse_width, x_end);
tx += macrotile_width, column_bo_offset += 4096) {
uint64_t image_offset =
((tx * macrotile_height * cpp) ^ x_swizzle) + row_image_offset;
/* Try to combine consecutive binds. In most cases, depending
* on the x_swizzle, we should be able to map the whole row of
* the sparse tile at once.
*/
if (!bind_range) {
prev_image_offset = image_offset;
prev_bo_offset = bo ? column_bo_offset : 0;
bind_range = 4096;
} else if (prev_image_offset + bind_range == image_offset &&
(!bo || prev_bo_offset + bind_range == bo_offset)) {
bind_range += 4096;
} else {
tu_submit_add_bind(device, submit, &image->vma,
prev_image_offset, bo, prev_bo_offset,
bind_range);
prev_image_offset = image_offset;
prev_bo_offset = bo ? column_bo_offset : 0;
bind_range = 4096;
}
}
}
}
}
if (bind_range) {
tu_submit_add_bind(device, submit, &image->vma, prev_image_offset, bo,
prev_bo_offset, bind_range);
}
}

View file

@ -122,6 +122,7 @@ bool
ubwc_possible(struct tu_device *device,
VkFormat format,
VkImageType type,
VkImageCreateFlags flags,
VkImageUsageFlags usage,
VkImageUsageFlags stencil_usage,
const struct fd_dev_info *info,
@ -144,4 +145,9 @@ VkResult
tu_image_update_layout(struct tu_device *device, struct tu_image *image,
uint64_t modifier, const VkSubresourceLayout *plane_layouts);
void
tu_bind_sparse_image(struct tu_device *device, void *submit,
struct tu_image *image,
const VkSparseImageMemoryBind *bind);
#endif /* TU_IMAGE_H */

View file

@ -288,6 +288,14 @@ queue_submit_sparse(struct vk_queue *_queue, struct vk_queue_submit *vk_submit)
}
}
for (uint32_t i = 0; i < vk_submit->image_bind_count; i++) {
const VkSparseImageMemoryBindInfo *bind = &vk_submit->image_binds[i];
VK_FROM_HANDLE(tu_image, image, bind->image);
for (uint32_t j = 0; j < bind->bindCount; j++)
tu_bind_sparse_image(device, submit, image, &bind->pBinds[j]);
}
for (uint32_t i = 0; i < vk_submit->image_opaque_bind_count; i++) {
const VkSparseImageOpaqueMemoryBindInfo *bind =
&vk_submit->image_opaque_binds[i];