From db45b29f5f596d02fc610429fee7a311a8aea960 Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Fri, 8 Mar 2024 13:37:54 -0600 Subject: [PATCH] nvk/queue: Add support for non-opaque sparse binds Part-of: --- src/nouveau/vulkan/nvk_queue_drm_nouveau.c | 198 ++++++++++++++++++++- 1 file changed, 192 insertions(+), 6 deletions(-) diff --git a/src/nouveau/vulkan/nvk_queue_drm_nouveau.c b/src/nouveau/vulkan/nvk_queue_drm_nouveau.c index 1c9d67dbb0a..2ee5abd7972 100644 --- a/src/nouveau/vulkan/nvk_queue_drm_nouveau.c +++ b/src/nouveau/vulkan/nvk_queue_drm_nouveau.c @@ -148,6 +148,117 @@ push_add_buffer_bind(struct push_builder *pb, } } +static void +push_add_image_plane_bind(struct push_builder *pb, + const struct nvk_image_plane *plane, + const VkSparseImageMemoryBind *bind) +{ + VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory); + uint64_t image_bind_offset_B; + + const uint64_t mem_bind_offset_B = bind->memoryOffset; + const uint32_t layer = bind->subresource.arrayLayer; + const uint32_t level = bind->subresource.mipLevel; + + const struct nil_tiling plane_tiling = plane->nil.levels[level].tiling; + const uint32_t tile_size_B = nil_tiling_size_B(plane_tiling); + + const struct nil_extent4d bind_extent_px = { + .w = bind->extent.width, + .h = bind->extent.height, + .d = bind->extent.depth, + .a = 1, + }; + const struct nil_offset4d bind_offset_px = { + .x = bind->offset.x, + .y = bind->offset.y, + .z = bind->offset.z, + .a = layer, + }; + + const struct nil_extent4d level_extent_px = + nil_image_level_extent_px(&plane->nil, level); + const struct nil_extent4d level_extent_tl = + nil_extent4d_px_to_tl(level_extent_px, plane_tiling, + plane->nil.format, + plane->nil.sample_layout); + + /* Convert the extent and offset to tiles */ + struct nil_extent4d bind_extent_tl = + nil_extent4d_px_to_tl(bind_extent_px, plane_tiling, + plane->nil.format, + plane->nil.sample_layout); + struct nil_offset4d bind_offset_tl = + nil_offset4d_px_to_tl(bind_offset_px, plane_tiling, + plane->nil.format, + plane->nil.sample_layout); + + + image_bind_offset_B = + nil_image_level_layer_offset_B(&plane->nil, level, layer); + + /* We can only bind contiguous ranges, so we'll split the image into rows + * of tiles that are guaranteed to be contiguous, and bind in terms of + * these rows + */ + + /* First, get the size of the bind. Since we have the extent in terms of + * tiles already, we just need to multiply that by the tile size to get + * the size in bytes + */ + uint64_t row_bind_size_B = bind_extent_tl.width * tile_size_B; + + /* Second, start walking the binding region in units of tiles, starting + * from the third dimension + */ + for (uint32_t z_tl = 0; z_tl < bind_extent_tl.depth; z_tl++) { + /* Start walking the rows to be bound */ + for (uint32_t y_tl = 0; y_tl < bind_extent_tl.height; y_tl++) { + /* For the bind offset, get a memory offset to the start of the row + * in terms of the bind extent + */ + const uint64_t mem_row_start_tl = + y_tl * bind_extent_tl.width + + z_tl * bind_extent_tl.width * bind_extent_tl.height; + + const uint32_t image_x_tl = bind_offset_tl.x; + const uint32_t image_y_tl = bind_offset_tl.y + y_tl; + const uint32_t image_z_tl = bind_offset_tl.z + z_tl; + + /* The image offset is calculated in terms of the level extent */ + const uint64_t image_row_start_tl = + image_x_tl + + image_y_tl * level_extent_tl.width + + image_z_tl * level_extent_tl.width * level_extent_tl.height; + + push_bind(pb, &(struct drm_nouveau_vm_bind_op) { + .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP : + DRM_NOUVEAU_VM_BIND_OP_UNMAP, + .handle = mem ? mem->bo->handle : 0, + .addr = plane->addr + image_bind_offset_B + + image_row_start_tl * tile_size_B, + .bo_offset = mem_bind_offset_B + + mem_row_start_tl * tile_size_B, + .range = row_bind_size_B, + .flags = plane->nil.pte_kind, + }); + } + } +} + +static void +push_add_image_bind(struct push_builder *pb, + VkSparseImageMemoryBindInfo *bind_info) +{ + VK_FROM_HANDLE(nvk_image, image, bind_info->image); + /* Sparse residency with multiplane is currently not supported */ + assert(image->plane_count == 1); + for (unsigned i = 0; i < bind_info->bindCount; i++) { + push_add_image_plane_bind(pb, &image->planes[0], + &bind_info->pBinds[i]); + } +} + static bool next_opaque_bind_plane(const VkSparseMemoryBind *bind, uint64_t size_B, uint32_t align_B, @@ -163,7 +274,10 @@ next_opaque_bind_plane(const VkSparseMemoryBind *bind, const uint64_t image_plane_offset_B = *image_plane_offset_B_iter; *image_plane_offset_B_iter += size_B; - const uint64_t bind_offset_B = bind->resourceOffset; + /* Offset into the image or image mip tail, as appropriate */ + uint64_t bind_offset_B = bind->resourceOffset; + if (bind_offset_B >= NVK_MIP_TAIL_START_OFFSET) + bind_offset_B -= NVK_MIP_TAIL_START_OFFSET; if (bind_offset_B < image_plane_offset_B) { /* The offset of the plane within the bind */ @@ -223,22 +337,89 @@ push_add_image_plane_opaque_bind(struct push_builder *pb, }); } +static void +push_add_image_plane_mip_tail_bind(struct push_builder *pb, + const struct nvk_image_plane *plane, + const VkSparseMemoryBind *bind, + uint64_t *image_plane_offset_B) +{ + const uint64_t mip_tail_offset_B = + nil_image_mip_tail_offset_B(&plane->nil); + const uint64_t mip_tail_size_B = + nil_image_mip_tail_size_B(&plane->nil); + const uint64_t mip_tail_stride_B = plane->nil.array_stride_B; + + const uint64_t whole_mip_tail_size_B = + mip_tail_size_B * plane->nil.extent_px.a; + + uint64_t plane_offset_B, mem_offset_B, bind_size_B; + if (!next_opaque_bind_plane(bind, whole_mip_tail_size_B, plane->nil.align_B, + &plane_offset_B, &mem_offset_B, &bind_size_B, + image_plane_offset_B)) + return; + + VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory); + + /* Range within the virtual mip_tail space */ + const uint64_t mip_bind_start_B = plane_offset_B; + const uint64_t mip_bind_end_B = mip_bind_start_B + bind_size_B; + + /* Range of array slices covered by this bind */ + const uint32_t start_a = mip_bind_start_B / mip_tail_size_B; + const uint32_t end_a = DIV_ROUND_UP(mip_bind_end_B, mip_tail_size_B); + + for (uint32_t a = start_a; a < end_a; a++) { + /* Range within the virtual mip_tail space of this array slice */ + const uint64_t a_mip_bind_start_B = + MAX2(a * mip_tail_size_B, mip_bind_start_B); + const uint64_t a_mip_bind_end_B = + MIN2((a + 1) * mip_tail_size_B, mip_bind_end_B); + + /* Offset and range within this mip_tail slice */ + const uint64_t a_offset_B = a_mip_bind_start_B - a * mip_tail_size_B; + const uint64_t a_range_B = a_mip_bind_end_B - a_mip_bind_start_B; + + /* Offset within the current bind operation */ + const uint64_t a_bind_offset_B = + a_mip_bind_start_B - mip_bind_start_B; + + /* Offset within the image */ + const uint64_t a_image_offset_B = + mip_tail_offset_B + (a * mip_tail_stride_B) + a_offset_B; + + push_bind(pb, &(struct drm_nouveau_vm_bind_op) { + .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP : + DRM_NOUVEAU_VM_BIND_OP_UNMAP, + .handle = mem ? mem->bo->handle : 0, + .addr = plane->addr + a_image_offset_B, + .bo_offset = mem_offset_B + a_bind_offset_B, + .range = a_range_B, + .flags = plane->nil.pte_kind, + }); + } +} + static void push_add_image_opaque_bind(struct push_builder *pb, VkSparseImageOpaqueMemoryBindInfo *bind_info) { VK_FROM_HANDLE(nvk_image, image, bind_info->image); for (unsigned i = 0; i < bind_info->bindCount; i++) { + const VkSparseMemoryBind *bind = &bind_info->pBinds[i]; + uint64_t image_plane_offset_B = 0; for (unsigned plane = 0; plane < image->plane_count; plane++) { - push_add_image_plane_opaque_bind(pb, &image->planes[plane], - &bind_info->pBinds[i], - &image_plane_offset_B); + if (bind->resourceOffset >= NVK_MIP_TAIL_START_OFFSET) { + push_add_image_plane_mip_tail_bind(pb, &image->planes[plane], + bind, &image_plane_offset_B); + } else { + push_add_image_plane_opaque_bind(pb, &image->planes[plane], + bind, &image_plane_offset_B); + } } if (image->stencil_copy_temp.nil.size_B > 0) { push_add_image_plane_opaque_bind(pb, &image->stencil_copy_temp, - &bind_info->pBinds[i], - &image_plane_offset_B); + bind, &image_plane_offset_B); } } } @@ -414,6 +595,7 @@ nvk_queue_submit_drm_nouveau(struct nvk_queue *queue, return result; const bool is_vmbind = submit->buffer_bind_count > 0 || + submit->image_bind_count > 0 || submit->image_opaque_bind_count > 0; push_builder_init(queue, &pb, is_vmbind); @@ -429,10 +611,14 @@ nvk_queue_submit_drm_nouveau(struct nvk_queue *queue, for (uint32_t i = 0; i < submit->buffer_bind_count; i++) push_add_buffer_bind(&pb, &submit->buffer_binds[i]); + for (uint32_t i = 0; i < submit->image_bind_count; i++) + push_add_image_bind(&pb, &submit->image_binds[i]); + for (uint32_t i = 0; i < submit->image_opaque_bind_count; i++) push_add_image_opaque_bind(&pb, &submit->image_opaque_binds[i]); } else if (submit->command_buffer_count > 0) { assert(submit->buffer_bind_count == 0); + assert(submit->image_bind_count == 0); assert(submit->image_opaque_bind_count == 0); push_add_queue_state(&pb, &queue->state);