nvk/queue: Add support for non-opaque sparse binds

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/26719>

src/nouveau/vulkan/nvk_queue_drm_nouveau.c

@@ -148,6 +148,117 @@ push_add_buffer_bind(struct push_builder *pb,
    }
 }
 
+static void
+push_add_image_plane_bind(struct push_builder *pb,
+                          const struct nvk_image_plane *plane,
+                          const VkSparseImageMemoryBind *bind)
+{
+   VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
+   uint64_t image_bind_offset_B;
+
+   const uint64_t mem_bind_offset_B = bind->memoryOffset;
+   const uint32_t layer = bind->subresource.arrayLayer;
+   const uint32_t level = bind->subresource.mipLevel;
+
+   const struct nil_tiling plane_tiling = plane->nil.levels[level].tiling;
+   const uint32_t tile_size_B = nil_tiling_size_B(plane_tiling);
+
+   const struct nil_extent4d bind_extent_px = {
+      .w = bind->extent.width,
+      .h = bind->extent.height,
+      .d = bind->extent.depth,
+      .a = 1,
+   };
+   const struct nil_offset4d bind_offset_px = {
+      .x = bind->offset.x,
+      .y = bind->offset.y,
+      .z = bind->offset.z,
+      .a = layer,
+   };
+
+   const struct nil_extent4d level_extent_px =
+      nil_image_level_extent_px(&plane->nil, level);
+   const struct nil_extent4d level_extent_tl =
+      nil_extent4d_px_to_tl(level_extent_px, plane_tiling,
+                            plane->nil.format,
+                            plane->nil.sample_layout);
+
+   /* Convert the extent and offset to tiles */
+   struct nil_extent4d bind_extent_tl =
+      nil_extent4d_px_to_tl(bind_extent_px, plane_tiling,
+                            plane->nil.format,
+                            plane->nil.sample_layout);
+   struct nil_offset4d bind_offset_tl =
+      nil_offset4d_px_to_tl(bind_offset_px, plane_tiling,
+                            plane->nil.format,
+                            plane->nil.sample_layout);
+
+
+   image_bind_offset_B =
+      nil_image_level_layer_offset_B(&plane->nil, level, layer);
+
+   /* We can only bind contiguous ranges, so we'll split the image into rows
+    * of tiles that are guaranteed to be contiguous, and bind in terms of
+    * these rows
+    */
+
+   /* First, get the size of the bind. Since we have the extent in terms of
+    * tiles already, we just need to multiply that by the tile size to get
+    * the size in bytes
+    */
+   uint64_t row_bind_size_B = bind_extent_tl.width * tile_size_B;
+
+   /* Second, start walking the binding region in units of tiles, starting
+    * from the third dimension
+    */
+   for (uint32_t z_tl = 0; z_tl < bind_extent_tl.depth; z_tl++) {
+      /* Start walking the rows to be bound */
+      for (uint32_t y_tl = 0; y_tl < bind_extent_tl.height; y_tl++) {
+         /* For the bind offset, get a memory offset to the start of the row
+          * in terms of the bind extent
+          */
+         const uint64_t mem_row_start_tl =
+            y_tl * bind_extent_tl.width +
+            z_tl * bind_extent_tl.width * bind_extent_tl.height;
+
+         const uint32_t image_x_tl = bind_offset_tl.x;
+         const uint32_t image_y_tl = bind_offset_tl.y + y_tl;
+         const uint32_t image_z_tl = bind_offset_tl.z + z_tl;
+
+         /* The image offset is calculated in terms of the level extent */
+         const uint64_t image_row_start_tl =
+            image_x_tl +
+            image_y_tl * level_extent_tl.width +
+            image_z_tl * level_extent_tl.width * level_extent_tl.height;
+
+         push_bind(pb, &(struct drm_nouveau_vm_bind_op) {
+            .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP :
+                        DRM_NOUVEAU_VM_BIND_OP_UNMAP,
+            .handle = mem ? mem->bo->handle : 0,
+            .addr = plane->addr + image_bind_offset_B +
+                    image_row_start_tl * tile_size_B,
+            .bo_offset = mem_bind_offset_B +
+                         mem_row_start_tl * tile_size_B,
+            .range = row_bind_size_B,
+            .flags = plane->nil.pte_kind,
+         });
+      }
+   }
+}
+
+static void
+push_add_image_bind(struct push_builder *pb,
+                    VkSparseImageMemoryBindInfo *bind_info)
+{
+   VK_FROM_HANDLE(nvk_image, image, bind_info->image);
+   /* Sparse residency with multiplane is currently not supported */
+   assert(image->plane_count == 1);
+   for (unsigned i = 0; i < bind_info->bindCount; i++) {
+      push_add_image_plane_bind(pb, &image->planes[0],
+                                &bind_info->pBinds[i]);
+   }
+}
+
 static bool
 next_opaque_bind_plane(const VkSparseMemoryBind *bind,
                        uint64_t size_B, uint32_t align_B,
@@ -163,7 +274,10 @@ next_opaque_bind_plane(const VkSparseMemoryBind *bind,
    const uint64_t image_plane_offset_B = *image_plane_offset_B_iter;
    *image_plane_offset_B_iter += size_B;
 
-   const uint64_t bind_offset_B = bind->resourceOffset;
+   /* Offset into the image or image mip tail, as appropriate */
+   uint64_t bind_offset_B = bind->resourceOffset;
+   if (bind_offset_B >= NVK_MIP_TAIL_START_OFFSET)
+      bind_offset_B -= NVK_MIP_TAIL_START_OFFSET;
 
    if (bind_offset_B < image_plane_offset_B) {
       /* The offset of the plane within the bind */
@@ -223,22 +337,89 @@ push_add_image_plane_opaque_bind(struct push_builder *pb,
    });
 }
 
+static void
+push_add_image_plane_mip_tail_bind(struct push_builder *pb,
+                                   const struct nvk_image_plane *plane,
+                                   const VkSparseMemoryBind *bind,
+                                   uint64_t *image_plane_offset_B)
+{
+   const uint64_t mip_tail_offset_B =
+      nil_image_mip_tail_offset_B(&plane->nil);
+   const uint64_t mip_tail_size_B =
+      nil_image_mip_tail_size_B(&plane->nil);
+   const uint64_t mip_tail_stride_B = plane->nil.array_stride_B;
+
+   const uint64_t whole_mip_tail_size_B =
+      mip_tail_size_B * plane->nil.extent_px.a;
+
+   uint64_t plane_offset_B, mem_offset_B, bind_size_B;
+   if (!next_opaque_bind_plane(bind, whole_mip_tail_size_B, plane->nil.align_B,
+                               &plane_offset_B, &mem_offset_B, &bind_size_B,
+                               image_plane_offset_B))
+      return;
+
+   VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
+
+   /* Range within the virtual mip_tail space */
+   const uint64_t mip_bind_start_B = plane_offset_B;
+   const uint64_t mip_bind_end_B = mip_bind_start_B + bind_size_B;
+
+   /* Range of array slices covered by this bind */
+   const uint32_t start_a = mip_bind_start_B / mip_tail_size_B;
+   const uint32_t end_a = DIV_ROUND_UP(mip_bind_end_B, mip_tail_size_B);
+
+   for (uint32_t a = start_a; a < end_a; a++) {
+      /* Range within the virtual mip_tail space of this array slice */
+      const uint64_t a_mip_bind_start_B =
+         MAX2(a * mip_tail_size_B, mip_bind_start_B);
+      const uint64_t a_mip_bind_end_B =
+         MIN2((a + 1) * mip_tail_size_B, mip_bind_end_B);
+
+      /* Offset and range within this mip_tail slice */
+      const uint64_t a_offset_B = a_mip_bind_start_B - a * mip_tail_size_B;
+      const uint64_t a_range_B = a_mip_bind_end_B - a_mip_bind_start_B;
+
+      /* Offset within the current bind operation */
+      const uint64_t a_bind_offset_B =
+         a_mip_bind_start_B - mip_bind_start_B;
+
+      /* Offset within the image */
+      const uint64_t a_image_offset_B =
+         mip_tail_offset_B + (a * mip_tail_stride_B) + a_offset_B;
+
+      push_bind(pb, &(struct drm_nouveau_vm_bind_op) {
+         .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP :
+                     DRM_NOUVEAU_VM_BIND_OP_UNMAP,
+         .handle = mem ? mem->bo->handle : 0,
+         .addr = plane->addr + a_image_offset_B,
+         .bo_offset = mem_offset_B + a_bind_offset_B,
+         .range = a_range_B,
+         .flags = plane->nil.pte_kind,
+      });
+   }
+}
+
 static void
 push_add_image_opaque_bind(struct push_builder *pb,
                            VkSparseImageOpaqueMemoryBindInfo *bind_info)
 {
    VK_FROM_HANDLE(nvk_image, image, bind_info->image);
    for (unsigned i = 0; i < bind_info->bindCount; i++) {
+      const VkSparseMemoryBind *bind = &bind_info->pBinds[i];
+
       uint64_t image_plane_offset_B = 0;
       for (unsigned plane = 0; plane < image->plane_count; plane++) {
-         push_add_image_plane_opaque_bind(pb, &image->planes[plane],
-                                          &bind_info->pBinds[i],
-                                          &image_plane_offset_B);
+         if (bind->resourceOffset >= NVK_MIP_TAIL_START_OFFSET) {
+            push_add_image_plane_mip_tail_bind(pb, &image->planes[plane],
+                                               bind, &image_plane_offset_B);
+         } else {
+            push_add_image_plane_opaque_bind(pb, &image->planes[plane],
+                                             bind, &image_plane_offset_B);
+         }
       }
       if (image->stencil_copy_temp.nil.size_B > 0) {
          push_add_image_plane_opaque_bind(pb, &image->stencil_copy_temp,
-                                          &bind_info->pBinds[i],
-                                          &image_plane_offset_B);
+                                          bind, &image_plane_offset_B);
       }
    }
 }
@@ -414,6 +595,7 @@ nvk_queue_submit_drm_nouveau(struct nvk_queue *queue,
       return result;
 
    const bool is_vmbind = submit->buffer_bind_count > 0 ||
+                          submit->image_bind_count > 0  ||
                           submit->image_opaque_bind_count > 0;
    push_builder_init(queue, &pb, is_vmbind);
 
@@ -429,10 +611,14 @@ nvk_queue_submit_drm_nouveau(struct nvk_queue *queue,
       for (uint32_t i = 0; i < submit->buffer_bind_count; i++)
          push_add_buffer_bind(&pb, &submit->buffer_binds[i]);
 
+      for (uint32_t i = 0; i < submit->image_bind_count; i++)
+         push_add_image_bind(&pb, &submit->image_binds[i]);
+
       for (uint32_t i = 0; i < submit->image_opaque_bind_count; i++)
          push_add_image_opaque_bind(&pb, &submit->image_opaque_binds[i]);
    } else if (submit->command_buffer_count > 0) {
       assert(submit->buffer_bind_count == 0);
+      assert(submit->image_bind_count == 0);
       assert(submit->image_opaque_bind_count == 0);
 
       push_add_queue_state(&pb, &queue->state);