radv: Fix corrupted mipmap copies on GFX9+

GFX9+ hardware has an issue where mipmap degradations are calculated
incorrectly due to using divide-by-two integer math and certain mipmap
sizes lose blocks.

This issue has been documented before, and we ported a workaround from
AMDVLK to increase the extent that is programmed into the descriptor, so
that the hardware arrives at the correct result. However, this is
insufficient as we cannot safely increase the extent beyond the physical
extent of the image in memory. If we can't increase it enough, the image
will still be missing blocks.

But there is still hope. In cases where RADV is responsible for copying to
or from an image (such as vkCmdCopyBufferToImage/vkCmdCopyImageToBuffer),
we can perform a second copy of the blocks that the hardware excluded so
that the resulting image is complete. This is another workaround from
AMDVLK.

This fixes corrupted textures in Halo: The Master Chief Collection.

v2: Add RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_INV_VCACHE to flush_bits
    just in case (Samuel Pitoiset)

Closes: #3347

Signed-off-by: John Brooks <john@fastquake.com>
Reviewed-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Acked-by: Acked-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17970>

src/amd/common/ac_surface.c

@@ -3009,6 +3009,37 @@ uint64_t ac_surface_get_plane_size(const struct radeon_surf *surf,
    }
 }
 
+uint64_t
+ac_surface_addr_from_coord(struct ac_addrlib *addrlib, const struct radeon_info *info,
+                           const struct radeon_surf *surf, const struct ac_surf_info *surf_info,
+                           unsigned level, unsigned x, unsigned y, unsigned layer, bool is_3d)
+{
+   /* Only implemented for GFX9+ */
+   assert(info->gfx_level >= GFX9);
+
+   ADDR2_COMPUTE_SURFACE_ADDRFROMCOORD_INPUT input = {0};
+   input.size = sizeof(ADDR_COMPUTE_SURFACE_ADDRFROMCOORD_INPUT);
+   input.slice = layer;
+   input.mipId = level;
+   input.unalignedWidth = DIV_ROUND_UP(surf_info->width, surf->blk_w);
+   input.unalignedHeight = DIV_ROUND_UP(surf_info->height, surf->blk_h);
+   input.numSlices = is_3d ? surf_info->depth : surf_info->array_size;
+   input.numMipLevels = surf_info->levels;
+   input.numSamples = surf_info->samples;
+   input.numFrags = surf_info->samples;
+   input.swizzleMode = surf->u.gfx9.swizzle_mode;
+   input.resourceType = surf->u.gfx9.resource_type;
+   input.pipeBankXor = surf->tile_swizzle;
+   input.bpp = surf->bpe * 8;
+   input.x = x;
+   input.y = y;
+
+   ADDR2_COMPUTE_SURFACE_ADDRFROMCOORD_OUTPUT output = {0};
+   output.size = sizeof(ADDR2_COMPUTE_SURFACE_ADDRFROMCOORD_OUTPUT);
+   Addr2ComputeSurfaceAddrFromCoord(addrlib->handle, &input, &output);
+   return output.addr;
+}
+
 void ac_surface_print_info(FILE *out, const struct radeon_info *info,
                            const struct radeon_surf *surf)
 {

src/amd/common/ac_surface.h

@@ -470,6 +470,11 @@ uint64_t ac_surface_get_plane_stride(enum amd_gfx_level gfx_level,
 uint64_t ac_surface_get_plane_size(const struct radeon_surf *surf,
                                    unsigned plane);
 
+uint64_t ac_surface_addr_from_coord(struct ac_addrlib *addrlib, const struct radeon_info *info,
+                                    const struct radeon_surf *surf,
+                                    const struct ac_surf_info *surf_info, unsigned level,
+                                    unsigned x, unsigned y, unsigned layer, bool is_3d);
+
 void ac_surface_print_info(FILE *out, const struct radeon_info *info,
                            const struct radeon_surf *surf);
 

src/amd/vulkan/radv_meta_bufimage.c

@@ -1303,6 +1303,87 @@ create_bview_for_r32g32b32(struct radv_cmd_buffer *cmd_buffer, struct radv_buffe
                          });
 }
 
+/* GFX9+ has an issue where the HW does not calculate mipmap degradations
+ * for block-compressed images correctly (see the comment in
+ * radv_image_view_init). Some texels are unaddressable and cannot be copied
+ * to/from by a compute shader. Here we will perform a buffer copy to copy the
+ * texels that the hardware missed.
+ */
+static void
+fixup_gfx9_cs_copy(struct radv_cmd_buffer *cmd_buffer,
+                   const struct radv_meta_blit2d_buffer *buf_bsurf,
+                   const struct radv_meta_blit2d_surf *img_bsurf,
+                   const struct radv_meta_blit2d_rect *rect, bool to_image)
+{
+   const unsigned mip_level = img_bsurf->level;
+   const struct radv_image *image = img_bsurf->image;
+   const struct radeon_surf *surf = &image->planes[0].surface;
+   const struct radv_device *device = cmd_buffer->device;
+   const struct radeon_info *rad_info = &device->physical_device->rad_info;
+   struct ac_addrlib *addrlib = device->ws->get_addrlib(device->ws);
+
+   if (rad_info->gfx_level < GFX9 || image->vk.mip_levels == 1 ||
+       !vk_format_is_block_compressed(image->vk.format))
+      return;
+
+   /* The physical extent of the base mip */
+   VkExtent2D hw_base_extent = {surf->u.gfx9.base_mip_width, surf->u.gfx9.base_mip_height};
+
+   /* The hardware-calculated extent of the selected mip
+    * (naive divide-by-two integer math)
+    */
+   VkExtent2D hw_mip_extent = {radv_minify(hw_base_extent.width, mip_level),
+                               radv_minify(hw_base_extent.height, mip_level)};
+
+   /* The actual extent we want to copy */
+   VkExtent2D mip_extent = {rect->width, rect->height};
+
+   VkOffset2D mip_offset = {to_image ? rect->dst_x : rect->src_x,
+                            to_image ? rect->dst_y : rect->src_y};
+
+   if (hw_mip_extent.width >= mip_offset.x + mip_extent.width &&
+       hw_mip_extent.height >= mip_offset.y + mip_extent.height)
+      return;
+
+   if (!to_image) {
+      /* If we are writing to a buffer, then we need to wait for the compute
+       * shader to finish because it may write over the unaddressable texels
+       * while we're fixing them. If we're writing to an image, we do not need
+       * to wait because the compute shader cannot write to those texels
+       */
+      cmd_buffer->state.flush_bits |=
+         RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_INV_VCACHE;
+   }
+
+   for (uint32_t y = 0; y < mip_extent.width; y++) {
+      uint32_t coordY = y + mip_offset.y;
+      /* If the default copy algorithm (done previously) has already seen this
+       * scanline, then we can bias the starting X coordinate over to skip the
+       * region already copied by the default copy.
+       */
+      uint32_t x = (coordY < hw_mip_extent.height) ? hw_mip_extent.width : 0;
+      for (; x < mip_extent.width; x++) {
+         uint32_t coordX = x + mip_offset.x;
+         uint64_t addr = ac_surface_addr_from_coord(addrlib, rad_info, surf, &image->info,
+                                                    mip_level, coordX, coordY, img_bsurf->layer,
+                                                    image->vk.image_type == VK_IMAGE_TYPE_3D);
+         struct radeon_winsys_bo *img_bo = image->bindings[0].bo;
+         struct radeon_winsys_bo *mem_bo = buf_bsurf->buffer->bo;
+         const uint64_t img_offset = image->bindings[0].offset + addr;
+         /* buf_bsurf->offset already includes the layer offset */
+         const uint64_t mem_offset = buf_bsurf->buffer->offset +
+                                     buf_bsurf->offset +
+                                     y * buf_bsurf->pitch * surf->bpe +
+                                     x * surf->bpe;
+         if (to_image) {
+            radv_copy_buffer(cmd_buffer, mem_bo, img_bo, mem_offset, img_offset, surf->bpe);
+         } else {
+            radv_copy_buffer(cmd_buffer, img_bo, mem_bo, img_offset, mem_offset, surf->bpe);
+         }
+      }
+   }
+}
+
 static unsigned
 get_image_stride_for_r32g32b32(struct radv_cmd_buffer *cmd_buffer,
                                struct radv_meta_blit2d_surf *surf)
@@ -1378,6 +1459,7 @@ radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_b
                             16, push_constants);
 
       radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
+      fixup_gfx9_cs_copy(cmd_buffer, dst, src, &rects[r], false);
    }
 
    radv_image_view_finish(&src_view);
@@ -1532,6 +1614,7 @@ radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer,
                             16, push_constants);
 
       radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1);
+      fixup_gfx9_cs_copy(cmd_buffer, src, dst, &rects[r], true);
    }
 
    radv_image_view_finish(&dst_view);