pan/lib: introduce standard_sparse_mapping_granularity

And immediately implement it in terms of
DRM_FORMAT_MOD_ARM_INTERLEAVED_64K.

Also ban DRM_FORMAT_MOD_ARM_INTERLEAVED_64K for WSI in panfrost.
Normally, the modifier's test_props would take care of but as
panfrost doesn't use test_props, this has to be handled in
panfrost itself.

Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/38986>

src/gallium/drivers/panfrost/pan_screen.c

@@ -401,6 +401,13 @@ panfrost_walk_dmabuf_modifiers(struct pipe_screen *screen,
    for (unsigned i = 0; i < ARRAY_SIZE(native_mods); ++i) {
       uint64_t mod =  native_mods[i];
 
+      /* We don't implement the WSI interface for
+       * DRM_FORMAT_MOD_ARM_INTERLEAVED_64K so let's just not advertise images
+       * using this modifier, which will take care of it being advertised for
+       * WSI. */
+      if (mod == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K)
+         continue;
+
       if ((dev->debug & PAN_DBG_NO_AFBC) && drm_is_afbc(mod))
          continue;
 

src/panfrost/lib/pan_desc.c

@@ -360,6 +360,48 @@ GENX(pan_emit_afbc_zs_attachment)(const struct pan_fb_info *fb,
    }
 }
 
+#if PAN_ARCH >= 10
+void
+GENX(pan_emit_interleaved_64k_s_attachment)(const struct pan_fb_info *fb,
+                                            unsigned layer_or_z_slice, void *payload)
+{
+   const struct pan_image_view *s = fb->zs.view.s;
+   uint64_t base, row_stride, surf_stride;
+
+   get_tiled_or_linear_att_mem_props(pan_image_view_get_s_plane(s),
+                                     s->first_level, layer_or_z_slice, &base,
+                                     &row_stride, &surf_stride);
+   pan_cast_and_pack(payload, S_TARGET, cfg) {
+      cfg.msaa = mali_sampling_mode(s);
+      cfg.write_format = translate_s_format(s->format);
+      cfg.block_format = MALI_BLOCK_FORMAT_INTERLEAVED_64K;
+      cfg.base = base;
+      cfg.row_stride = row_stride;
+      SET_SURFACE_STRIDE(cfg, surf_stride);
+   }
+}
+
+void
+GENX(pan_emit_interleaved_64k_zs_attachment)(const struct pan_fb_info *fb,
+                                             unsigned layer_or_z_slice, void *payload)
+{
+   const struct pan_image_view *zs = fb->zs.view.zs;
+   uint64_t base, row_stride, surf_stride;
+
+   get_tiled_or_linear_att_mem_props(pan_image_view_get_zs_plane(zs),
+                                     zs->first_level, layer_or_z_slice, &base,
+                                     &row_stride, &surf_stride);
+   pan_cast_and_pack(payload, ZS_TARGET, cfg) {
+      cfg.msaa = mali_sampling_mode(zs);
+      cfg.write_format = translate_zs_format(zs->format);
+      cfg.block_format = MALI_BLOCK_FORMAT_INTERLEAVED_64K;
+      cfg.base = base;
+      cfg.row_stride = row_stride;
+      SET_SURFACE_STRIDE(cfg, surf_stride);
+   }
+}
+#endif
+
 static void
 pan_prepare_crc(const struct pan_fb_info *fb, int rt_crc,
                 struct MALI_CRC *crc)
@@ -774,6 +816,35 @@ GENX(pan_emit_linear_color_attachment)(const struct pan_fb_info *fb,
 }
 
 #if PAN_ARCH >= 10
+void
+GENX(pan_emit_interleaved_64k_color_attachment)(const struct pan_fb_info *fb,
+                                                unsigned rt_idx,
+                                                unsigned layer_or_z_slice,
+                                                unsigned cbuf_offset, void *payload)
+{
+   const struct pan_fb_color_attachment *rt = &fb->rts[rt_idx];
+   const struct pan_image_view *iview = rt->view;
+   uint64_t base, row_stride, surf_stride;
+
+   get_tiled_or_linear_att_mem_props(pan_image_view_get_color_plane(iview),
+                                     iview->first_level, layer_or_z_slice,
+                                     &base, &row_stride, &surf_stride);
+
+   /* TODO: YUV RT. */
+   assert(!pan_format_is_yuv(iview->format));
+   pan_cast_and_pack(payload, RGB_RENDER_TARGET, cfg) {
+      rt_common_cfg(rt, cbuf_offset, fb->tile_size, cfg);
+      cfg.write_enable = true;
+      cfg.writeback_block_format = MALI_BLOCK_FORMAT_INTERLEAVED_64K;
+      get_rt_formats(iview->format, &cfg.writeback_format, &cfg.internal_format,
+                     &cfg.swizzle);
+      cfg.srgb = util_format_is_srgb(iview->format);
+      cfg.writeback_buffer.base = base;
+      cfg.writeback_buffer.row_stride = row_stride;
+      cfg.writeback_buffer.surface_stride = surf_stride;
+   }
+}
+
 void
 GENX(pan_emit_afrc_color_attachment)(const struct pan_fb_info *fb,
                                      unsigned rt_idx, unsigned layer_or_z_slice,

src/panfrost/lib/pan_desc.h

@@ -290,6 +290,18 @@ void GENX(pan_emit_afbc_s_attachment)(const struct pan_fb_info *fb,
 #endif
 
 #if PAN_ARCH >= 10
+void
+GENX(pan_emit_interleaved_64k_color_attachment)(const struct pan_fb_info *fb,
+                                                unsigned rt_idx,
+                                                unsigned layer_or_z_slice,
+                                                unsigned cbuf_offset, void *payload);
+void GENX(pan_emit_interleaved_64k_zs_attachment)(const struct pan_fb_info *fb,
+                                                  unsigned layer_or_z_slice,
+                                                  void *payload);
+void GENX(pan_emit_interleaved_64k_s_attachment)(const struct pan_fb_info *fb,
+                                                 unsigned layer_or_z_slice,
+                                                 void *payload);
+
 void GENX(pan_emit_afrc_color_attachment)(const struct pan_fb_info *fb,
                                           unsigned rt_idx,
                                           unsigned layer_or_z_slice,

src/panfrost/lib/pan_format.h

@@ -44,6 +44,7 @@
       DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 |               \
                               AFBC_FORMAT_MOD_SPARSE),                         \
                                                                                \
+      DRM_FORMAT_MOD_ARM_INTERLEAVED_64K,                                      \
       DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED,                            \
       DRM_FORMAT_MOD_LINEAR,                                                   \
                                                                                \

src/panfrost/lib/pan_image.h

@@ -255,6 +255,9 @@ struct pan_image_usage {
    /* PAN_BIND_xxx flags. */
    uint32_t bind;
 
+   /* Image needs to be mappable at standard sparse block granularity. */
+   bool standard_sparse_mapping_granularity;
+
    /* Image filled directly from the CPU. */
    bool host_copy;
 

src/panfrost/lib/pan_layout.h

@@ -255,6 +255,22 @@ pan_u_interleaved_tile_size_el(enum pipe_format format)
    }
 }
 
+/* Given a format, determine the tile size used for interleaved 64k. */
+static inline struct pan_image_block_size
+pan_interleaved_64k_tile_size_el(enum pipe_format format)
+{
+   switch (util_format_get_blocksize(format)) {
+   case 1: return (struct pan_image_block_size){256, 256};
+   case 2: return (struct pan_image_block_size){256, 128};
+   case 4: return (struct pan_image_block_size){128, 128};
+   case 8: return (struct pan_image_block_size){128, 64};
+   case 16: return (struct pan_image_block_size){64, 64};
+   default:
+      UNREACHABLE("unsupported format");
+      return (struct pan_image_block_size){0, 0};
+   }
+}
+
 #ifdef __cplusplus
 } /* extern C */
 #endif

src/panfrost/lib/pan_mod.c

@@ -195,6 +195,10 @@ pan_mod_afbc_test_props(const struct pan_kmod_dev_props *dprops,
    if (iusage && iusage->bind & PAN_BIND_STORAGE_IMAGE)
       return PAN_MOD_NOT_SUPPORTED;
 
+   /* We don't implement mapping individual tiles with AFBC. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
    /* AFBC not supported. */
    if (!pan_query_afbc(dprops))
       return PAN_MOD_NOT_SUPPORTED;
@@ -312,6 +316,10 @@ pan_mod_afrc_test_props(const struct pan_kmod_dev_props *dprops,
    if (iusage && iusage->bind & PAN_BIND_STORAGE_IMAGE)
       return PAN_MOD_NOT_SUPPORTED;
 
+   /* We don't implement mapping individual tiles with AFRC. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
    /* We can't write to an AFRC resource directly. */
    if (iusage && iusage->host_copy)
       return PAN_MOD_NOT_SUPPORTED;
@@ -445,6 +453,10 @@ pan_mod_u_tiled_test_props(const struct pan_kmod_dev_props *dprops,
    if (pan_format_is_yuv(iprops->format))
       return PAN_MOD_NOT_SUPPORTED;
 
+   /* We don't implement mapping individual tiles in this layout. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
    /* The purpose of tiling is improving locality in both X- and
     * Y-directions. If there is only a single pixel in either direction,
     * tiling does not make sense; using a linear layout instead is optimal
@@ -574,6 +586,103 @@ pan_mod_u_tiled_init_slice_layout(
 #define pan_mod_u_tiled_emit_zs_attachment GENX(pan_emit_u_tiled_zs_attachment)
 #define pan_mod_u_tiled_emit_s_attachment  GENX(pan_emit_u_tiled_s_attachment)
 
+#if PAN_ARCH >= 10
+#define pan_mod_interleaved_64k_init_plane_layout NULL
+
+static bool
+pan_mod_interleaved_64k_init_slice_layout(
+   const struct pan_image_props *props, unsigned plane_idx,
+   struct pan_image_extent mip_extent_px,
+   const struct pan_image_layout_constraints *layout_constraints,
+   struct pan_image_slice_layout *slayout)
+{
+   assert(!(layout_constraints && layout_constraints->wsi_row_pitch_B));
+
+   struct pan_image_block_size tile_extent_el =
+      pan_interleaved_64k_tile_size_el(props->format);
+   struct pan_image_block_size tile_extent_px = {
+      tile_extent_el.width * util_format_get_blockwidth(props->format),
+      tile_extent_el.height * util_format_get_blockheight(props->format),
+   };
+   uint64_t tile_size_B = 65536;
+
+   struct pan_image_extent mip_extent_tiles = {
+      DIV_ROUND_UP(mip_extent_px.width, tile_extent_px.width),
+      DIV_ROUND_UP(mip_extent_px.height, tile_extent_px.height),
+      mip_extent_px.depth,
+   };
+
+   uint64_t row_stride_B = mip_extent_tiles.width * tile_size_B;
+   uint64_t surf_stride_B = mip_extent_tiles.height * row_stride_B;
+
+   slayout->offset_B = layout_constraints ? layout_constraints->offset_B : 0;
+   slayout->size_B = mip_extent_tiles.depth * surf_stride_B;
+   slayout->tiled_or_linear.row_stride_B = row_stride_B;
+   slayout->tiled_or_linear.surface_stride_B = surf_stride_B;
+
+   if (slayout->size_B > MAX_SIZE_B ||
+       slayout->tiled_or_linear.surface_stride_B > MAX_SLICE_STRIDE_B)
+      return false;
+
+   return true;
+}
+
+static uint32_t
+pan_mod_interleaved_64k_get_wsi_row_pitch(const struct pan_image *image,
+                                          unsigned plane_idx, unsigned mip_level)
+{
+   UNREACHABLE("interleaved 64k cannot be used for wsi");
+}
+
+static bool
+pan_mod_interleaved_64k_match(uint64_t mod)
+{
+   return mod == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K;
+}
+
+static enum pan_mod_support
+pan_mod_interleaved_64k_test_props(const struct pan_kmod_dev_props *dprops,
+                                   const struct pan_image_props *iprops,
+                                   const struct pan_image_usage *iusage)
+{
+   assert(GENX(pan_format_from_pipe_format)(iprops->format)->hw);
+
+   /* YUV not supported. */
+   if (pan_format_is_yuv(iprops->format))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* Non-po2 byte texel blocks not supported. */
+   if (!util_is_power_of_two_nonzero(util_format_get_blocksize(iprops->format)))
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement multisampling with this layout. */
+   if (iprops->nr_samples > 1)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't implement tiling/detiling of this layout on host. */
+   if (iusage->host_copy)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   /* We don't respect wsi_row_pitch_B so this layout is not usable for WSI. */
+   if (iusage->wsi)
+      return PAN_MOD_NOT_SUPPORTED;
+
+   if (iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_OPTIMAL;
+
+   /* This layout doesn't offer perf benefits over plain U-interleaved but uses
+    * more memory. */
+   return PAN_MOD_NOT_OPTIMAL;
+}
+
+#define pan_mod_interleaved_64k_emit_tex_payload_entry                                 \
+   GENX(pan_tex_emit_interleaved_64k_payload_entry)
+#define pan_mod_interleaved_64k_emit_color_attachment                                  \
+   GENX(pan_emit_interleaved_64k_color_attachment)
+#define pan_mod_interleaved_64k_emit_zs_attachment GENX(pan_emit_interleaved_64k_zs_attachment)
+#define pan_mod_interleaved_64k_emit_s_attachment  GENX(pan_emit_interleaved_64k_s_attachment)
+#endif
+
 static bool
 pan_mod_linear_match(uint64_t mod)
 {
@@ -587,6 +696,11 @@ pan_mod_linear_test_props(const struct pan_kmod_dev_props *dprops,
 {
    assert(GENX(pan_format_from_pipe_format)(iprops->format)->hw);
 
+   /* We can't implement mapping of tiles at standard sparse granularity using
+    * this layout. */
+   if (iusage && iusage->standard_sparse_mapping_granularity)
+      return PAN_MOD_NOT_SUPPORTED;
+
    switch (iprops->format) {
    /* AFBC-only formats. */
    case PIPE_FORMAT_R8G8B8_420_UNORM_PACKED:
@@ -724,6 +838,7 @@ static const struct pan_mod_handler pan_mod_handlers[] = {
    PAN_MOD_DEF(u_tiled),
    PAN_MOD_DEF(linear),
 #if PAN_ARCH >= 10
+   PAN_MOD_DEF(interleaved_64k),
    PAN_MOD_DEF(afrc),
 #endif
 };

src/panfrost/lib/pan_texture.c

@@ -292,6 +292,23 @@ pan_clump_format(enum pipe_format format)
    }
 }
 
+static enum mali_clump_ordering
+modifier_clump_ordering(uint64_t modifier)
+{
+   switch (modifier) {
+   case DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED:
+      return MALI_CLUMP_ORDERING_TILED_U_INTERLEAVED;
+#if PAN_ARCH >= 10
+   case DRM_FORMAT_MOD_ARM_INTERLEAVED_64K:
+      return MALI_CLUMP_ORDERING_INTERLEAVED_64K;
+#endif
+   case DRM_FORMAT_MOD_LINEAR:
+      return MALI_CLUMP_ORDERING_LINEAR;
+   default:
+      UNREACHABLE("");
+   }
+}
+
 static enum mali_afbc_superblock_size
 translate_superblock_size(uint64_t modifier)
 {
@@ -391,6 +408,7 @@ emit_generic_plane(const struct pan_image_view *iview, int plane_idx,
    /* 3-planar formats must use Chroma 2p planes for the U V planes. */
    assert(plane_idx == 0 || desc->layout != UTIL_FORMAT_LAYOUT_PLANAR3);
    assert(props->modifier == DRM_FORMAT_MOD_LINEAR ||
+          props->modifier == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K ||
           props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED);
 
    get_linear_or_u_tiled_plane_props(iview, plane_idx, mip_level,
@@ -398,10 +416,7 @@ emit_generic_plane(const struct pan_image_view *iview, int plane_idx,
                                      &slice_stride, &plane_size);
 
    pan_cast_and_pack(payload, GENERIC_PLANE, cfg) {
-      cfg.clump_ordering =
+      cfg.clump_ordering = modifier_clump_ordering(props->modifier);
-         props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
-            ? MALI_CLUMP_ORDERING_TILED_U_INTERLEAVED
-            : MALI_CLUMP_ORDERING_LINEAR;
       cfg.clump_format = pan_clump_format(iview->format);
       PLANE_SET_SIZE(cfg, plane_size);
       cfg.pointer = plane_addr;
@@ -434,6 +449,7 @@ emit_astc_plane(const struct pan_image_view *iview, int plane_idx,
 
    assert(desc->layout == UTIL_FORMAT_LAYOUT_ASTC && desc->block.depth == 1);
    assert(props->modifier == DRM_FORMAT_MOD_LINEAR ||
+          props->modifier == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K ||
           props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED);
 
    get_linear_or_u_tiled_plane_props(iview, plane_idx, mip_level,
@@ -441,10 +457,7 @@ emit_astc_plane(const struct pan_image_view *iview, int plane_idx,
                                      &slice_stride, &plane_size);
 
 #define ASTC_PLANE_SET_COMMON_PROPS()                                          \
-   cfg.clump_ordering =                                                        \
+   cfg.clump_ordering = modifier_clump_ordering(props->modifier);              \
-      props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED          \
-         ? MALI_CLUMP_ORDERING_TILED_U_INTERLEAVED                             \
-         : MALI_CLUMP_ORDERING_LINEAR;                                         \
    cfg.decode_hdr = iview->astc.hdr;                                           \
    cfg.decode_wide = wide;                                                     \
    PLANE_SET_SIZE(cfg, plane_size);                                            \
@@ -518,13 +531,11 @@ emit_linear_or_u_tiled_chroma_2p_plane(const struct pan_image_view *iview,
 
    assert(desc->layout == UTIL_FORMAT_LAYOUT_PLANAR3);
    assert(props->modifier == DRM_FORMAT_MOD_LINEAR ||
+          props->modifier == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K ||
           props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED);
 
    pan_cast_and_pack(payload, CHROMA_2P_PLANE, cfg) {
-      cfg.clump_ordering =
+      cfg.clump_ordering = modifier_clump_ordering(props->modifier);
-         props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED
-            ? MALI_CLUMP_ORDERING_TILED_U_INTERLEAVED
-            : MALI_CLUMP_ORDERING_LINEAR;
       cfg.clump_format = pan_clump_format(iview->format);
       PLANE_SET_SIZE(cfg, cplane1_size);
       cfg.pointer = cplane1_addr;
@@ -748,6 +759,16 @@ emit_afrc_chroma_2p_plane(const struct pan_image_view *iview,
       cfg.secondary_pointer = cplane2_addr;
    }
 }
+
+#define emit_interleaved_64k_plane emit_linear_or_u_tiled_plane
+
+static void
+emit_interleaved_64k_chroma_2p_plane(const struct pan_image_view *iview,
+                                     unsigned mip_level,
+                                     unsigned layer_or_z_slice, void *payload)
+{
+   UNREACHABLE("not implemented");
+}
 #endif
 
 #else
@@ -1005,6 +1026,7 @@ PAN_TEX_EMIT_HELPER(linear)
 PAN_TEX_EMIT_HELPER(u_tiled)
 PAN_TEX_EMIT_HELPER(afbc)
 #if PAN_ARCH >= 10
+PAN_TEX_EMIT_HELPER(interleaved_64k)
 PAN_TEX_EMIT_HELPER(afrc)
 #endif
 

src/panfrost/lib/pan_texture.h

@@ -69,6 +69,10 @@ void GENX(pan_tex_emit_afbc_payload_entry)(const struct pan_image_view *iview,
                                            unsigned sample, void **payload);
 
 #if PAN_ARCH >= 10
+void GENX(pan_tex_emit_interleaved_64k_payload_entry)(
+      const struct pan_image_view *iview, unsigned mip_level,
+      unsigned layer_or_z_slice, unsigned sample, void **payload);
+
 void GENX(pan_tex_emit_afrc_payload_entry)(
       const struct pan_image_view *iview, unsigned mip_level,
       unsigned layer_or_z_slice, unsigned sample, void **payload);

src/panfrost/lib/tests/test-layout.cpp

@@ -573,6 +573,8 @@ format_can_do_mod(unsigned arch, enum pipe_format format, unsigned plane_idx,
       return pan_afbc_format(arch, format, plane_idx) != PAN_AFBC_MODE_INVALID;
    } else if (drm_is_afrc(modifier)) {
       return arch >= 10 && pan_afrc_supports_format(format);
+   } else if (modifier == DRM_FORMAT_MOD_ARM_INTERLEAVED_64K) {
+      return false;
    } else {
       assert(modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED ||
              modifier == DRM_FORMAT_MOD_LINEAR);