brw: implement read without format lowering

Load the format enum and then just go through a series of :

   if format == R16G16B16A16_UNORM
      color = lower_r32g32_uint_tor_r16g16b16a16_unorm(color)
   else if format == R16G16B16A16_SNORM
      ...

For Gfx12.5, there is no in-shader conversion.

For Gfx12/11, the in-shader conversion covers the following formats :
    - ISL_FORMAT_R10G10B10A2_UNORM
    - ISL_FORMAT_R10G10B10A2_UINT
    - ISL_FORMAT_R11G11B10_FLOAT

For Gfx9, the following formats :
    - ISL_FORMAT_R16G16B16A16_UNORM
    - ISL_FORMAT_R16G16B16A16_SNORM
    - ISL_FORMAT_R10G10B10A2_UNORM
    - ISL_FORMAT_R10G10B10A2_UINT
    - ISL_FORMAT_R8G8B8A8_UNORM
    - ISL_FORMAT_R8G8B8A8_SNORM
    - ISL_FORMAT_R16G16_UNORM
    - ISL_FORMAT_R16G16_SNORM
    - ISL_FORMAT_R11G11B10_FLOAT
    - ISL_FORMAT_R8G8_UNORM
    - ISL_FORMAT_R8G8_SNORM
    - ISL_FORMAT_R16_UNORM
    - ISL_FORMAT_R16_SNORM
    - ISL_FORMAT_R8_UNORM
    - ISL_FORMAT_R8_SNORM

Signed-off-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Reviewed-by: Caio Oliveira <caio.oliveira@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22524>

src/gallium/drivers/iris/iris_program.c

@@ -3785,7 +3785,6 @@ static char *
 iris_finalize_nir(struct pipe_screen *_screen, struct nir_shader *nir)
 {
    struct iris_screen *screen = (struct iris_screen *)_screen;
-   const struct intel_device_info *devinfo = screen->devinfo;
 
    NIR_PASS_V(nir, iris_fix_edge_flags);
 
@@ -3794,13 +3793,14 @@ iris_finalize_nir(struct pipe_screen *_screen, struct nir_shader *nir)
       brw_preprocess_nir(screen->brw, nir, &opts);
 
       NIR_PASS_V(nir, brw_nir_lower_storage_image,
+                 screen->brw,
                  &(struct brw_nir_lower_storage_image_opts) {
-                    .devinfo      = devinfo,
                     .lower_loads  = true,
                     .lower_stores = true,
                  });
    } else {
 #ifdef INTEL_USE_ELK
+      const struct intel_device_info *devinfo = screen->devinfo;
       assert(screen->elk);
 
       struct elk_nir_compiler_opts opts = {};

src/intel/compiler/brw_compiler.c

@@ -27,6 +27,7 @@
 #include "brw_private.h"
 #include "dev/intel_debug.h"
 #include "compiler/nir/nir.h"
+#include "isl/isl.h"
 #include "util/u_debug.h"
 
 const struct nir_shader_compiler_options brw_scalar_nir_options = {
@@ -194,6 +195,31 @@ brw_compiler_create(void *mem_ctx, const struct intel_device_info *devinfo)
       compiler->nir_options[i] = nir_options;
    }
 
+   /* Build a list of storage format compatible in component bit size &
+    * isl_base_type. We can apply the same lowering to those.
+    */
+   compiler->num_lowered_storage_formats = 0;
+   for (enum isl_format fmt = 0; fmt < ISL_FORMAT_RAW; fmt++) {
+      if (!isl_is_storage_image_format(devinfo, fmt))
+         continue;
+
+      if (isl_lower_storage_image_format(devinfo, fmt) == fmt)
+         continue;
+
+      compiler->lowered_storage_formats =
+         reralloc(compiler, compiler->lowered_storage_formats,
+                  uint32_t, compiler->num_lowered_storage_formats + 1);
+      compiler->lowered_storage_formats[
+         compiler->num_lowered_storage_formats++] = fmt;
+   }
+   assert((devinfo->verx10 >= 125 &&
+           compiler->num_lowered_storage_formats == 0) ||
+          (devinfo->verx10 >= 110 && devinfo->verx10 <= 120 &&
+           compiler->num_lowered_storage_formats == 3) ||
+          devinfo->verx10 == 90);
+   fprintf(stderr, "num_lowered_storage_formats=%i\n",
+           compiler->num_lowered_storage_formats);
+
    return compiler;
 }
 

src/intel/compiler/brw_compiler.h

@@ -49,6 +49,11 @@ typedef struct nir_shader nir_shader;
 
 #define REG_CLASS_COUNT 20
 
+struct brw_storage_format {
+   uint32_t num_formats;
+   uint32_t isl_formats[3];
+};
+
 struct brw_compiler {
    const struct intel_device_info *devinfo;
 
@@ -122,6 +127,15 @@ struct brw_compiler {
    int spilling_rate;
 
    struct nir_shader *clc_shader;
+
+   /**
+    * A list of storage formats to lower from the matching return HW format.
+    *
+    * This list is used to build the lowering of read without format in
+    * brw_nir_lower_storage_image.c
+    */
+   uint32_t num_lowered_storage_formats;
+   uint32_t *lowered_storage_formats;
 };
 
 #define brw_shader_debug_log(compiler, data, fmt, ... ) do {    \

src/intel/compiler/brw_nir.h

@@ -196,14 +196,14 @@ void brw_nir_lower_fs_outputs(nir_shader *nir);
 bool brw_nir_lower_cmat(nir_shader *nir, unsigned subgroup_size);
 
 struct brw_nir_lower_storage_image_opts {
-   const struct intel_device_info *devinfo;
-
    bool lower_loads;
    bool lower_stores;
    bool lower_stores_64bit;
+   bool lower_loads_without_formats;
 };
 
 bool brw_nir_lower_storage_image(nir_shader *nir,
+                                 const struct brw_compiler *compiler,
                                  const struct brw_nir_lower_storage_image_opts *opts);
 
 bool brw_nir_lower_texel_address(nir_shader *shader,

src/intel/compiler/brw_nir_lower_storage_image.c

@@ -27,6 +27,11 @@
 #include "compiler/nir/nir_builder.h"
 #include "compiler/nir/nir_format_convert.h"
 
+struct brw_nir_lower_storage_image_state {
+   const struct brw_compiler *compiler;
+   struct brw_nir_lower_storage_image_opts opts;
+};
+
 struct format_info {
    const struct isl_format_layout *fmtl;
    unsigned chans;
@@ -50,6 +55,16 @@ get_format_info(enum isl_format fmt)
    };
 }
 
+static bool
+skip_storage_format(const struct intel_device_info *devinfo,
+                    enum isl_format format)
+{
+   if (!isl_is_storage_image_format(devinfo, format))
+      return true;
+
+   return format == isl_lower_storage_image_format(devinfo, format);
+}
+
 static nir_def *
 convert_color_for_load(nir_builder *b, const struct intel_device_info *devinfo,
                        nir_def *color,
@@ -147,6 +162,80 @@ expand_vec:
    return nir_vec(b, comps, dest_components);
 }
 
+static nir_def *
+convert_color_for_load_format(nir_builder *b,
+                              const struct brw_compiler *compiler,
+                              nir_def *color,
+                              nir_def *surface_format)
+{
+   nir_def *conversions[20] = {};
+   assert((compiler->num_lowered_storage_formats + 1) < ARRAY_SIZE(conversions));
+
+   for (unsigned i = 0; i < compiler->num_lowered_storage_formats; i++) {
+      enum isl_format format =
+         compiler->lowered_storage_formats[i];
+      enum isl_format lowered_format =
+         isl_lower_storage_image_format(compiler->devinfo, format);
+      unsigned lowered_components =
+         isl_format_get_num_channels(lowered_format);
+
+      nir_push_if(b, nir_ieq_imm(b, surface_format, format));
+      {
+         conversions[i] = convert_color_for_load(
+            b, compiler->devinfo,
+            nir_channels(b, color, nir_component_mask(lowered_components)),
+            format, lowered_format, color->num_components);
+      }
+      nir_push_else(b, NULL);
+   }
+
+   /* When the HW does the conversion automatically */
+   conversions[compiler->num_lowered_storage_formats] = nir_mov(b, color);
+
+   for (unsigned f = 0; f < compiler->num_lowered_storage_formats; f++) {
+      nir_pop_if(b, NULL);
+
+      conversions[compiler->num_lowered_storage_formats - f - 1] =
+         nir_if_phi(b, conversions[compiler->num_lowered_storage_formats - f - 1],
+                       conversions[compiler->num_lowered_storage_formats - f]);
+   }
+
+   return conversions[0];
+}
+
+static bool
+lower_image_load_instr_without_format(nir_builder *b,
+                                      const struct brw_nir_lower_storage_image_state *state,
+                                      nir_intrinsic_instr *intrin)
+{
+   /* This lowering relies on Gfx9+ HW behavior for typed reads (RAW values) */
+   assert(state->compiler->devinfo->ver >= 9);
+
+   /* Use an undef to hold the uses of the load while we do the color
+    * conversion.
+    */
+   nir_def *placeholder = nir_undef(b, 4, 32);
+   nir_def_rewrite_uses(&intrin->def, placeholder);
+
+   b->cursor = nir_after_instr(&intrin->instr);
+
+   nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+   nir_variable *var = nir_deref_instr_get_variable(deref);
+
+   assert(var->data.image.format == PIPE_FORMAT_NONE);
+
+   nir_def *image_fmt = nir_image_deref_load_param_intel(
+      b, 1, 32, &deref->def, .base = ISL_SURF_PARAM_FORMAT);
+
+   nir_def *color = convert_color_for_load_format(
+      b, state->compiler, &intrin->def, image_fmt);
+
+   nir_def_rewrite_uses(placeholder, color);
+   nir_instr_remove(placeholder->parent_instr);
+
+   return true;
+}
+
 static bool
 lower_image_load_instr(nir_builder *b,
                        const struct intel_device_info *devinfo,
@@ -156,8 +245,7 @@ lower_image_load_instr(nir_builder *b,
    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
    nir_variable *var = nir_deref_instr_get_variable(deref);
 
-   if (var->data.image.format == PIPE_FORMAT_NONE)
+   assert(var->data.image.format != PIPE_FORMAT_NONE);
-      return false;
 
    const enum isl_format image_fmt =
       isl_format_for_pipe_format(var->data.image.format);
@@ -285,9 +373,9 @@ convert_color_for_store(nir_builder *b, const struct intel_device_info *devinfo,
 static bool
 lower_image_store_instr(nir_builder *b,
                         const struct brw_nir_lower_storage_image_opts *opts,
+                        const struct intel_device_info *devinfo,
                         nir_intrinsic_instr *intrin)
 {
-   const struct intel_device_info *devinfo = opts->devinfo;
    nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
    nir_variable *var = nir_deref_instr_get_variable(deref);
 
@@ -334,22 +422,46 @@ brw_nir_lower_storage_image_instr(nir_builder *b,
 {
    if (instr->type != nir_instr_type_intrinsic)
       return false;
-   const struct brw_nir_lower_storage_image_opts *opts = cb_data;
+
+   const struct brw_nir_lower_storage_image_state *state = cb_data;
 
    nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
    switch (intrin->intrinsic) {
-   case nir_intrinsic_image_deref_load:
+   case nir_intrinsic_image_deref_load: {
-      if (opts->lower_loads)
+      nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
-         return lower_image_load_instr(b, opts->devinfo, intrin, false);
+      nir_variable *var = nir_deref_instr_get_variable(deref);
-      return false;
 
-   case nir_intrinsic_image_deref_sparse_load:
+      if (var->data.image.format == PIPE_FORMAT_NONE) {
-      if (opts->lower_loads)
+         if (state->opts.lower_loads_without_formats)
-         return lower_image_load_instr(b, opts->devinfo, intrin, true);
+            return lower_image_load_instr_without_format(b, state, intrin);
+      } else {
+         if (state->opts.lower_loads) {
+            return lower_image_load_instr(b, state->compiler->devinfo,
+                                          intrin, false);
+         }
+      }
       return false;
+      }
+
+   case nir_intrinsic_image_deref_sparse_load: {
+      nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+      nir_variable *var = nir_deref_instr_get_variable(deref);
+
+      if (var->data.image.format == PIPE_FORMAT_NONE) {
+         if (state->opts.lower_loads_without_formats)
+            return lower_image_load_instr_without_format(b, state, intrin);
+      } else {
+         if (state->opts.lower_loads) {
+            return lower_image_load_instr(b, state->compiler->devinfo,
+                                          intrin, true);
+         }
+      }
+      return false;
+   }
 
    case nir_intrinsic_image_deref_store:
-      return lower_image_store_instr(b, opts, intrin);
+      return lower_image_store_instr(
+         b, &state->opts, state->compiler->devinfo, intrin);
 
    default:
       /* Nothing to do */
@@ -359,6 +471,7 @@ brw_nir_lower_storage_image_instr(nir_builder *b,
 
 bool
 brw_nir_lower_storage_image(nir_shader *shader,
+                            const struct brw_compiler *compiler,
                             const struct brw_nir_lower_storage_image_opts *opts)
 {
    bool progress = false;
@@ -370,10 +483,14 @@ brw_nir_lower_storage_image(nir_shader *shader,
 
    progress |= nir_lower_image(shader, &image_options);
 
+   const struct brw_nir_lower_storage_image_state storage_options = {
+      .compiler = compiler,
+      .opts = *opts,
+   };
    progress |= nir_shader_instructions_pass(shader,
                                             brw_nir_lower_storage_image_instr,
                                             nir_metadata_none,
-                                            (void *)opts);
+                                            (void *)&storage_options);
 
    return progress;
 }

src/intel/compiler/meson.build

@@ -161,7 +161,7 @@ libintel_compiler_brw = static_library(
 )
 
 idep_intel_compiler_brw = declare_dependency(
-  link_with : [libintel_compiler_brw],
+  link_with : [libintel_compiler_brw, libisl],
   dependencies : [
     idep_nir,
     idep_mesautil,

src/intel/genxml/meson.build

@@ -68,6 +68,7 @@ genX_bits_included_symbols = [
   'RENDER_SURFACE_STATE::Width',
   'RENDER_SURFACE_STATE::Height',
   'RENDER_SURFACE_STATE::Depth',
+  'RENDER_SURFACE_STATE::Surface Format',
   'RENDER_SURFACE_STATE::Surface Type',
   'RENDER_SURFACE_STATE::Render Target View Extent',
   'RENDER_SURFACE_STATE::Tile Mode',

src/intel/isl/isl.h

@@ -1990,6 +1990,7 @@ enum isl_surf_param {
    ISL_SURF_PARAM_TILE_MODE,
    ISL_SURF_PARAM_PITCH,
    ISL_SURF_PARAM_QPITCH,
+   ISL_SURF_PARAM_FORMAT,
 };
 
 /*

src/intel/vulkan/anv_pipeline.c

@@ -1027,12 +1027,11 @@ anv_pipeline_lower_nir(struct anv_pipeline *pipeline,
                accept_64bit_atomic_cb, NULL);
    }
 
-   NIR_PASS(_, nir, brw_nir_lower_storage_image,
+   NIR_PASS(_, nir, brw_nir_lower_storage_image, compiler,
             &(struct brw_nir_lower_storage_image_opts) {
                /* Anv only supports Gfx9+ which has better defined typed
                 * read behavior.
                 */
-               .devinfo = compiler->devinfo,
                .lower_loads = true,
                .lower_stores_64bit = true,
             });