From 52f73db5b78a4c105eb78c80487e6651d492ceb5 Mon Sep 17 00:00:00 2001 From: Lionel Landwerlin Date: Mon, 10 Apr 2023 13:28:48 +0300 Subject: [PATCH] brw: implement read without format lowering Load the format enum and then just go through a series of : if format == R16G16B16A16_UNORM color = lower_r32g32_uint_tor_r16g16b16a16_unorm(color) else if format == R16G16B16A16_SNORM ... For Gfx12.5, there is no in-shader conversion. For Gfx12/11, the in-shader conversion covers the following formats : - ISL_FORMAT_R10G10B10A2_UNORM - ISL_FORMAT_R10G10B10A2_UINT - ISL_FORMAT_R11G11B10_FLOAT For Gfx9, the following formats : - ISL_FORMAT_R16G16B16A16_UNORM - ISL_FORMAT_R16G16B16A16_SNORM - ISL_FORMAT_R10G10B10A2_UNORM - ISL_FORMAT_R10G10B10A2_UINT - ISL_FORMAT_R8G8B8A8_UNORM - ISL_FORMAT_R8G8B8A8_SNORM - ISL_FORMAT_R16G16_UNORM - ISL_FORMAT_R16G16_SNORM - ISL_FORMAT_R11G11B10_FLOAT - ISL_FORMAT_R8G8_UNORM - ISL_FORMAT_R8G8_SNORM - ISL_FORMAT_R16_UNORM - ISL_FORMAT_R16_SNORM - ISL_FORMAT_R8_UNORM - ISL_FORMAT_R8_SNORM Signed-off-by: Lionel Landwerlin Reviewed-by: Caio Oliveira Part-of: --- src/gallium/drivers/iris/iris_program.c | 4 +- src/intel/compiler/brw_compiler.c | 26 ++++ src/intel/compiler/brw_compiler.h | 14 ++ src/intel/compiler/brw_nir.h | 4 +- .../compiler/brw_nir_lower_storage_image.c | 143 ++++++++++++++++-- src/intel/compiler/meson.build | 2 +- src/intel/genxml/meson.build | 1 + src/intel/isl/isl.h | 1 + src/intel/vulkan/anv_pipeline.c | 3 +- 9 files changed, 178 insertions(+), 20 deletions(-) diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c index a48fc3a685b..42dd3fc7179 100644 --- a/src/gallium/drivers/iris/iris_program.c +++ b/src/gallium/drivers/iris/iris_program.c @@ -3785,7 +3785,6 @@ static char * iris_finalize_nir(struct pipe_screen *_screen, struct nir_shader *nir) { struct iris_screen *screen = (struct iris_screen *)_screen; - const struct intel_device_info *devinfo = screen->devinfo; NIR_PASS_V(nir, iris_fix_edge_flags); @@ -3794,13 +3793,14 @@ iris_finalize_nir(struct pipe_screen *_screen, struct nir_shader *nir) brw_preprocess_nir(screen->brw, nir, &opts); NIR_PASS_V(nir, brw_nir_lower_storage_image, + screen->brw, &(struct brw_nir_lower_storage_image_opts) { - .devinfo = devinfo, .lower_loads = true, .lower_stores = true, }); } else { #ifdef INTEL_USE_ELK + const struct intel_device_info *devinfo = screen->devinfo; assert(screen->elk); struct elk_nir_compiler_opts opts = {}; diff --git a/src/intel/compiler/brw_compiler.c b/src/intel/compiler/brw_compiler.c index 4e3f185c805..7562bc0b493 100644 --- a/src/intel/compiler/brw_compiler.c +++ b/src/intel/compiler/brw_compiler.c @@ -27,6 +27,7 @@ #include "brw_private.h" #include "dev/intel_debug.h" #include "compiler/nir/nir.h" +#include "isl/isl.h" #include "util/u_debug.h" const struct nir_shader_compiler_options brw_scalar_nir_options = { @@ -194,6 +195,31 @@ brw_compiler_create(void *mem_ctx, const struct intel_device_info *devinfo) compiler->nir_options[i] = nir_options; } + /* Build a list of storage format compatible in component bit size & + * isl_base_type. We can apply the same lowering to those. + */ + compiler->num_lowered_storage_formats = 0; + for (enum isl_format fmt = 0; fmt < ISL_FORMAT_RAW; fmt++) { + if (!isl_is_storage_image_format(devinfo, fmt)) + continue; + + if (isl_lower_storage_image_format(devinfo, fmt) == fmt) + continue; + + compiler->lowered_storage_formats = + reralloc(compiler, compiler->lowered_storage_formats, + uint32_t, compiler->num_lowered_storage_formats + 1); + compiler->lowered_storage_formats[ + compiler->num_lowered_storage_formats++] = fmt; + } + assert((devinfo->verx10 >= 125 && + compiler->num_lowered_storage_formats == 0) || + (devinfo->verx10 >= 110 && devinfo->verx10 <= 120 && + compiler->num_lowered_storage_formats == 3) || + devinfo->verx10 == 90); + fprintf(stderr, "num_lowered_storage_formats=%i\n", + compiler->num_lowered_storage_formats); + return compiler; } diff --git a/src/intel/compiler/brw_compiler.h b/src/intel/compiler/brw_compiler.h index 8ff518b08d9..606211c2f4e 100644 --- a/src/intel/compiler/brw_compiler.h +++ b/src/intel/compiler/brw_compiler.h @@ -49,6 +49,11 @@ typedef struct nir_shader nir_shader; #define REG_CLASS_COUNT 20 +struct brw_storage_format { + uint32_t num_formats; + uint32_t isl_formats[3]; +}; + struct brw_compiler { const struct intel_device_info *devinfo; @@ -122,6 +127,15 @@ struct brw_compiler { int spilling_rate; struct nir_shader *clc_shader; + + /** + * A list of storage formats to lower from the matching return HW format. + * + * This list is used to build the lowering of read without format in + * brw_nir_lower_storage_image.c + */ + uint32_t num_lowered_storage_formats; + uint32_t *lowered_storage_formats; }; #define brw_shader_debug_log(compiler, data, fmt, ... ) do { \ diff --git a/src/intel/compiler/brw_nir.h b/src/intel/compiler/brw_nir.h index f9dc7dce621..878074f4b81 100644 --- a/src/intel/compiler/brw_nir.h +++ b/src/intel/compiler/brw_nir.h @@ -196,14 +196,14 @@ void brw_nir_lower_fs_outputs(nir_shader *nir); bool brw_nir_lower_cmat(nir_shader *nir, unsigned subgroup_size); struct brw_nir_lower_storage_image_opts { - const struct intel_device_info *devinfo; - bool lower_loads; bool lower_stores; bool lower_stores_64bit; + bool lower_loads_without_formats; }; bool brw_nir_lower_storage_image(nir_shader *nir, + const struct brw_compiler *compiler, const struct brw_nir_lower_storage_image_opts *opts); bool brw_nir_lower_texel_address(nir_shader *shader, diff --git a/src/intel/compiler/brw_nir_lower_storage_image.c b/src/intel/compiler/brw_nir_lower_storage_image.c index 6ae2c17b685..7ad248f767f 100644 --- a/src/intel/compiler/brw_nir_lower_storage_image.c +++ b/src/intel/compiler/brw_nir_lower_storage_image.c @@ -27,6 +27,11 @@ #include "compiler/nir/nir_builder.h" #include "compiler/nir/nir_format_convert.h" +struct brw_nir_lower_storage_image_state { + const struct brw_compiler *compiler; + struct brw_nir_lower_storage_image_opts opts; +}; + struct format_info { const struct isl_format_layout *fmtl; unsigned chans; @@ -50,6 +55,16 @@ get_format_info(enum isl_format fmt) }; } +static bool +skip_storage_format(const struct intel_device_info *devinfo, + enum isl_format format) +{ + if (!isl_is_storage_image_format(devinfo, format)) + return true; + + return format == isl_lower_storage_image_format(devinfo, format); +} + static nir_def * convert_color_for_load(nir_builder *b, const struct intel_device_info *devinfo, nir_def *color, @@ -147,6 +162,80 @@ expand_vec: return nir_vec(b, comps, dest_components); } +static nir_def * +convert_color_for_load_format(nir_builder *b, + const struct brw_compiler *compiler, + nir_def *color, + nir_def *surface_format) +{ + nir_def *conversions[20] = {}; + assert((compiler->num_lowered_storage_formats + 1) < ARRAY_SIZE(conversions)); + + for (unsigned i = 0; i < compiler->num_lowered_storage_formats; i++) { + enum isl_format format = + compiler->lowered_storage_formats[i]; + enum isl_format lowered_format = + isl_lower_storage_image_format(compiler->devinfo, format); + unsigned lowered_components = + isl_format_get_num_channels(lowered_format); + + nir_push_if(b, nir_ieq_imm(b, surface_format, format)); + { + conversions[i] = convert_color_for_load( + b, compiler->devinfo, + nir_channels(b, color, nir_component_mask(lowered_components)), + format, lowered_format, color->num_components); + } + nir_push_else(b, NULL); + } + + /* When the HW does the conversion automatically */ + conversions[compiler->num_lowered_storage_formats] = nir_mov(b, color); + + for (unsigned f = 0; f < compiler->num_lowered_storage_formats; f++) { + nir_pop_if(b, NULL); + + conversions[compiler->num_lowered_storage_formats - f - 1] = + nir_if_phi(b, conversions[compiler->num_lowered_storage_formats - f - 1], + conversions[compiler->num_lowered_storage_formats - f]); + } + + return conversions[0]; +} + +static bool +lower_image_load_instr_without_format(nir_builder *b, + const struct brw_nir_lower_storage_image_state *state, + nir_intrinsic_instr *intrin) +{ + /* This lowering relies on Gfx9+ HW behavior for typed reads (RAW values) */ + assert(state->compiler->devinfo->ver >= 9); + + /* Use an undef to hold the uses of the load while we do the color + * conversion. + */ + nir_def *placeholder = nir_undef(b, 4, 32); + nir_def_rewrite_uses(&intrin->def, placeholder); + + b->cursor = nir_after_instr(&intrin->instr); + + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + nir_variable *var = nir_deref_instr_get_variable(deref); + + assert(var->data.image.format == PIPE_FORMAT_NONE); + + nir_def *image_fmt = nir_image_deref_load_param_intel( + b, 1, 32, &deref->def, .base = ISL_SURF_PARAM_FORMAT); + + nir_def *color = convert_color_for_load_format( + b, state->compiler, &intrin->def, image_fmt); + + nir_def_rewrite_uses(placeholder, color); + nir_instr_remove(placeholder->parent_instr); + + return true; +} + static bool lower_image_load_instr(nir_builder *b, const struct intel_device_info *devinfo, @@ -156,8 +245,7 @@ lower_image_load_instr(nir_builder *b, nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); nir_variable *var = nir_deref_instr_get_variable(deref); - if (var->data.image.format == PIPE_FORMAT_NONE) - return false; + assert(var->data.image.format != PIPE_FORMAT_NONE); const enum isl_format image_fmt = isl_format_for_pipe_format(var->data.image.format); @@ -285,9 +373,9 @@ convert_color_for_store(nir_builder *b, const struct intel_device_info *devinfo, static bool lower_image_store_instr(nir_builder *b, const struct brw_nir_lower_storage_image_opts *opts, + const struct intel_device_info *devinfo, nir_intrinsic_instr *intrin) { - const struct intel_device_info *devinfo = opts->devinfo; nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); nir_variable *var = nir_deref_instr_get_variable(deref); @@ -334,22 +422,46 @@ brw_nir_lower_storage_image_instr(nir_builder *b, { if (instr->type != nir_instr_type_intrinsic) return false; - const struct brw_nir_lower_storage_image_opts *opts = cb_data; + + const struct brw_nir_lower_storage_image_state *state = cb_data; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { - case nir_intrinsic_image_deref_load: - if (opts->lower_loads) - return lower_image_load_instr(b, opts->devinfo, intrin, false); - return false; + case nir_intrinsic_image_deref_load: { + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + nir_variable *var = nir_deref_instr_get_variable(deref); - case nir_intrinsic_image_deref_sparse_load: - if (opts->lower_loads) - return lower_image_load_instr(b, opts->devinfo, intrin, true); + if (var->data.image.format == PIPE_FORMAT_NONE) { + if (state->opts.lower_loads_without_formats) + return lower_image_load_instr_without_format(b, state, intrin); + } else { + if (state->opts.lower_loads) { + return lower_image_load_instr(b, state->compiler->devinfo, + intrin, false); + } + } return false; + } + + case nir_intrinsic_image_deref_sparse_load: { + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + nir_variable *var = nir_deref_instr_get_variable(deref); + + if (var->data.image.format == PIPE_FORMAT_NONE) { + if (state->opts.lower_loads_without_formats) + return lower_image_load_instr_without_format(b, state, intrin); + } else { + if (state->opts.lower_loads) { + return lower_image_load_instr(b, state->compiler->devinfo, + intrin, true); + } + } + return false; + } case nir_intrinsic_image_deref_store: - return lower_image_store_instr(b, opts, intrin); + return lower_image_store_instr( + b, &state->opts, state->compiler->devinfo, intrin); default: /* Nothing to do */ @@ -359,6 +471,7 @@ brw_nir_lower_storage_image_instr(nir_builder *b, bool brw_nir_lower_storage_image(nir_shader *shader, + const struct brw_compiler *compiler, const struct brw_nir_lower_storage_image_opts *opts) { bool progress = false; @@ -370,10 +483,14 @@ brw_nir_lower_storage_image(nir_shader *shader, progress |= nir_lower_image(shader, &image_options); + const struct brw_nir_lower_storage_image_state storage_options = { + .compiler = compiler, + .opts = *opts, + }; progress |= nir_shader_instructions_pass(shader, brw_nir_lower_storage_image_instr, nir_metadata_none, - (void *)opts); + (void *)&storage_options); return progress; } diff --git a/src/intel/compiler/meson.build b/src/intel/compiler/meson.build index 4448a76f3f5..ca99c5b7b50 100644 --- a/src/intel/compiler/meson.build +++ b/src/intel/compiler/meson.build @@ -161,7 +161,7 @@ libintel_compiler_brw = static_library( ) idep_intel_compiler_brw = declare_dependency( - link_with : [libintel_compiler_brw], + link_with : [libintel_compiler_brw, libisl], dependencies : [ idep_nir, idep_mesautil, diff --git a/src/intel/genxml/meson.build b/src/intel/genxml/meson.build index 362f7531b68..379082119ec 100644 --- a/src/intel/genxml/meson.build +++ b/src/intel/genxml/meson.build @@ -68,6 +68,7 @@ genX_bits_included_symbols = [ 'RENDER_SURFACE_STATE::Width', 'RENDER_SURFACE_STATE::Height', 'RENDER_SURFACE_STATE::Depth', + 'RENDER_SURFACE_STATE::Surface Format', 'RENDER_SURFACE_STATE::Surface Type', 'RENDER_SURFACE_STATE::Render Target View Extent', 'RENDER_SURFACE_STATE::Tile Mode', diff --git a/src/intel/isl/isl.h b/src/intel/isl/isl.h index c6dae605d92..ca81c2bf195 100644 --- a/src/intel/isl/isl.h +++ b/src/intel/isl/isl.h @@ -1990,6 +1990,7 @@ enum isl_surf_param { ISL_SURF_PARAM_TILE_MODE, ISL_SURF_PARAM_PITCH, ISL_SURF_PARAM_QPITCH, + ISL_SURF_PARAM_FORMAT, }; /* diff --git a/src/intel/vulkan/anv_pipeline.c b/src/intel/vulkan/anv_pipeline.c index 0bbcc67e395..721d96043fd 100644 --- a/src/intel/vulkan/anv_pipeline.c +++ b/src/intel/vulkan/anv_pipeline.c @@ -1027,12 +1027,11 @@ anv_pipeline_lower_nir(struct anv_pipeline *pipeline, accept_64bit_atomic_cb, NULL); } - NIR_PASS(_, nir, brw_nir_lower_storage_image, + NIR_PASS(_, nir, brw_nir_lower_storage_image, compiler, &(struct brw_nir_lower_storage_image_opts) { /* Anv only supports Gfx9+ which has better defined typed * read behavior. */ - .devinfo = compiler->devinfo, .lower_loads = true, .lower_stores_64bit = true, });