diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 3e76366c2e4..40a628f23b6 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -2555,8 +2555,9 @@ radv_link_shaders(struct radv_pipeline *pipeline, if (nir_lower_io_to_scalar_early(ordered_shaders[i], mask)) { /* Optimize the new vector code and then remove dead vars */ nir_copy_prop(ordered_shaders[i]); - nir_opt_shrink_vectors(ordered_shaders[i], - !pipeline->device->instance->disable_shrink_image_store); + nir_opt_shrink_stores(ordered_shaders[i], + !pipeline->device->instance->disable_shrink_image_store); + nir_opt_shrink_vectors(ordered_shaders[i]); if (ordered_shaders[i]->info.stage != last) { /* Optimize swizzled movs of load_const for @@ -3895,7 +3896,8 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout } lower_to_scalar |= - nir_opt_shrink_vectors(nir[i], !device->instance->disable_shrink_image_store); + nir_opt_shrink_stores(nir[i], !device->instance->disable_shrink_image_store); + nir_opt_shrink_vectors(nir[i]); if (lower_to_scalar) nir_lower_alu_to_scalar(nir[i], NULL, NULL); diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c index efd478a9926..6fe9a82f2fb 100644 --- a/src/amd/vulkan/radv_shader.c +++ b/src/amd/vulkan/radv_shader.c @@ -192,8 +192,9 @@ radv_optimize_nir(const struct radv_device *device, struct nir_shader *shader, NIR_PASS(progress, shader, nir_opt_algebraic); NIR_PASS(progress, shader, nir_opt_undef); - NIR_PASS(progress, shader, nir_opt_shrink_vectors, + NIR_PASS(progress, shader, nir_opt_shrink_stores, !device->instance->disable_shrink_image_store); + NIR_PASS(progress, shader, nir_opt_shrink_vectors); if (shader->options->max_unroll_iterations) { NIR_PASS(progress, shader, nir_opt_loop_unroll); } diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 47b21da231b..839f76ab4d7 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -5283,7 +5283,7 @@ bool nir_opt_phi_precision(nir_shader *shader); bool nir_opt_shrink_stores(nir_shader *shader, bool shrink_image_store); -bool nir_opt_shrink_vectors(nir_shader *shader, bool shrink_image_store); +bool nir_opt_shrink_vectors(nir_shader *shader); bool nir_opt_trivial_continues(nir_shader *shader); diff --git a/src/compiler/nir/nir_opt_shrink_vectors.c b/src/compiler/nir/nir_opt_shrink_vectors.c index 614f36c3541..0df7bcb5c9f 100644 --- a/src/compiler/nir/nir_opt_shrink_vectors.c +++ b/src/compiler/nir/nir_opt_shrink_vectors.c @@ -167,31 +167,7 @@ opt_shrink_vectors_alu(nir_builder *b, nir_alu_instr *instr) } static bool -opt_shrink_vectors_image_store(nir_builder *b, nir_intrinsic_instr *instr) -{ - enum pipe_format format; - if (instr->intrinsic == nir_intrinsic_image_deref_store) { - nir_deref_instr *deref = nir_src_as_deref(instr->src[0]); - format = nir_deref_instr_get_variable(deref)->data.image.format; - } else { - format = nir_intrinsic_format(instr); - } - if (format == PIPE_FORMAT_NONE) - return false; - - unsigned components = util_format_get_nr_components(format); - if (components >= instr->num_components) - return false; - - nir_ssa_def *data = nir_channels(b, instr->src[3].ssa, BITSET_MASK(components)); - nir_instr_rewrite_src(&instr->instr, &instr->src[3], nir_src_for_ssa(data)); - instr->num_components = components; - - return true; -} - -static bool -opt_shrink_vectors_intrinsic(nir_builder *b, nir_intrinsic_instr *instr, bool shrink_image_store) +opt_shrink_vectors_intrinsic(nir_builder *b, nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_uniform: @@ -208,17 +184,7 @@ opt_shrink_vectors_intrinsic(nir_builder *b, nir_intrinsic_instr *instr, bool sh case nir_intrinsic_load_global_constant: case nir_intrinsic_load_kernel_input: case nir_intrinsic_load_scratch: - case nir_intrinsic_store_output: - case nir_intrinsic_store_per_vertex_output: - case nir_intrinsic_store_ssbo: - case nir_intrinsic_store_shared: - case nir_intrinsic_store_global: - case nir_intrinsic_store_scratch: break; - case nir_intrinsic_bindless_image_store: - case nir_intrinsic_image_deref_store: - case nir_intrinsic_image_store: - return shrink_image_store && opt_shrink_vectors_image_store(b, instr); default: return false; } @@ -226,29 +192,10 @@ opt_shrink_vectors_intrinsic(nir_builder *b, nir_intrinsic_instr *instr, bool sh /* Must be a vectorized intrinsic that we can resize. */ assert(instr->num_components != 0); - if (nir_intrinsic_infos[instr->intrinsic].has_dest) { - /* loads: Trim the dest to the used channels */ - - if (shrink_dest_to_read_mask(&instr->dest.ssa)) { - instr->num_components = instr->dest.ssa.num_components; - return true; - } - } else { - /* Stores: trim the num_components stored according to the write - * mask. - */ - unsigned write_mask = nir_intrinsic_write_mask(instr); - unsigned last_bit = util_last_bit(write_mask); - if (last_bit < instr->num_components && instr->src[0].is_ssa) { - nir_ssa_def *def = nir_channels(b, instr->src[0].ssa, - BITSET_MASK(last_bit)); - nir_instr_rewrite_src(&instr->instr, - &instr->src[0], - nir_src_for_ssa(def)); - instr->num_components = last_bit; - - return true; - } + /* Trim the dest to the used channels */ + if (shrink_dest_to_read_mask(&instr->dest.ssa)) { + instr->num_components = instr->dest.ssa.num_components; + return true; } return false; @@ -267,7 +214,7 @@ opt_shrink_vectors_ssa_undef(nir_ssa_undef_instr *instr) } static bool -opt_shrink_vectors_instr(nir_builder *b, nir_instr *instr, bool shrink_image_store) +opt_shrink_vectors_instr(nir_builder *b, nir_instr *instr) { b->cursor = nir_before_instr(instr); @@ -276,7 +223,7 @@ opt_shrink_vectors_instr(nir_builder *b, nir_instr *instr, bool shrink_image_sto return opt_shrink_vectors_alu(b, nir_instr_as_alu(instr)); case nir_instr_type_intrinsic: - return opt_shrink_vectors_intrinsic(b, nir_instr_as_intrinsic(instr), shrink_image_store); + return opt_shrink_vectors_intrinsic(b, nir_instr_as_intrinsic(instr)); case nir_instr_type_load_const: return opt_shrink_vectors_load_const(nir_instr_as_load_const(instr)); @@ -292,7 +239,7 @@ opt_shrink_vectors_instr(nir_builder *b, nir_instr *instr, bool shrink_image_sto } bool -nir_opt_shrink_vectors(nir_shader *shader, bool shrink_image_store) +nir_opt_shrink_vectors(nir_shader *shader) { bool progress = false; @@ -305,7 +252,7 @@ nir_opt_shrink_vectors(nir_shader *shader, bool shrink_image_store) nir_foreach_block_reverse(block, function->impl) { nir_foreach_instr_reverse(instr, block) { - progress |= opt_shrink_vectors_instr(&b, instr, shrink_image_store); + progress |= opt_shrink_vectors_instr(&b, instr); } } diff --git a/src/gallium/auxiliary/nir/nir_to_tgsi.c b/src/gallium/auxiliary/nir/nir_to_tgsi.c index 2cc6ce43e83..2d00e5cc5db 100644 --- a/src/gallium/auxiliary/nir/nir_to_tgsi.c +++ b/src/gallium/auxiliary/nir/nir_to_tgsi.c @@ -3084,7 +3084,8 @@ ntt_optimize_nir(struct nir_shader *s, struct pipe_screen *screen) .robust_modes = 0, }; NIR_PASS(progress, s, nir_opt_load_store_vectorize, &vectorize_opts); - NIR_PASS(progress, s, nir_opt_shrink_vectors, true); + NIR_PASS(progress, s, nir_opt_shrink_stores, true); + NIR_PASS(progress, s, nir_opt_shrink_vectors); NIR_PASS(progress, s, nir_opt_trivial_continues); NIR_PASS(progress, s, nir_opt_vectorize, ntt_should_vectorize_instr, NULL); NIR_PASS(progress, s, nir_opt_undef); diff --git a/src/gallium/drivers/etnaviv/etnaviv_compiler_nir.c b/src/gallium/drivers/etnaviv/etnaviv_compiler_nir.c index 55085e0d523..4ce6d3c125c 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_compiler_nir.c +++ b/src/gallium/drivers/etnaviv/etnaviv_compiler_nir.c @@ -146,7 +146,8 @@ etna_optimize_loop(nir_shader *s) NIR_PASS_V(s, nir_lower_vars_to_ssa); progress |= OPT(s, nir_opt_copy_prop_vars); - progress |= OPT(s, nir_opt_shrink_vectors, true); + progress |= OPT(s, nir_opt_shrink_stores, true); + progress |= OPT(s, nir_opt_shrink_vectors); progress |= OPT(s, nir_copy_prop); progress |= OPT(s, nir_opt_dce); progress |= OPT(s, nir_opt_cse); diff --git a/src/gallium/drivers/i915/i915_screen.c b/src/gallium/drivers/i915/i915_screen.c index d82a3565336..994496be4ca 100644 --- a/src/gallium/drivers/i915/i915_screen.c +++ b/src/gallium/drivers/i915/i915_screen.c @@ -202,7 +202,8 @@ i915_optimize_nir(struct nir_shader *s) true, true); NIR_PASS(progress, s, nir_opt_algebraic); NIR_PASS(progress, s, nir_opt_constant_folding); - NIR_PASS(progress, s, nir_opt_shrink_vectors, true); + NIR_PASS(progress, s, nir_opt_shrink_stores, true); + NIR_PASS(progress, s, nir_opt_shrink_vectors); NIR_PASS(progress, s, nir_opt_trivial_continues); NIR_PASS(progress, s, nir_opt_undef); NIR_PASS(progress, s, nir_opt_loop_unroll); diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c index 800f99e69e3..9a8d4db133e 100644 --- a/src/intel/compiler/brw_nir.c +++ b/src/intel/compiler/brw_nir.c @@ -553,7 +553,8 @@ brw_nir_optimize(nir_shader *nir, const struct brw_compiler *compiler, if (is_scalar) { OPT(nir_lower_alu_to_scalar, NULL, NULL); } else { - OPT(nir_opt_shrink_vectors, true); + OPT(nir_opt_shrink_stores, true); + OPT(nir_opt_shrink_vectors); } OPT(nir_copy_prop);