From 61a96be4947209ebc65a8bd3391f140e61e43f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Ol=C5=A1=C3=A1k?= Date: Fri, 6 Feb 2026 14:01:06 -0500 Subject: [PATCH] nir/lower_non_uniform_access: add an option not to lower tex & image queries AMD can do non-uniform queries. The RADV change will be in a separate commit. NFC for drivers. Reviewed-by: Rhys Perry Part-of: --- src/amd/vulkan/radv_pipeline.c | 2 +- src/compiler/nir/nir.h | 4 +- .../nir/nir_lower_non_uniform_access.c | 61 ++++++++++++++----- src/gallium/drivers/radeonsi/si_shader.c | 2 +- src/gallium/frontends/lavapipe/lvp_pipeline.c | 3 +- src/intel/vulkan/anv_shader_compile.c | 2 + src/intel/vulkan_hasvk/anv_pipeline.c | 3 +- src/nouveau/vulkan/nvk_shader.c | 4 +- src/panfrost/vulkan/panvk_vX_shader.c | 2 + 9 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index c3a91024d84..aef549d05d4 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -309,7 +309,7 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_stat enum nir_lower_non_uniform_access_type lower_non_uniform_access_types = nir_lower_non_uniform_ubo_access | nir_lower_non_uniform_ssbo_access | nir_lower_non_uniform_texture_access | - nir_lower_non_uniform_image_access; + nir_lower_non_uniform_image_access | nir_lower_non_uniform_texture_query | nir_lower_non_uniform_image_query; /* In practice, most shaders do not have non-uniform-qualified * accesses (see diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 9991b0d787f..9aba4055fbb 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -6141,7 +6141,9 @@ enum nir_lower_non_uniform_access_type { nir_lower_non_uniform_image_access = (1 << 3), nir_lower_non_uniform_get_ssbo_size = (1 << 4), nir_lower_non_uniform_texture_offset_access = (1 << 5), - nir_lower_non_uniform_access_type_count = 6, + nir_lower_non_uniform_texture_query = (1 << 6), + nir_lower_non_uniform_image_query = (1 << 7), + nir_lower_non_uniform_access_type_count = 8, }; typedef bool (*nir_lower_non_uniform_src_access_callback)(const nir_tex_instr *, unsigned, void *); diff --git a/src/compiler/nir/nir_lower_non_uniform_access.c b/src/compiler/nir/nir_lower_non_uniform_access.c index 19e48d6397d..e7332c83faa 100644 --- a/src/compiler/nir/nir_lower_non_uniform_access.c +++ b/src/compiler/nir/nir_lower_non_uniform_access.c @@ -207,10 +207,33 @@ static bool lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex, const nir_lower_non_uniform_access_options *opts) { - if (!(tex->texture_non_uniform && (opts->types & nir_lower_non_uniform_texture_access)) && - !(tex->sampler_non_uniform && (opts->types & nir_lower_non_uniform_texture_access)) && - !(tex->offset_non_uniform && (opts->types & nir_lower_non_uniform_texture_offset_access))) - return false; + enum nir_lower_non_uniform_access_type base_access_type; + + switch (tex->op) { + case nir_texop_txs: + case nir_texop_query_levels: + case nir_texop_texture_samples: + case nir_texop_descriptor_amd: + if (!(tex->texture_non_uniform && (opts->types & nir_lower_non_uniform_texture_query))) + return false; + base_access_type = nir_lower_non_uniform_texture_query; + break; + + case nir_texop_lod_bias: + case nir_texop_sampler_descriptor_amd: + if (!(tex->sampler_non_uniform && (opts->types & nir_lower_non_uniform_texture_query))) + return false; + base_access_type = nir_lower_non_uniform_texture_query; + break; + + default: + if (!(tex->texture_non_uniform && (opts->types & nir_lower_non_uniform_texture_access)) && + !(tex->sampler_non_uniform && (opts->types & nir_lower_non_uniform_texture_access)) && + !(tex->offset_non_uniform && (opts->types & nir_lower_non_uniform_texture_offset_access))) + return false; + base_access_type = nir_lower_non_uniform_texture_access; + break; + } /* We can have at most one texture and one sampler handle */ unsigned num_handles = 0; @@ -224,7 +247,7 @@ lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex, case nir_tex_src_texture_2_deref: if (!tex->texture_non_uniform) continue; - if (!(opts->types & nir_lower_non_uniform_texture_access)) + if (!(opts->types & base_access_type)) continue; if (opts->tex_src_callback && !opts->tex_src_callback(tex, i, opts->callback_data)) continue; @@ -236,7 +259,7 @@ lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex, case nir_tex_src_sampler_2_deref: if (!tex->sampler_non_uniform) continue; - if (!(opts->types & nir_lower_non_uniform_texture_access)) + if (!(opts->types & base_access_type)) continue; if (opts->tex_src_callback && !opts->tex_src_callback(tex, i, opts->callback_data)) continue; @@ -274,7 +297,7 @@ lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex, tex->offset_non_uniform = false; add_non_uniform_instr(state, handles, srcs, num_handles, true, - nir_lower_non_uniform_texture_access); + base_access_type); return true; } @@ -334,6 +357,7 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_instr_type_tex: { nir_tex_instr *tex = nir_instr_as_tex(instr); if ((options->types & (nir_lower_non_uniform_texture_access | + nir_lower_non_uniform_texture_query | nir_lower_non_uniform_texture_offset_access)) && lower_non_uniform_tex_access(&state, tex, options)) progress = true; @@ -384,9 +408,6 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_image_store: case nir_intrinsic_image_atomic: case nir_intrinsic_image_atomic_swap: - case nir_intrinsic_image_levels: - case nir_intrinsic_image_size: - case nir_intrinsic_image_samples: case nir_intrinsic_image_samples_identical: case nir_intrinsic_image_fragment_mask_load_amd: case nir_intrinsic_bindless_image_load: @@ -394,9 +415,6 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_atomic: case nir_intrinsic_bindless_image_atomic_swap: - case nir_intrinsic_bindless_image_levels: - case nir_intrinsic_bindless_image_size: - case nir_intrinsic_bindless_image_samples: case nir_intrinsic_bindless_image_samples_identical: case nir_intrinsic_bindless_image_fragment_mask_load_amd: case nir_intrinsic_image_deref_load: @@ -404,9 +422,6 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic: case nir_intrinsic_image_deref_atomic_swap: - case nir_intrinsic_image_deref_levels: - case nir_intrinsic_image_deref_size: - case nir_intrinsic_image_deref_samples: case nir_intrinsic_image_deref_samples_identical: case nir_intrinsic_image_deref_fragment_mask_load_amd: if ((options->types & nir_lower_non_uniform_image_access) && @@ -414,6 +429,20 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, progress = true; break; + case nir_intrinsic_image_levels: + case nir_intrinsic_image_size: + case nir_intrinsic_image_samples: + case nir_intrinsic_bindless_image_levels: + case nir_intrinsic_bindless_image_size: + case nir_intrinsic_bindless_image_samples: + case nir_intrinsic_image_deref_levels: + case nir_intrinsic_image_deref_size: + case nir_intrinsic_image_deref_samples: + if ((options->types & nir_lower_non_uniform_image_query) && + lower_non_uniform_access_intrin(&state, intrin, 0, nir_lower_non_uniform_image_query)) + progress = true; + break; + case nir_intrinsic_load_tile_pan: case nir_intrinsic_load_tile_res_pan: /* render target can be nonuniform, but not conversion descriptor */ diff --git a/src/gallium/drivers/radeonsi/si_shader.c b/src/gallium/drivers/radeonsi/si_shader.c index 6e82d93e08f..db8be0f5f82 100644 --- a/src/gallium/drivers/radeonsi/si_shader.c +++ b/src/gallium/drivers/radeonsi/si_shader.c @@ -954,7 +954,7 @@ static void si_postprocess_nir(struct si_nir_shader_ctx *ctx) /* LLVM does not work well with this, so is handled in llvm backend waterfall. */ if (nir->info.use_aco_amd && ctx->temp_info.has_non_uniform_tex_access) { nir_lower_non_uniform_access_options options = { - .types = nir_lower_non_uniform_texture_access, + .types = nir_lower_non_uniform_texture_access | nir_lower_non_uniform_texture_query, }; NIR_PASS(progress, nir, nir_lower_non_uniform_access, &options); } diff --git a/src/gallium/frontends/lavapipe/lvp_pipeline.c b/src/gallium/frontends/lavapipe/lvp_pipeline.c index 5c272f01a56..3baf0734f83 100644 --- a/src/gallium/frontends/lavapipe/lvp_pipeline.c +++ b/src/gallium/frontends/lavapipe/lvp_pipeline.c @@ -466,7 +466,8 @@ lvp_shader_lower(struct lvp_device *pdevice, nir_shader *nir, struct lvp_pipelin NIR_PASS(_, nir, nir_vk_lower_ycbcr_tex, lvp_ycbcr_conversion_lookup, layout); nir_lower_non_uniform_access_options options = { - .types = nir_lower_non_uniform_ubo_access | nir_lower_non_uniform_texture_access | nir_lower_non_uniform_image_access, + .types = nir_lower_non_uniform_ubo_access | nir_lower_non_uniform_texture_access | nir_lower_non_uniform_image_access | + nir_lower_non_uniform_texture_query | nir_lower_non_uniform_image_query, }; NIR_PASS(_, nir, nir_lower_non_uniform_access, &options); diff --git a/src/intel/vulkan/anv_shader_compile.c b/src/intel/vulkan/anv_shader_compile.c index 852bc01fea4..8fbe0aff572 100644 --- a/src/intel/vulkan/anv_shader_compile.c +++ b/src/intel/vulkan/anv_shader_compile.c @@ -1502,7 +1502,9 @@ anv_shader_lower_nir(struct anv_device *device, enum nir_lower_non_uniform_access_type lower_non_uniform_access_types = nir_lower_non_uniform_texture_access | + nir_lower_non_uniform_texture_query | nir_lower_non_uniform_image_access | + nir_lower_non_uniform_image_query | nir_lower_non_uniform_get_ssbo_size | (lower_non_uniform_texture_offsets ? nir_lower_non_uniform_texture_offset_access : 0); diff --git a/src/intel/vulkan_hasvk/anv_pipeline.c b/src/intel/vulkan_hasvk/anv_pipeline.c index bdfdcb72dcd..5c921022a41 100644 --- a/src/intel/vulkan_hasvk/anv_pipeline.c +++ b/src/intel/vulkan_hasvk/anv_pipeline.c @@ -524,7 +524,8 @@ anv_pipeline_lower_nir(struct anv_pipeline *pipeline, NIR_PASS(_, nir, anv_nir_lower_ubo_loads); enum nir_lower_non_uniform_access_type lower_non_uniform_access_types = - nir_lower_non_uniform_texture_access | nir_lower_non_uniform_image_access; + nir_lower_non_uniform_texture_access | nir_lower_non_uniform_image_access | + nir_lower_non_uniform_texture_query | nir_lower_non_uniform_image_query; /* In practice, most shaders do not have non-uniform-qualified * accesses (see diff --git a/src/nouveau/vulkan/nvk_shader.c b/src/nouveau/vulkan/nvk_shader.c index 917eab011de..03e93f4ab24 100644 --- a/src/nouveau/vulkan/nvk_shader.c +++ b/src/nouveau/vulkan/nvk_shader.c @@ -413,7 +413,9 @@ nvk_lower_nir(struct nvk_device *dev, nir_shader *nir, */ struct nir_lower_non_uniform_access_options opts = { .types = nir_lower_non_uniform_texture_access | - nir_lower_non_uniform_image_access, + nir_lower_non_uniform_texture_query | + nir_lower_non_uniform_image_access | + nir_lower_non_uniform_image_query, .callback = NULL, }; /* In practice, most shaders do not have non-uniform-qualified accesses diff --git a/src/panfrost/vulkan/panvk_vX_shader.c b/src/panfrost/vulkan/panvk_vX_shader.c index 5d431689be9..c696a103925 100644 --- a/src/panfrost/vulkan/panvk_vX_shader.c +++ b/src/panfrost/vulkan/panvk_vX_shader.c @@ -805,7 +805,9 @@ panvk_lower_nir(struct panvk_device *dev, nir_shader *nir, nir_lower_non_uniform_ubo_access | nir_lower_non_uniform_ssbo_access | nir_lower_non_uniform_texture_access | + nir_lower_non_uniform_texture_query | nir_lower_non_uniform_image_access | + nir_lower_non_uniform_image_query | nir_lower_non_uniform_get_ssbo_size; #if PAN_ARCH < 9 lower_non_uniform_access_types |=