diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index b5281b7b1c9..254c0553f0f 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -2435,6 +2435,9 @@ typedef struct nir_tex_instr { */ bool sampler_non_uniform; + /** True if the offset is not dynamically uniform */ + bool offset_non_uniform; + /** The texture index * * If this texture instruction has a nir_tex_src_texture_offset source, @@ -5647,14 +5650,20 @@ enum nir_lower_non_uniform_access_type { nir_lower_non_uniform_texture_access = (1 << 2), nir_lower_non_uniform_image_access = (1 << 3), nir_lower_non_uniform_get_ssbo_size = (1 << 4), - nir_lower_non_uniform_access_type_count = 5, + nir_lower_non_uniform_texture_offset_access = (1 << 5), + nir_lower_non_uniform_access_type_count = 6, }; +typedef bool (*nir_lower_non_uniform_src_access_callback)(const nir_tex_instr *, unsigned, void *); /* Given the nir_src used for the resource, return the channels which might be non-uniform. */ typedef nir_component_mask_t (*nir_lower_non_uniform_access_callback)(const nir_src *, void *); typedef struct nir_lower_non_uniform_access_options { enum nir_lower_non_uniform_access_type types; + /* Called on nir_tex_instr to ask whether non-uniform lowering is required + * for a particular source + */ + nir_lower_non_uniform_src_access_callback tex_src_callback; nir_lower_non_uniform_access_callback callback; void *callback_data; } nir_lower_non_uniform_access_options; diff --git a/src/compiler/nir/nir_clone.c b/src/compiler/nir/nir_clone.c index 8b450292b42..1668eeaddbf 100644 --- a/src/compiler/nir/nir_clone.c +++ b/src/compiler/nir/nir_clone.c @@ -417,6 +417,7 @@ clone_tex(clone_state *state, const nir_tex_instr *tex) ntex->texture_non_uniform = tex->texture_non_uniform; ntex->sampler_non_uniform = tex->sampler_non_uniform; + ntex->offset_non_uniform = tex->offset_non_uniform; ntex->backend_flags = tex->backend_flags; diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index f8b2bd03948..cfb3688f702 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -958,6 +958,10 @@ visit_tex(nir_tex_instr *instr, struct divergence_state *state) is_divergent |= src_divergent(instr->src[i].src, state) && instr->texture_non_uniform; break; + case nir_tex_src_offset: + instr->offset_non_uniform = src_divergent(instr->src[i].src, state); + is_divergent |= instr->offset_non_uniform; + break; default: is_divergent |= src_divergent(instr->src[i].src, state); break; diff --git a/src/compiler/nir/nir_lower_non_uniform_access.c b/src/compiler/nir/nir_lower_non_uniform_access.c index 1d10b6d7d5a..7a84aa2d272 100644 --- a/src/compiler/nir/nir_lower_non_uniform_access.c +++ b/src/compiler/nir/nir_lower_non_uniform_access.c @@ -204,9 +204,12 @@ add_non_uniform_instr(struct nu_state *state, struct nu_handle *handles, } static bool -lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex) +lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex, + const nir_lower_non_uniform_access_options *opts) { - if (!tex->texture_non_uniform && !tex->sampler_non_uniform) + if (!(tex->texture_non_uniform && (opts->types & nir_lower_non_uniform_texture_access)) && + !(tex->sampler_non_uniform && (opts->types & nir_lower_non_uniform_texture_access)) && + !(tex->offset_non_uniform && (opts->types & nir_lower_non_uniform_texture_offset_access))) return false; /* We can have at most one texture and one sampler handle */ @@ -220,6 +223,10 @@ lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex) case nir_tex_src_texture_deref: if (!tex->texture_non_uniform) continue; + if (!(opts->types & nir_lower_non_uniform_texture_access)) + continue; + if (opts->tex_src_callback && !opts->tex_src_callback(tex, i, opts->callback_data)) + continue; break; case nir_tex_src_sampler_offset: @@ -227,6 +234,19 @@ lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex) case nir_tex_src_sampler_deref: if (!tex->sampler_non_uniform) continue; + if (!(opts->types & nir_lower_non_uniform_texture_access)) + continue; + if (opts->tex_src_callback && !opts->tex_src_callback(tex, i, opts->callback_data)) + continue; + break; + + case nir_tex_src_offset: + if (!tex->offset_non_uniform) + continue; + if (!(opts->types & nir_lower_non_uniform_texture_offset_access)) + continue; + if (opts->tex_src_callback && !opts->tex_src_callback(tex, i, opts->callback_data)) + continue; break; default: @@ -243,11 +263,13 @@ lower_non_uniform_tex_access(struct nu_state *state, nir_tex_instr *tex) /* nu_handle_init() returned false because the handles are uniform. */ tex->texture_non_uniform = false; tex->sampler_non_uniform = false; + tex->offset_non_uniform = false; return false; } tex->texture_non_uniform = false; tex->sampler_non_uniform = false; + tex->offset_non_uniform = false; add_non_uniform_instr(state, handles, srcs, num_handles, true, nir_lower_non_uniform_texture_access); @@ -309,8 +331,9 @@ nir_lower_non_uniform_access_impl(nir_function_impl *impl, switch (instr->type) { case nir_instr_type_tex: { nir_tex_instr *tex = nir_instr_as_tex(instr); - if ((options->types & nir_lower_non_uniform_texture_access) && - lower_non_uniform_tex_access(&state, tex)) + if ((options->types & (nir_lower_non_uniform_texture_access | + nir_lower_non_uniform_texture_offset_access)) && + lower_non_uniform_tex_access(&state, tex, options)) progress = true; break; } diff --git a/src/compiler/nir/nir_lower_tex.c b/src/compiler/nir/nir_lower_tex.c index fcc06f6f18f..d4bda42da40 100644 --- a/src/compiler/nir/nir_lower_tex.c +++ b/src/compiler/nir/nir_lower_tex.c @@ -1402,6 +1402,7 @@ nir_lower_ms_txf_to_fragment_fetch(nir_builder *b, nir_tex_instr *tex) fmask_fetch->sampler_dim = tex->sampler_dim; fmask_fetch->is_array = tex->is_array; fmask_fetch->texture_non_uniform = tex->texture_non_uniform; + fmask_fetch->offset_non_uniform = tex->offset_non_uniform; fmask_fetch->dest_type = nir_type_uint32; nir_def_init(&fmask_fetch->instr, &fmask_fetch->def, 1, 32); diff --git a/src/compiler/nir/nir_opt_non_uniform_access.c b/src/compiler/nir/nir_opt_non_uniform_access.c index 2c2a1de000c..34e83f70837 100644 --- a/src/compiler/nir/nir_opt_non_uniform_access.c +++ b/src/compiler/nir/nir_opt_non_uniform_access.c @@ -81,9 +81,15 @@ is_image_intrinsic(nir_intrinsic_instr *intrin) } static bool -has_non_uniform_tex_access(nir_tex_instr *tex) +has_non_uniform_tex_access(nir_tex_instr *tex, enum nir_lower_non_uniform_access_type types) { - return tex->texture_non_uniform || tex->sampler_non_uniform; + bool ret = false; + + if (types & nir_lower_non_uniform_texture_access) + ret |= tex->texture_non_uniform || tex->sampler_non_uniform; + if (types & nir_lower_non_uniform_texture_offset_access) + ret |= tex->offset_non_uniform; + return ret; } static bool @@ -100,8 +106,7 @@ nir_has_non_uniform_access_impl(nir_function_impl *impl, enum nir_lower_non_unif switch (instr->type) { case nir_instr_type_tex: { nir_tex_instr *tex = nir_instr_as_tex(instr); - if ((types & nir_lower_non_uniform_texture_access) && - has_non_uniform_tex_access(tex)) + if (has_non_uniform_tex_access(tex, types)) return true; break; } @@ -154,7 +159,9 @@ nir_has_non_uniform_access(nir_shader *shader, enum nir_lower_non_uniform_access static bool opt_non_uniform_tex_access(nir_tex_instr *tex) { - if (!has_non_uniform_tex_access(tex)) + if (!has_non_uniform_tex_access(tex, + nir_lower_non_uniform_texture_access | + nir_lower_non_uniform_texture_offset_access)) return false; bool progress = false; @@ -179,6 +186,13 @@ opt_non_uniform_tex_access(nir_tex_instr *tex) } break; + case nir_tex_src_offset: + if (tex->offset_non_uniform && !nir_src_is_divergent(&tex->src[i].src)) { + tex->offset_non_uniform = false; + progress = true; + } + break; + default: break; } diff --git a/src/compiler/nir/nir_serialize.c b/src/compiler/nir/nir_serialize.c index ab0cfbea63e..49a07f4f0ef 100644 --- a/src/compiler/nir/nir_serialize.c +++ b/src/compiler/nir/nir_serialize.c @@ -1364,9 +1364,10 @@ union packed_tex_data { unsigned component : 2; unsigned texture_non_uniform : 1; unsigned sampler_non_uniform : 1; + unsigned offset_non_uniform : 1; unsigned array_is_lowered_cube : 1; unsigned is_gather_implicit_lod : 1; - unsigned unused : 5; /* Mark unused for valgrind. */ + unsigned unused : 4; /* Mark unused for valgrind. */ } u; }; @@ -1403,6 +1404,7 @@ write_tex(write_ctx *ctx, const nir_tex_instr *tex) .u.component = tex->component, .u.texture_non_uniform = tex->texture_non_uniform, .u.sampler_non_uniform = tex->sampler_non_uniform, + .u.offset_non_uniform = tex->offset_non_uniform, .u.array_is_lowered_cube = tex->array_is_lowered_cube, .u.is_gather_implicit_lod = tex->is_gather_implicit_lod, }; @@ -1442,6 +1444,7 @@ read_tex(read_ctx *ctx, union packed_instr header) tex->component = packed.u.component; tex->texture_non_uniform = packed.u.texture_non_uniform; tex->sampler_non_uniform = packed.u.sampler_non_uniform; + tex->offset_non_uniform = packed.u.offset_non_uniform; tex->array_is_lowered_cube = packed.u.array_is_lowered_cube; tex->is_gather_implicit_lod = packed.u.is_gather_implicit_lod;