From b68f2e747c71dd3524ef9514099ea28c96534c7f Mon Sep 17 00:00:00 2001 From: Faith Ekstrand Date: Mon, 11 Mar 2024 19:30:11 -0500 Subject: [PATCH] zink: Rework sparse texture lowering Instead of the previous fragile attempt to handle sparse_resident_and by crawling deref chains, we now insert an is_sparse_resident_zink intrinsic immediately after the tex or sparse_load intrinsic and define Zink's sparse resident codes to always be 0/1. Then sparse_resident_and becomes iand and is_sparse_texels_resident becomes != 0 and everything is well-defined and robust. Part-of: --- src/compiler/nir/nir_intrinsics.py | 7 + .../drivers/zink/nir_to_spirv/nir_to_spirv.c | 8 +- src/gallium/drivers/zink/zink_compiler.c | 123 ++++++------------ 3 files changed, 51 insertions(+), 87 deletions(-) diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 7a4a3040d62..e976ca5b01d 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -402,6 +402,13 @@ intrinsic("is_sparse_texels_resident", dest_comp=1, src_comp=[1], bit_sizes=[1,3 intrinsic("sparse_residency_code_and", dest_comp=1, src_comp=[1, 1], bit_sizes=[32], flags=[CAN_ELIMINATE, CAN_REORDER]) +# Unlike is_sparse_texels_resident, this intrinsic is required to consume +# the destination of the nir_tex_instr or sparse_load intrinsic directly. +# As such it is allowed to ignore the .e component where we usually store +# sparse information. +intrinsic("is_sparse_resident_zink", dest_comp=1, src_comp=[0], bit_sizes=[1], + flags=[CAN_ELIMINATE, CAN_REORDER]) + # a barrier is an intrinsic with no inputs/outputs but which can't be moved # around/optimized in general def barrier(name): diff --git a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c index 718f4ea2fbe..2ef93153b2d 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c +++ b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c @@ -3140,11 +3140,7 @@ emit_is_sparse_texels_resident(struct ntv_context *ctx, nir_intrinsic_instr *int SpvId type = get_def_type(ctx, &intr->def, nir_type_uint); - /* this will always be stored with the ssa index of the parent instr */ - nir_def *ssa = intr->src[0].ssa; - assert(ssa->parent_instr->type == nir_instr_type_alu); - nir_alu_instr *alu = nir_instr_as_alu(ssa->parent_instr); - unsigned index = alu->src[0].src.ssa->index; + unsigned index = intr->src[0].ssa->index; assert(index < ctx->num_defs); assert(ctx->resident_defs[index] != 0); SpvId resident = ctx->resident_defs[index]; @@ -3480,7 +3476,7 @@ emit_intrinsic(struct ntv_context *ctx, nir_intrinsic_instr *intr) emit_vote(ctx, intr); break; - case nir_intrinsic_is_sparse_texels_resident: + case nir_intrinsic_is_sparse_resident_zink: emit_is_sparse_texels_resident(ctx, intr); break; diff --git a/src/gallium/drivers/zink/zink_compiler.c b/src/gallium/drivers/zink/zink_compiler.c index c0866a5fc56..dca01f14c4f 100644 --- a/src/gallium/drivers/zink/zink_compiler.c +++ b/src/gallium/drivers/zink/zink_compiler.c @@ -3546,98 +3546,60 @@ invert_point_coord(nir_shader *nir) } static bool -is_residency_code(nir_def *src) +lower_sparse_instr(nir_builder *b, nir_instr *instr, void *data) { - nir_instr *parent = src->parent_instr; - while (1) { - if (parent->type == nir_instr_type_intrinsic) { - ASSERTED nir_intrinsic_instr *intr = nir_instr_as_intrinsic(parent); - assert(intr->intrinsic == nir_intrinsic_is_sparse_texels_resident); + b->cursor = nir_after_instr(instr); + + switch (instr->type) { + case nir_instr_type_tex: { + nir_tex_instr *tex = nir_instr_as_tex(instr); + if (!tex->is_sparse) + return false; + + nir_def *res = nir_b2i32(b, nir_is_sparse_resident_zink(b, &tex->def)); + nir_def *vec = nir_vector_insert_imm(b, &tex->def, res, + tex->def.num_components - 1); + nir_def_rewrite_uses_after(&tex->def, vec, vec->parent_instr); + return true; + } + + case nir_instr_type_intrinsic: { + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + switch (intrin->intrinsic) { + case nir_intrinsic_image_deref_sparse_load: { + nir_def *res = nir_b2i32(b, nir_is_sparse_resident_zink(b, &intrin->def)); + nir_def *vec = nir_vector_insert_imm(b, &intrin->def, res, 4); + nir_def_rewrite_uses_after(&intrin->def, vec, vec->parent_instr); + return true; + } + + case nir_intrinsic_sparse_residency_code_and: { + nir_def *res = nir_iand(b, intrin->src[0].ssa, intrin->src[1].ssa); + nir_def_rewrite_uses(&intrin->def, res); + return true; + } + + case nir_intrinsic_is_sparse_texels_resident: { + nir_def *res = nir_i2b(b, intrin->src[0].ssa); + nir_def_rewrite_uses(&intrin->def, res); + return true; + } + + default: return false; } - if (parent->type == nir_instr_type_tex) - return true; - assert(parent->type == nir_instr_type_alu); - nir_alu_instr *alu = nir_instr_as_alu(parent); - parent = alu->src[0].src.ssa->parent_instr; } -} -static bool -lower_sparse_and_instr(nir_builder *b, nir_intrinsic_instr *instr, void *data) -{ - if (instr->intrinsic != nir_intrinsic_sparse_residency_code_and) + default: return false; - - b->cursor = nir_before_instr(&instr->instr); - nir_def *src0; - if (is_residency_code(instr->src[0].ssa)) - src0 = nir_is_sparse_texels_resident(b, 1, instr->src[0].ssa); - else - src0 = instr->src[0].ssa; - nir_def *src1; - if (is_residency_code(instr->src[1].ssa)) - src1 = nir_is_sparse_texels_resident(b, 1, instr->src[1].ssa); - else - src1 = instr->src[1].ssa; - nir_def *def = nir_iand(b, src0, src1); - nir_def_rewrite_uses_after(&instr->def, def, &instr->instr); - nir_instr_remove(&instr->instr); - return true; -} - -static bool -lower_sparse_and(nir_shader *shader) -{ - return nir_shader_intrinsics_pass(shader, lower_sparse_and_instr, - nir_metadata_dominance, NULL); -} - -static bool -lower_sparse_instr(nir_builder *b, nir_intrinsic_instr *instr, void *data) -{ - if (instr->intrinsic != nir_intrinsic_is_sparse_texels_resident) - return false; - - /* vulkan vec can only be a vec4, but this is (maybe) vec5, - * so just rewrite as the first component since ntv is going to use a different - * method for storing the residency value anyway - */ - b->cursor = nir_before_instr(&instr->instr); - nir_instr *parent = instr->src[0].ssa->parent_instr; - if (is_residency_code(instr->src[0].ssa)) { - assert(parent->type == nir_instr_type_alu); - nir_alu_instr *alu = nir_instr_as_alu(parent); - nir_def_rewrite_uses_after(instr->src[0].ssa, nir_channel(b, alu->src[0].src.ssa, 0), parent); - nir_instr_remove(parent); - } else { - nir_def *src; - if (parent->type == nir_instr_type_intrinsic) { - nir_intrinsic_instr *intr = nir_instr_as_intrinsic(parent); - assert(intr->intrinsic == nir_intrinsic_is_sparse_texels_resident); - src = intr->src[0].ssa; - } else { - assert(parent->type == nir_instr_type_alu); - nir_alu_instr *alu = nir_instr_as_alu(parent); - src = alu->src[0].src.ssa; - } - if (instr->def.bit_size != 32) { - if (instr->def.bit_size == 1) - src = nir_ieq_imm(b, src, 1); - else - src = nir_u2uN(b, src, instr->def.bit_size); - } - nir_def_rewrite_uses(&instr->def, src); - nir_instr_remove(&instr->instr); } - return true; } static bool lower_sparse(nir_shader *shader) { - return nir_shader_intrinsics_pass(shader, lower_sparse_instr, - nir_metadata_dominance, NULL); + return nir_shader_instructions_pass(shader, lower_sparse_instr, + nir_metadata_dominance, NULL); } static bool @@ -5572,7 +5534,6 @@ zink_shader_create(struct zink_screen *screen, struct nir_shader *nir) NIR_PASS_V(nir, lower_basevertex); NIR_PASS_V(nir, lower_baseinstance); - NIR_PASS_V(nir, lower_sparse_and); NIR_PASS_V(nir, split_bitfields); if (!screen->info.feats.features.shaderStorageImageMultisample) NIR_PASS_V(nir, strip_tex_ms);