From bf453a7c6a1a07927add13e28fbddaea8bbda249 Mon Sep 17 00:00:00 2001 From: Georg Lehmann Date: Mon, 25 Aug 2025 15:50:14 +0200 Subject: [PATCH] aco/isel: add init_disable_wqm helper Reviewed-by: Rhys Perry Part-of: --- .../aco_instruction_selection.h | 13 ++++++ .../aco_isel_helpers.cpp | 8 +--- .../instruction_selection/aco_select_nir.cpp | 7 +--- .../aco_select_nir_intrinsics.cpp | 40 ++++--------------- 4 files changed, 23 insertions(+), 45 deletions(-) diff --git a/src/amd/compiler/instruction_selection/aco_instruction_selection.h b/src/amd/compiler/instruction_selection/aco_instruction_selection.h index 73ece52ad89..f0e34ab7ba7 100644 --- a/src/amd/compiler/instruction_selection/aco_instruction_selection.h +++ b/src/amd/compiler/instruction_selection/aco_instruction_selection.h @@ -7,6 +7,7 @@ #ifndef ACO_INSTRUCTION_SELECTION_H #define ACO_INSTRUCTION_SELECTION_H +#include "aco_builder.h" #include "aco_ir.h" #include "nir.h" @@ -178,6 +179,18 @@ should_declare_array(ac_image_dim dim) dim == ac_image_2darraymsaa; } +template +inline void +init_disable_wqm(Builder& bld, T& instr, bool disable_wqm) +{ + if (disable_wqm) { + instr_exact_mask(&instr) = Operand(); + instr_wqm_mask(&instr) = Operand(); + instr.disable_wqm = true; + bld.program->needs_exact = true; + } +} + /* aco_isel_setup.cpp */ void init_context(isel_context* ctx, nir_shader* shader); void cleanup_context(isel_context* ctx); diff --git a/src/amd/compiler/instruction_selection/aco_isel_helpers.cpp b/src/amd/compiler/instruction_selection/aco_isel_helpers.cpp index 6157d2ee7bc..bb1bc69f842 100644 --- a/src/amd/compiler/instruction_selection/aco_isel_helpers.cpp +++ b/src/amd/compiler/instruction_selection/aco_isel_helpers.cpp @@ -551,13 +551,7 @@ emit_mimg(Builder& bld, aco_opcode op, std::vector dsts, Temp rsrc, Operan for (unsigned i = 0; i < coords.size(); i++) mimg->operands[3 + i] = Operand(coords[i]); - if (disable_wqm) { - instr_exact_mask(mimg.get()) = Operand(); - instr_wqm_mask(mimg.get()) = Operand(); - mimg->mimg().disable_wqm = true; - bld.program->needs_exact = true; - } - + init_disable_wqm(bld, mimg->mimg(), disable_wqm); mimg->mimg().strict_wqm = strict_wqm; return &bld.insert(std::move(mimg))->mimg(); diff --git a/src/amd/compiler/instruction_selection/aco_select_nir.cpp b/src/amd/compiler/instruction_selection/aco_select_nir.cpp index 9f3be113d5e..5b9296b4e1b 100644 --- a/src/amd/compiler/instruction_selection/aco_select_nir.cpp +++ b/src/amd/compiler/instruction_selection/aco_select_nir.cpp @@ -465,12 +465,7 @@ visit_tex(isel_context* ctx, nir_tex_instr* instr) mubuf->mubuf().tfe = instr->is_sparse; if (mubuf->mubuf().tfe) mubuf->operands[3] = emit_tfe_init(bld, tmp_dst); - if (disable_wqm) { - instr_exact_mask(mubuf.get()) = Operand(); - instr_wqm_mask(mubuf.get()) = Operand(); - mubuf->mubuf().disable_wqm = true; - bld.program->needs_exact = true; - } + init_disable_wqm(bld, mubuf->mubuf(), disable_wqm); ctx->block->instructions.emplace_back(std::move(mubuf)); expand_vector(ctx, tmp_dst, dst, instr->def.num_components, dmask); diff --git a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp index 9afc894fe4e..73f8bd4b4bd 100644 --- a/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp +++ b/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp @@ -1990,13 +1990,10 @@ visit_image_store(isel_context* ctx, nir_intrinsic_instr* instr) store->operands[1] = Operand(vindex); store->operands[2] = Operand::c32(0); store->operands[3] = Operand(data); - store->operands[4] = Operand(); - store->operands[5] = Operand(); store->mubuf().idxen = true; store->mubuf().cache = cache; - store->mubuf().disable_wqm = true; store->mubuf().sync = sync; - ctx->program->needs_exact = true; + init_disable_wqm(bld, store->mubuf(), true); ctx->block->instructions.emplace_back(std::move(store)); return; } @@ -2142,8 +2139,6 @@ visit_image_atomic(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->operands[1] = Operand(vindex); mubuf->operands[2] = Operand::c32(0); mubuf->operands[3] = Operand(data); - mubuf->operands[4] = Operand(); - mubuf->operands[5] = Operand(); Definition def = return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition(); if (return_previous) @@ -2151,9 +2146,8 @@ visit_image_atomic(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->mubuf().offset = 0; mubuf->mubuf().idxen = true; mubuf->mubuf().cache = get_atomic_cache_flags(ctx, return_previous); - mubuf->mubuf().disable_wqm = true; mubuf->mubuf().sync = sync; - ctx->program->needs_exact = true; + init_disable_wqm(bld, mubuf->mubuf(), true); ctx->block->instructions.emplace_back(std::move(mubuf)); if (return_previous && cmpswap) bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), def.getTemp(), Operand::zero()); @@ -2234,14 +2228,11 @@ visit_store_ssbo(isel_context* ctx, nir_intrinsic_instr* instr) store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1); store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0); store->operands[3] = Operand(write_datas[i]); - store->operands[4] = Operand(); - store->operands[5] = Operand(); store->mubuf().offset = offsets[i]; store->mubuf().offen = (offset.type() == RegType::vgpr); store->mubuf().cache = get_cache_flags(ctx, access, type); - store->mubuf().disable_wqm = true; store->mubuf().sync = sync; - ctx->program->needs_exact = true; + init_disable_wqm(bld, store->mubuf(), true); ctx->block->instructions.emplace_back(std::move(store)); } } @@ -2273,8 +2264,6 @@ visit_atomic_ssbo(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1); mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0); mubuf->operands[3] = Operand(data); - mubuf->operands[4] = Operand(); - mubuf->operands[5] = Operand(); Definition def = return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition(); if (return_previous) @@ -2282,9 +2271,8 @@ visit_atomic_ssbo(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->mubuf().offset = 0; mubuf->mubuf().offen = (offset.type() == RegType::vgpr); mubuf->mubuf().cache = get_atomic_cache_flags(ctx, return_previous); - mubuf->mubuf().disable_wqm = true; mubuf->mubuf().sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw); - ctx->program->needs_exact = true; + init_disable_wqm(bld, mubuf->mubuf(), true); ctx->block->instructions.emplace_back(std::move(mubuf)); if (return_previous && cmpswap) bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), def.getTemp(), Operand::zero()); @@ -2409,14 +2397,11 @@ visit_store_global(isel_context* ctx, nir_intrinsic_instr* instr) flat->operands[1] = Operand(s1); } flat->operands[2] = Operand(write_datas[i]); - flat->operands[3] = Operand(); - flat->operands[4] = Operand(); flat->flatlike().cache = get_cache_flags(ctx, access, type); assert(global || !write_const_offset); flat->flatlike().offset = write_const_offset; - flat->flatlike().disable_wqm = true; flat->flatlike().sync = sync; - ctx->program->needs_exact = true; + init_disable_wqm(bld, flat->flatlike(), true); ctx->block->instructions.emplace_back(std::move(flat)); } else { assert(ctx->options->gfx_level == GFX6 || write_address.type() != RegType::vgpr); @@ -2436,15 +2421,12 @@ visit_store_global(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->operands[2] = write_offset.type() == RegType::sgpr ? Operand(write_offset) : Operand::c32(0); mubuf->operands[3] = Operand(write_datas[i]); - mubuf->operands[4] = Operand(); - mubuf->operands[5] = Operand(); mubuf->mubuf().offen = write_offset.type() == RegType::vgpr; mubuf->mubuf().cache = get_cache_flags(ctx, access, type); mubuf->mubuf().offset = write_const_offset; mubuf->mubuf().addr64 = write_address.type() == RegType::vgpr; - mubuf->mubuf().disable_wqm = true; mubuf->mubuf().sync = sync; - ctx->program->needs_exact = true; + init_disable_wqm(bld, mubuf->mubuf(), true); ctx->block->instructions.emplace_back(std::move(mubuf)); } } @@ -2548,16 +2530,13 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr) flat->operands[1] = Operand(s1); } flat->operands[2] = Operand(data); - flat->operands[3] = Operand(); - flat->operands[4] = Operand(); if (return_previous) flat->definitions[0] = Definition(dst); flat->flatlike().cache = get_atomic_cache_flags(ctx, return_previous); assert(global || !const_offset); flat->flatlike().offset = const_offset; - flat->flatlike().disable_wqm = true; flat->flatlike().sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw); - ctx->program->needs_exact = true; + init_disable_wqm(bld, flat->flatlike(), true); ctx->block->instructions.emplace_back(std::move(flat)); } else { assert(ctx->options->gfx_level == GFX6 || addr.type() != RegType::vgpr); @@ -2579,8 +2558,6 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->operands[1] = Operand(v1); mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand::c32(0); mubuf->operands[3] = Operand(data); - mubuf->operands[4] = Operand(); - mubuf->operands[5] = Operand(); Definition def = return_previous ? (cmpswap ? bld.def(data.regClass()) : Definition(dst)) : Definition(); if (return_previous) @@ -2589,9 +2566,8 @@ visit_global_atomic(isel_context* ctx, nir_intrinsic_instr* instr) mubuf->mubuf().cache = get_atomic_cache_flags(ctx, return_previous); mubuf->mubuf().offset = const_offset; mubuf->mubuf().addr64 = addr.type() == RegType::vgpr; - mubuf->mubuf().disable_wqm = true; mubuf->mubuf().sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw); - ctx->program->needs_exact = true; + init_disable_wqm(bld, mubuf->mubuf(), true); ctx->block->instructions.emplace_back(std::move(mubuf)); if (return_previous && cmpswap) bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), def.getTemp(), Operand::zero());