fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2026-02-21 03:30:30 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

r600: Use unified atomics

Browse source 
Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Reviewed-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/23026>
This commit is contained in:
Alyssa Rosenzweig 2023-05-15 09:17:32 -04:00 • committed by Marge Bot
parent a0e51fcc4e
commit 6c90fe189f
4 changed files with 61 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
src/gallium/drivers/r600/sfn/sfn_instr_mem.cpp
View file

@ -443,88 +443,60 @@ RatInstr::do_print(std::ostream& os) const
}
static RatInstr::ERatOp
get_rat_opcode(const nir_intrinsic_op opcode, pipe_format format)
get_rat_opcode(const nir_atomic_op opcode)
{
switch (opcode) {
case nir_intrinsic_image_load:
return RatInstr::NOP_RTN;
case nir_intrinsic_ssbo_atomic_add:
case nir_intrinsic_image_atomic_add:
case nir_atomic_op_iadd:
return RatInstr::ADD_RTN;
case nir_intrinsic_ssbo_atomic_and:
case nir_intrinsic_image_atomic_and:
case nir_atomic_op_iand:
return RatInstr::AND_RTN;
case nir_intrinsic_ssbo_atomic_or:
case nir_intrinsic_image_atomic_or:
case nir_atomic_op_ior:
return RatInstr::OR_RTN;
case nir_intrinsic_ssbo_atomic_imin:
case nir_intrinsic_image_atomic_imin:
case nir_atomic_op_imin:
return RatInstr::MIN_INT_RTN;
case nir_intrinsic_ssbo_atomic_imax:
case nir_intrinsic_image_atomic_imax:
case nir_atomic_op_imax:
return RatInstr::MAX_INT_RTN;
case nir_intrinsic_ssbo_atomic_umin:
case nir_intrinsic_image_atomic_umin:
case nir_atomic_op_umin:
return RatInstr::MIN_UINT_RTN;
case nir_intrinsic_ssbo_atomic_umax:
case nir_intrinsic_image_atomic_umax:
case nir_atomic_op_umax:
return RatInstr::MAX_UINT_RTN;
case nir_intrinsic_ssbo_atomic_xor:
case nir_intrinsic_image_atomic_xor:
case nir_atomic_op_ixor:
return RatInstr::XOR_RTN;
case nir_intrinsic_ssbo_atomic_comp_swap:
case nir_intrinsic_image_atomic_comp_swap:
if (util_format_is_float(format))
return RatInstr::CMPXCHG_FLT_RTN;
else
return RatInstr::CMPXCHG_INT_RTN;
case nir_intrinsic_ssbo_atomic_exchange:
case nir_intrinsic_image_atomic_exchange:
case nir_atomic_op_cmpxchg:
return RatInstr::CMPXCHG_INT_RTN;
case nir_atomic_op_xchg:
return RatInstr::XCHG_RTN;
default:
unreachable("Unsupported WO RAT instruction");
unreachable("Unsupported atomic");
}
}
static RatInstr::ERatOp
get_rat_opcode_wo(const nir_intrinsic_op opcode, pipe_format format)
get_rat_opcode_wo(const nir_atomic_op opcode)
{
switch (opcode) {
case nir_intrinsic_ssbo_atomic_add:
case nir_intrinsic_image_atomic_add:
case nir_atomic_op_iadd:
return RatInstr::ADD;
case nir_intrinsic_ssbo_atomic_and:
case nir_intrinsic_image_atomic_and:
case nir_atomic_op_iand:
return RatInstr::AND;
case nir_intrinsic_ssbo_atomic_or:
case nir_intrinsic_image_atomic_or:
case nir_atomic_op_ior:
return RatInstr::OR;
case nir_intrinsic_ssbo_atomic_imin:
case nir_intrinsic_image_atomic_imin:
case nir_atomic_op_imin:
return RatInstr::MIN_INT;
case nir_intrinsic_ssbo_atomic_imax:
case nir_intrinsic_image_atomic_imax:
case nir_atomic_op_imax:
return RatInstr::MAX_INT;
case nir_intrinsic_ssbo_atomic_umin:
case nir_intrinsic_image_atomic_umin:
case nir_atomic_op_umin:
return RatInstr::MIN_UINT;
case nir_intrinsic_ssbo_atomic_umax:
case nir_intrinsic_image_atomic_umax:
case nir_atomic_op_umax:
return RatInstr::MAX_UINT;
case nir_intrinsic_ssbo_atomic_xor:
case nir_intrinsic_image_atomic_xor:
case nir_atomic_op_ixor:
return RatInstr::XOR;
case nir_intrinsic_ssbo_atomic_comp_swap:
case nir_intrinsic_image_atomic_comp_swap:
if (util_format_is_float(format))
return RatInstr::CMPXCHG_FLT;
else
return RatInstr::CMPXCHG_INT;
case nir_intrinsic_ssbo_atomic_exchange:
case nir_intrinsic_image_atomic_exchange:
case nir_atomic_op_cmpxchg:
return RatInstr::CMPXCHG_INT;
case nir_atomic_op_xchg:
return RatInstr::XCHG_RTN;
default:
unreachable("Unsupported WO RAT instruction");
unreachable("Unsupported atomic");
}
}
@ -536,30 +508,14 @@ RatInstr::emit(nir_intrinsic_instr *intr, Shader& shader)
return emit_ssbo_load(intr, shader);
case nir_intrinsic_store_ssbo:
return emit_ssbo_store(intr, shader);
case nir_intrinsic_ssbo_atomic_add:
case nir_intrinsic_ssbo_atomic_comp_swap:
case nir_intrinsic_ssbo_atomic_or:
case nir_intrinsic_ssbo_atomic_xor:
case nir_intrinsic_ssbo_atomic_imax:
case nir_intrinsic_ssbo_atomic_imin:
case nir_intrinsic_ssbo_atomic_umax:
case nir_intrinsic_ssbo_atomic_umin:
case nir_intrinsic_ssbo_atomic_and:
case nir_intrinsic_ssbo_atomic_exchange:
case nir_intrinsic_ssbo_atomic:
case nir_intrinsic_ssbo_atomic_swap:
return emit_ssbo_atomic_op(intr, shader);
case nir_intrinsic_image_store:
return emit_image_store(intr, shader);
case nir_intrinsic_image_load:
case nir_intrinsic_image_atomic_add:
case nir_intrinsic_image_atomic_and:
case nir_intrinsic_image_atomic_or:
case nir_intrinsic_image_atomic_xor:
case nir_intrinsic_image_atomic_exchange:
case nir_intrinsic_image_atomic_comp_swap:
case nir_intrinsic_image_atomic_umin:
case nir_intrinsic_image_atomic_umax:
case nir_intrinsic_image_atomic_imin:
case nir_intrinsic_image_atomic_imax:
case nir_intrinsic_image_atomic:
case nir_intrinsic_image_atomic_swap:
return emit_image_load_or_atomic(intr, shader);
case nir_intrinsic_image_size:
return emit_image_size(intr, shader);
@ -662,8 +618,8 @@ RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader)
}
bool read_result = !intr->dest.is_ssa || !list_is_empty(&intr->dest.ssa.uses);
auto opcode = read_result ? get_rat_opcode(intr->intrinsic, PIPE_FORMAT_R32_UINT)
: get_rat_opcode_wo(intr->intrinsic, PIPE_FORMAT_R32_UINT);
auto opcode = read_result ? get_rat_opcode(nir_intrinsic_atomic_op(intr))
: get_rat_opcode_wo(nir_intrinsic_atomic_op(intr));
auto coord_orig = vf.src(intr->src[1], 0);
auto coord = vf.temp_register(0);
@ -676,7 +632,7 @@ RatInstr::emit_ssbo_atomic_op(nir_intrinsic_instr *intr, Shader& shader)
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write));
if (intr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) {
if (intr->intrinsic == nir_intrinsic_ssbo_atomic_swap) {
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[0], vf.src(intr->src[3], 0), AluInstr::write));
shader.emit_instruction(
@ -799,8 +755,10 @@ RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& shader)
}
bool read_result = !intrin->dest.is_ssa || !list_is_empty(&intrin->dest.ssa.uses);
auto opcode = read_result ? get_rat_opcode(intrin->intrinsic, PIPE_FORMAT_R32_UINT)
: get_rat_opcode_wo(intrin->intrinsic, PIPE_FORMAT_R32_UINT);
bool image_load = (intrin->intrinsic == nir_intrinsic_image_load);
auto opcode = image_load ? RatInstr::NOP_RTN :
read_result ? get_rat_opcode(nir_intrinsic_atomic_op(intrin))
: get_rat_opcode_wo(nir_intrinsic_atomic_op(intrin));
auto coord_orig = vf.src_vec4(intrin->src[1], pin_chan);
auto coord = vf.temp_vec4(pin_chgr);
@ -821,7 +779,7 @@ RatInstr::emit_image_load_or_atomic(nir_intrinsic_instr *intrin, Shader& shader)
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[1], shader.rat_return_address(), AluInstr::write));
if (intrin->intrinsic == nir_intrinsic_image_atomic_comp_swap) {
if (intrin->intrinsic == nir_intrinsic_image_atomic_swap) {
shader.emit_instruction(
new AluInstr(op1_mov, data_vec4[0], vf.src(intrin->src[4], 0), AluInstr::write));
shader.emit_instruction(

1
src/gallium/drivers/r600/sfn/sfn_nir.cpp
View file

@ -782,6 +782,7 @@ r600_finalize_nir(pipe_screen *screen, void *shader)
NIR_PASS_V(nir, r600_nir_lower_pack_unpack_2x16);
NIR_PASS_V(nir, r600_lower_shared_io);
NIR_PASS_V(nir, nir_lower_legacy_atomics);
NIR_PASS_V(nir, r600_nir_lower_atomics);
if (rs->b.gfx_level == CAYMAN)

12
src/gallium/drivers/r600/sfn/sfn_nir_legalize_image_load_store.cpp
View file

@ -168,16 +168,8 @@ r600_legalize_image_load_store_filter(const nir_instr *instr, UNUSED const void
switch (ir->intrinsic) {
case nir_intrinsic_image_store:
case nir_intrinsic_image_load:
case nir_intrinsic_image_atomic_add:
case nir_intrinsic_image_atomic_and:
case nir_intrinsic_image_atomic_or:
case nir_intrinsic_image_atomic_xor:
case nir_intrinsic_image_atomic_exchange:
case nir_intrinsic_image_atomic_comp_swap:
case nir_intrinsic_image_atomic_umin:
case nir_intrinsic_image_atomic_umax:
case nir_intrinsic_image_atomic_imin:
case nir_intrinsic_image_atomic_imax:
case nir_intrinsic_image_atomic:
case nir_intrinsic_image_atomic_swap:
case nir_intrinsic_image_size:
return true;
default:

64
src/gallium/drivers/r600/sfn/sfn_shader.cpp
View file

@ -663,27 +663,11 @@ Shader::scan_instruction(nir_instr *instr)
// handle unhandled instructions
switch (intr->intrinsic) {
case nir_intrinsic_ssbo_atomic_add:
case nir_intrinsic_ssbo_atomic_comp_swap:
case nir_intrinsic_ssbo_atomic_or:
case nir_intrinsic_ssbo_atomic_xor:
case nir_intrinsic_ssbo_atomic_imax:
case nir_intrinsic_ssbo_atomic_imin:
case nir_intrinsic_ssbo_atomic_umax:
case nir_intrinsic_ssbo_atomic_umin:
case nir_intrinsic_ssbo_atomic_and:
case nir_intrinsic_ssbo_atomic_exchange:
case nir_intrinsic_ssbo_atomic:
case nir_intrinsic_ssbo_atomic_swap:
case nir_intrinsic_image_load:
case nir_intrinsic_image_atomic_add:
case nir_intrinsic_image_atomic_and:
case nir_intrinsic_image_atomic_or:
case nir_intrinsic_image_atomic_xor:
case nir_intrinsic_image_atomic_exchange:
case nir_intrinsic_image_atomic_comp_swap:
case nir_intrinsic_image_atomic_umin:
case nir_intrinsic_image_atomic_umax:
case nir_intrinsic_image_atomic_imin:
case nir_intrinsic_image_atomic_imax:
case nir_intrinsic_image_atomic:
case nir_intrinsic_image_atomic_swap:
m_flags.set(sh_needs_sbo_ret_address);
FALLTHROUGH;
case nir_intrinsic_image_store:
@ -907,16 +891,8 @@ Shader::process_intrinsic(nir_intrinsic_instr *intr)
case nir_intrinsic_memory_barrier:
return emit_wait_ack();
case nir_intrinsic_shared_atomic_add:
case nir_intrinsic_shared_atomic_and:
case nir_intrinsic_shared_atomic_or:
case nir_intrinsic_shared_atomic_imax:
case nir_intrinsic_shared_atomic_umax:
case nir_intrinsic_shared_atomic_imin:
case nir_intrinsic_shared_atomic_umin:
case nir_intrinsic_shared_atomic_xor:
case nir_intrinsic_shared_atomic_exchange:
case nir_intrinsic_shared_atomic_comp_swap:
case nir_intrinsic_shared_atomic:
case nir_intrinsic_shared_atomic_swap:
return emit_atomic_local_shared(intr);
case nir_intrinsic_shader_clock:
return emit_shader_clock(intr);
@ -927,31 +903,31 @@ Shader::process_intrinsic(nir_intrinsic_instr *intr)
}
static ESDOp
lds_op_from_intrinsic(nir_intrinsic_op op, bool ret)
lds_op_from_intrinsic(nir_atomic_op op, bool ret)
{
switch (op) {
case nir_intrinsic_shared_atomic_add:
case nir_atomic_op_iadd:
return ret ? LDS_ADD_RET : LDS_ADD;
case nir_intrinsic_shared_atomic_and:
case nir_atomic_op_iand:
return ret ? LDS_AND_RET : LDS_AND;
case nir_intrinsic_shared_atomic_or:
case nir_atomic_op_ior:
return ret ? LDS_OR_RET : LDS_OR;
case nir_intrinsic_shared_atomic_imax:
case nir_atomic_op_imax:
return ret ? LDS_MAX_INT_RET : LDS_MAX_INT;
case nir_intrinsic_shared_atomic_umax:
case nir_atomic_op_umax:
return ret ? LDS_MAX_UINT_RET : LDS_MAX_UINT;
case nir_intrinsic_shared_atomic_imin:
case nir_atomic_op_imin:
return ret ? LDS_MIN_INT_RET : LDS_MIN_INT;
case nir_intrinsic_shared_atomic_umin:
case nir_atomic_op_umin:
return ret ? LDS_MIN_UINT_RET : LDS_MIN_UINT;
case nir_intrinsic_shared_atomic_xor:
case nir_atomic_op_ixor:
return ret ? LDS_XOR_RET : LDS_XOR;
case nir_intrinsic_shared_atomic_exchange:
case nir_atomic_op_xchg:
return LDS_XCHG_RET;
case nir_intrinsic_shared_atomic_comp_swap:
case nir_atomic_op_cmpxchg:
return LDS_CMP_XCHG_RET;
default:
unreachable("Unsupported shared atomic opcode");
unreachable("Unsupported shared atomic_op opcode");
}
}
@ -977,14 +953,14 @@ Shader::emit_atomic_local_shared(nir_intrinsic_instr *instr)
auto dest_value = uses_retval ? vf.dest(instr->dest, 0, pin_free) : nullptr;
auto op = lds_op_from_intrinsic(instr->intrinsic, uses_retval);
auto op = lds_op_from_intrinsic(nir_intrinsic_atomic_op(instr), uses_retval);
auto address = vf.src(instr->src[0], 0);
AluInstr::SrcValues src;
src.push_back(vf.src(instr->src[1], 0));
if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap))
if (unlikely(instr->intrinsic == nir_intrinsic_shared_atomic_swap))
src.push_back(vf.src(instr->src[2], 0));
emit_instruction(new LDSAtomicInstr(op, dest_value, address, src));
return true;