mirror of
https://gitlab.freedesktop.org/mesa/mesa.git
synced 2026-05-08 02:38:04 +02:00
nv50/ir: Use unified atomics
Reviewed-by: Alyssa Rosenzweig <alyssa@rosenzweig.io> Reviewed-by: Jesse Natalie <jenatali@microsoft.com> Reviewed-by: M Henning <drawoc@darkrefraction.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22914>
This commit is contained in:
Karol Herbst
Marge Bot
parent
b98b7f4d85
commit
6ff97776b7
1 changed files with 73 additions and 207 deletions
|
|
@ -141,6 +141,7 @@ private:
|
|||
|
||||
int getSubOp(nir_intrinsic_op);
|
||||
int getSubOp(nir_op);
|
||||
int getAtomicSubOp(nir_atomic_op);
|
||||
|
||||
CondCode getCondCode(nir_op);
|
||||
|
||||
|
|
@ -268,27 +269,16 @@ Converter::getDType(nir_intrinsic_instr *insn)
|
|||
{
|
||||
bool isFloat, isSigned;
|
||||
switch (insn->intrinsic) {
|
||||
case nir_intrinsic_bindless_image_atomic_fadd:
|
||||
case nir_intrinsic_global_atomic_fadd:
|
||||
case nir_intrinsic_image_atomic_fadd:
|
||||
case nir_intrinsic_shared_atomic_fadd:
|
||||
case nir_intrinsic_ssbo_atomic_fadd:
|
||||
isFloat = true;
|
||||
isSigned = false;
|
||||
break;
|
||||
case nir_intrinsic_bindless_image_atomic_imax:
|
||||
case nir_intrinsic_bindless_image_atomic_imin:
|
||||
case nir_intrinsic_global_atomic_imax:
|
||||
case nir_intrinsic_global_atomic_imin:
|
||||
case nir_intrinsic_image_atomic_imax:
|
||||
case nir_intrinsic_image_atomic_imin:
|
||||
case nir_intrinsic_shared_atomic_imax:
|
||||
case nir_intrinsic_shared_atomic_imin:
|
||||
case nir_intrinsic_ssbo_atomic_imax:
|
||||
case nir_intrinsic_ssbo_atomic_imin:
|
||||
isFloat = false;
|
||||
isSigned = true;
|
||||
case nir_intrinsic_bindless_image_atomic:
|
||||
case nir_intrinsic_global_atomic:
|
||||
case nir_intrinsic_image_atomic:
|
||||
case nir_intrinsic_shared_atomic:
|
||||
case nir_intrinsic_ssbo_atomic: {
|
||||
nir_alu_type type = nir_atomic_op_type(nir_intrinsic_atomic_op(insn));
|
||||
isFloat = type == nir_type_float;
|
||||
isSigned = type == nir_type_int;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
isFloat = false;
|
||||
isSigned = false;
|
||||
|
|
@ -564,30 +554,10 @@ Converter::getOperation(nir_intrinsic_op op)
|
|||
return OP_EMIT;
|
||||
case nir_intrinsic_end_primitive:
|
||||
return OP_RESTART;
|
||||
case nir_intrinsic_bindless_image_atomic_add:
|
||||
case nir_intrinsic_image_atomic_add:
|
||||
case nir_intrinsic_bindless_image_atomic_and:
|
||||
case nir_intrinsic_image_atomic_and:
|
||||
case nir_intrinsic_bindless_image_atomic_comp_swap:
|
||||
case nir_intrinsic_image_atomic_comp_swap:
|
||||
case nir_intrinsic_bindless_image_atomic_exchange:
|
||||
case nir_intrinsic_image_atomic_exchange:
|
||||
case nir_intrinsic_bindless_image_atomic_imax:
|
||||
case nir_intrinsic_image_atomic_imax:
|
||||
case nir_intrinsic_bindless_image_atomic_umax:
|
||||
case nir_intrinsic_image_atomic_umax:
|
||||
case nir_intrinsic_bindless_image_atomic_imin:
|
||||
case nir_intrinsic_image_atomic_imin:
|
||||
case nir_intrinsic_bindless_image_atomic_umin:
|
||||
case nir_intrinsic_image_atomic_umin:
|
||||
case nir_intrinsic_bindless_image_atomic_or:
|
||||
case nir_intrinsic_image_atomic_or:
|
||||
case nir_intrinsic_bindless_image_atomic_xor:
|
||||
case nir_intrinsic_image_atomic_xor:
|
||||
case nir_intrinsic_bindless_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_bindless_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_bindless_image_atomic:
|
||||
case nir_intrinsic_image_atomic:
|
||||
case nir_intrinsic_bindless_image_atomic_swap:
|
||||
case nir_intrinsic_image_atomic_swap:
|
||||
return OP_SUREDP;
|
||||
case nir_intrinsic_bindless_image_load:
|
||||
case nir_intrinsic_image_load:
|
||||
|
|
@ -635,81 +605,44 @@ Converter::getSubOp(nir_op op)
|
|||
}
|
||||
}
|
||||
|
||||
int
|
||||
Converter::getAtomicSubOp(nir_atomic_op op)
|
||||
{
|
||||
switch (op) {
|
||||
case nir_atomic_op_fadd:
|
||||
case nir_atomic_op_iadd:
|
||||
return NV50_IR_SUBOP_ATOM_ADD;
|
||||
case nir_atomic_op_iand:
|
||||
return NV50_IR_SUBOP_ATOM_AND;
|
||||
case nir_atomic_op_cmpxchg:
|
||||
return NV50_IR_SUBOP_ATOM_CAS;
|
||||
case nir_atomic_op_imax:
|
||||
case nir_atomic_op_umax:
|
||||
return NV50_IR_SUBOP_ATOM_MAX;
|
||||
case nir_atomic_op_imin:
|
||||
case nir_atomic_op_umin:
|
||||
return NV50_IR_SUBOP_ATOM_MIN;
|
||||
case nir_atomic_op_xchg:
|
||||
return NV50_IR_SUBOP_ATOM_EXCH;
|
||||
case nir_atomic_op_ior:
|
||||
return NV50_IR_SUBOP_ATOM_OR;
|
||||
case nir_atomic_op_ixor:
|
||||
return NV50_IR_SUBOP_ATOM_XOR;
|
||||
case nir_atomic_op_dec_wrap:
|
||||
return NV50_IR_SUBOP_ATOM_DEC;
|
||||
case nir_atomic_op_inc_wrap:
|
||||
return NV50_IR_SUBOP_ATOM_INC;
|
||||
default:
|
||||
ERROR("couldn't get SubOp for atomic\n");
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
Converter::getSubOp(nir_intrinsic_op op)
|
||||
{
|
||||
switch (op) {
|
||||
case nir_intrinsic_bindless_image_atomic_add:
|
||||
case nir_intrinsic_global_atomic_add:
|
||||
case nir_intrinsic_image_atomic_add:
|
||||
case nir_intrinsic_shared_atomic_add:
|
||||
case nir_intrinsic_ssbo_atomic_add:
|
||||
return NV50_IR_SUBOP_ATOM_ADD;
|
||||
case nir_intrinsic_bindless_image_atomic_fadd:
|
||||
case nir_intrinsic_global_atomic_fadd:
|
||||
case nir_intrinsic_image_atomic_fadd:
|
||||
case nir_intrinsic_shared_atomic_fadd:
|
||||
case nir_intrinsic_ssbo_atomic_fadd:
|
||||
return NV50_IR_SUBOP_ATOM_ADD;
|
||||
case nir_intrinsic_bindless_image_atomic_and:
|
||||
case nir_intrinsic_global_atomic_and:
|
||||
case nir_intrinsic_image_atomic_and:
|
||||
case nir_intrinsic_shared_atomic_and:
|
||||
case nir_intrinsic_ssbo_atomic_and:
|
||||
return NV50_IR_SUBOP_ATOM_AND;
|
||||
case nir_intrinsic_bindless_image_atomic_comp_swap:
|
||||
case nir_intrinsic_global_atomic_comp_swap:
|
||||
case nir_intrinsic_image_atomic_comp_swap:
|
||||
case nir_intrinsic_shared_atomic_comp_swap:
|
||||
case nir_intrinsic_ssbo_atomic_comp_swap:
|
||||
return NV50_IR_SUBOP_ATOM_CAS;
|
||||
case nir_intrinsic_bindless_image_atomic_exchange:
|
||||
case nir_intrinsic_global_atomic_exchange:
|
||||
case nir_intrinsic_image_atomic_exchange:
|
||||
case nir_intrinsic_shared_atomic_exchange:
|
||||
case nir_intrinsic_ssbo_atomic_exchange:
|
||||
return NV50_IR_SUBOP_ATOM_EXCH;
|
||||
case nir_intrinsic_bindless_image_atomic_or:
|
||||
case nir_intrinsic_global_atomic_or:
|
||||
case nir_intrinsic_image_atomic_or:
|
||||
case nir_intrinsic_shared_atomic_or:
|
||||
case nir_intrinsic_ssbo_atomic_or:
|
||||
return NV50_IR_SUBOP_ATOM_OR;
|
||||
case nir_intrinsic_bindless_image_atomic_imax:
|
||||
case nir_intrinsic_bindless_image_atomic_umax:
|
||||
case nir_intrinsic_global_atomic_imax:
|
||||
case nir_intrinsic_global_atomic_umax:
|
||||
case nir_intrinsic_image_atomic_imax:
|
||||
case nir_intrinsic_image_atomic_umax:
|
||||
case nir_intrinsic_shared_atomic_imax:
|
||||
case nir_intrinsic_shared_atomic_umax:
|
||||
case nir_intrinsic_ssbo_atomic_imax:
|
||||
case nir_intrinsic_ssbo_atomic_umax:
|
||||
return NV50_IR_SUBOP_ATOM_MAX;
|
||||
case nir_intrinsic_bindless_image_atomic_imin:
|
||||
case nir_intrinsic_bindless_image_atomic_umin:
|
||||
case nir_intrinsic_global_atomic_imin:
|
||||
case nir_intrinsic_global_atomic_umin:
|
||||
case nir_intrinsic_image_atomic_imin:
|
||||
case nir_intrinsic_image_atomic_umin:
|
||||
case nir_intrinsic_shared_atomic_imin:
|
||||
case nir_intrinsic_shared_atomic_umin:
|
||||
case nir_intrinsic_ssbo_atomic_imin:
|
||||
case nir_intrinsic_ssbo_atomic_umin:
|
||||
return NV50_IR_SUBOP_ATOM_MIN;
|
||||
case nir_intrinsic_bindless_image_atomic_xor:
|
||||
case nir_intrinsic_global_atomic_xor:
|
||||
case nir_intrinsic_image_atomic_xor:
|
||||
case nir_intrinsic_shared_atomic_xor:
|
||||
case nir_intrinsic_ssbo_atomic_xor:
|
||||
return NV50_IR_SUBOP_ATOM_XOR;
|
||||
case nir_intrinsic_bindless_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_image_atomic_inc_wrap:
|
||||
return NV50_IR_SUBOP_ATOM_INC;
|
||||
case nir_intrinsic_bindless_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_image_atomic_dec_wrap:
|
||||
return NV50_IR_SUBOP_ATOM_DEC;
|
||||
|
||||
case nir_intrinsic_memory_barrier:
|
||||
case nir_intrinsic_memory_barrier_buffer:
|
||||
case nir_intrinsic_memory_barrier_image:
|
||||
|
|
@ -2117,40 +2050,22 @@ Converter::visit(nir_intrinsic_instr *insn)
|
|||
info_out->io.globalAccess |= 0x1;
|
||||
break;
|
||||
}
|
||||
case nir_intrinsic_shared_atomic_add:
|
||||
case nir_intrinsic_shared_atomic_fadd:
|
||||
case nir_intrinsic_shared_atomic_and:
|
||||
case nir_intrinsic_shared_atomic_comp_swap:
|
||||
case nir_intrinsic_shared_atomic_exchange:
|
||||
case nir_intrinsic_shared_atomic_or:
|
||||
case nir_intrinsic_shared_atomic_imax:
|
||||
case nir_intrinsic_shared_atomic_imin:
|
||||
case nir_intrinsic_shared_atomic_umax:
|
||||
case nir_intrinsic_shared_atomic_umin:
|
||||
case nir_intrinsic_shared_atomic_xor: {
|
||||
case nir_intrinsic_shared_atomic:
|
||||
case nir_intrinsic_shared_atomic_swap: {
|
||||
const DataType dType = getDType(insn);
|
||||
LValues &newDefs = convert(&insn->dest);
|
||||
Value *indirectOffset;
|
||||
uint32_t offset = getIndirect(&insn->src[0], 0, indirectOffset);
|
||||
Symbol *sym = mkSymbol(FILE_MEMORY_SHARED, 0, dType, offset);
|
||||
Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[1], 0));
|
||||
if (op == nir_intrinsic_shared_atomic_comp_swap)
|
||||
if (op == nir_intrinsic_shared_atomic_swap)
|
||||
atom->setSrc(2, getSrc(&insn->src[2], 0));
|
||||
atom->setIndirect(0, 0, indirectOffset);
|
||||
atom->subOp = getSubOp(op);
|
||||
atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
|
||||
break;
|
||||
}
|
||||
case nir_intrinsic_ssbo_atomic_add:
|
||||
case nir_intrinsic_ssbo_atomic_fadd:
|
||||
case nir_intrinsic_ssbo_atomic_and:
|
||||
case nir_intrinsic_ssbo_atomic_comp_swap:
|
||||
case nir_intrinsic_ssbo_atomic_exchange:
|
||||
case nir_intrinsic_ssbo_atomic_or:
|
||||
case nir_intrinsic_ssbo_atomic_imax:
|
||||
case nir_intrinsic_ssbo_atomic_imin:
|
||||
case nir_intrinsic_ssbo_atomic_umax:
|
||||
case nir_intrinsic_ssbo_atomic_umin:
|
||||
case nir_intrinsic_ssbo_atomic_xor: {
|
||||
case nir_intrinsic_ssbo_atomic:
|
||||
case nir_intrinsic_ssbo_atomic_swap: {
|
||||
const DataType dType = getDType(insn);
|
||||
LValues &newDefs = convert(&insn->dest);
|
||||
Value *indirectBuffer;
|
||||
|
|
@ -2161,26 +2076,17 @@ Converter::visit(nir_intrinsic_instr *insn)
|
|||
Symbol *sym = mkSymbol(FILE_MEMORY_BUFFER, buffer, dType, offset);
|
||||
Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym,
|
||||
getSrc(&insn->src[2], 0));
|
||||
if (op == nir_intrinsic_ssbo_atomic_comp_swap)
|
||||
if (op == nir_intrinsic_ssbo_atomic_swap)
|
||||
atom->setSrc(2, getSrc(&insn->src[3], 0));
|
||||
atom->setIndirect(0, 0, indirectOffset);
|
||||
atom->setIndirect(0, 1, indirectBuffer);
|
||||
atom->subOp = getSubOp(op);
|
||||
atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
|
||||
|
||||
info_out->io.globalAccess |= 0x2;
|
||||
break;
|
||||
}
|
||||
case nir_intrinsic_global_atomic_add:
|
||||
case nir_intrinsic_global_atomic_fadd:
|
||||
case nir_intrinsic_global_atomic_and:
|
||||
case nir_intrinsic_global_atomic_comp_swap:
|
||||
case nir_intrinsic_global_atomic_exchange:
|
||||
case nir_intrinsic_global_atomic_or:
|
||||
case nir_intrinsic_global_atomic_imax:
|
||||
case nir_intrinsic_global_atomic_imin:
|
||||
case nir_intrinsic_global_atomic_umax:
|
||||
case nir_intrinsic_global_atomic_umin:
|
||||
case nir_intrinsic_global_atomic_xor: {
|
||||
case nir_intrinsic_global_atomic:
|
||||
case nir_intrinsic_global_atomic_swap: {
|
||||
const DataType dType = getDType(insn);
|
||||
LValues &newDefs = convert(&insn->dest);
|
||||
Value *address;
|
||||
|
|
@ -2189,44 +2095,22 @@ Converter::visit(nir_intrinsic_instr *insn)
|
|||
Symbol *sym = mkSymbol(FILE_MEMORY_GLOBAL, 0, dType, offset);
|
||||
Instruction *atom =
|
||||
mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[1], 0));
|
||||
if (op == nir_intrinsic_global_atomic_comp_swap)
|
||||
if (op == nir_intrinsic_global_atomic_swap)
|
||||
atom->setSrc(2, getSrc(&insn->src[2], 0));
|
||||
atom->setIndirect(0, 0, address);
|
||||
atom->subOp = getSubOp(op);
|
||||
atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
|
||||
|
||||
info_out->io.globalAccess |= 0x2;
|
||||
break;
|
||||
}
|
||||
case nir_intrinsic_bindless_image_atomic_add:
|
||||
case nir_intrinsic_bindless_image_atomic_fadd:
|
||||
case nir_intrinsic_bindless_image_atomic_and:
|
||||
case nir_intrinsic_bindless_image_atomic_comp_swap:
|
||||
case nir_intrinsic_bindless_image_atomic_exchange:
|
||||
case nir_intrinsic_bindless_image_atomic_imax:
|
||||
case nir_intrinsic_bindless_image_atomic_umax:
|
||||
case nir_intrinsic_bindless_image_atomic_imin:
|
||||
case nir_intrinsic_bindless_image_atomic_umin:
|
||||
case nir_intrinsic_bindless_image_atomic_or:
|
||||
case nir_intrinsic_bindless_image_atomic_xor:
|
||||
case nir_intrinsic_bindless_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_bindless_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_bindless_image_atomic:
|
||||
case nir_intrinsic_bindless_image_atomic_swap:
|
||||
case nir_intrinsic_bindless_image_load:
|
||||
case nir_intrinsic_bindless_image_samples:
|
||||
case nir_intrinsic_bindless_image_size:
|
||||
case nir_intrinsic_bindless_image_store:
|
||||
case nir_intrinsic_image_atomic_add:
|
||||
case nir_intrinsic_image_atomic_fadd:
|
||||
case nir_intrinsic_image_atomic_and:
|
||||
case nir_intrinsic_image_atomic_comp_swap:
|
||||
case nir_intrinsic_image_atomic_exchange:
|
||||
case nir_intrinsic_image_atomic_imax:
|
||||
case nir_intrinsic_image_atomic_umax:
|
||||
case nir_intrinsic_image_atomic_imin:
|
||||
case nir_intrinsic_image_atomic_umin:
|
||||
case nir_intrinsic_image_atomic_or:
|
||||
case nir_intrinsic_image_atomic_xor:
|
||||
case nir_intrinsic_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_image_atomic:
|
||||
case nir_intrinsic_image_atomic_swap:
|
||||
case nir_intrinsic_image_load:
|
||||
case nir_intrinsic_image_samples:
|
||||
case nir_intrinsic_image_size:
|
||||
|
|
@ -2234,6 +2118,7 @@ Converter::visit(nir_intrinsic_instr *insn)
|
|||
std::vector<Value*> srcs, defs;
|
||||
Value *indirect;
|
||||
DataType ty;
|
||||
int subOp = 0;
|
||||
|
||||
uint32_t mask = 0;
|
||||
TexInstruction::Target target =
|
||||
|
|
@ -2252,41 +2137,21 @@ Converter::visit(nir_intrinsic_instr *insn)
|
|||
int lod_src = -1;
|
||||
bool bindless = false;
|
||||
switch (op) {
|
||||
case nir_intrinsic_bindless_image_atomic_add:
|
||||
case nir_intrinsic_bindless_image_atomic_fadd:
|
||||
case nir_intrinsic_bindless_image_atomic_and:
|
||||
case nir_intrinsic_bindless_image_atomic_comp_swap:
|
||||
case nir_intrinsic_bindless_image_atomic_exchange:
|
||||
case nir_intrinsic_bindless_image_atomic_imax:
|
||||
case nir_intrinsic_bindless_image_atomic_umax:
|
||||
case nir_intrinsic_bindless_image_atomic_imin:
|
||||
case nir_intrinsic_bindless_image_atomic_umin:
|
||||
case nir_intrinsic_bindless_image_atomic_or:
|
||||
case nir_intrinsic_bindless_image_atomic_xor:
|
||||
case nir_intrinsic_bindless_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_bindless_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_bindless_image_atomic:
|
||||
case nir_intrinsic_bindless_image_atomic_swap:
|
||||
ty = getDType(insn);
|
||||
bindless = true;
|
||||
info_out->io.globalAccess |= 0x2;
|
||||
mask = 0x1;
|
||||
subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
|
||||
break;
|
||||
case nir_intrinsic_image_atomic_add:
|
||||
case nir_intrinsic_image_atomic_fadd:
|
||||
case nir_intrinsic_image_atomic_and:
|
||||
case nir_intrinsic_image_atomic_comp_swap:
|
||||
case nir_intrinsic_image_atomic_exchange:
|
||||
case nir_intrinsic_image_atomic_imax:
|
||||
case nir_intrinsic_image_atomic_umax:
|
||||
case nir_intrinsic_image_atomic_imin:
|
||||
case nir_intrinsic_image_atomic_umin:
|
||||
case nir_intrinsic_image_atomic_or:
|
||||
case nir_intrinsic_image_atomic_xor:
|
||||
case nir_intrinsic_image_atomic_inc_wrap:
|
||||
case nir_intrinsic_image_atomic_dec_wrap:
|
||||
case nir_intrinsic_image_atomic:
|
||||
case nir_intrinsic_image_atomic_swap:
|
||||
ty = getDType(insn);
|
||||
bindless = false;
|
||||
info_out->io.globalAccess |= 0x2;
|
||||
mask = 0x1;
|
||||
subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn));
|
||||
break;
|
||||
case nir_intrinsic_bindless_image_load:
|
||||
case nir_intrinsic_image_load:
|
||||
|
|
@ -2361,7 +2226,7 @@ Converter::visit(nir_intrinsic_instr *insn)
|
|||
texi->tex.mask = mask;
|
||||
texi->cache = convert(nir_intrinsic_access(insn));
|
||||
texi->setType(ty);
|
||||
texi->subOp = getSubOp(op);
|
||||
texi->subOp = subOp;
|
||||
|
||||
if (indirect)
|
||||
texi->setIndirectR(indirect);
|
||||
|
|
@ -3341,6 +3206,7 @@ Converter::run()
|
|||
type_size, (nir_lower_io_options)0);
|
||||
|
||||
NIR_PASS_V(nir, nir_lower_subgroups, &subgroup_options);
|
||||
NIR_PASS_V(nir, nir_lower_legacy_atomics);
|
||||
|
||||
struct nir_lower_tex_options tex_options = {};
|
||||
tex_options.lower_txp = ~0;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue