diff --git a/src/nouveau/codegen/nv50_ir_from_nir.cpp b/src/nouveau/codegen/nv50_ir_from_nir.cpp index 7b6392032e7..3ab457a651c 100644 --- a/src/nouveau/codegen/nv50_ir_from_nir.cpp +++ b/src/nouveau/codegen/nv50_ir_from_nir.cpp @@ -141,6 +141,7 @@ private: int getSubOp(nir_intrinsic_op); int getSubOp(nir_op); + int getAtomicSubOp(nir_atomic_op); CondCode getCondCode(nir_op); @@ -268,27 +269,16 @@ Converter::getDType(nir_intrinsic_instr *insn) { bool isFloat, isSigned; switch (insn->intrinsic) { - case nir_intrinsic_bindless_image_atomic_fadd: - case nir_intrinsic_global_atomic_fadd: - case nir_intrinsic_image_atomic_fadd: - case nir_intrinsic_shared_atomic_fadd: - case nir_intrinsic_ssbo_atomic_fadd: - isFloat = true; - isSigned = false; - break; - case nir_intrinsic_bindless_image_atomic_imax: - case nir_intrinsic_bindless_image_atomic_imin: - case nir_intrinsic_global_atomic_imax: - case nir_intrinsic_global_atomic_imin: - case nir_intrinsic_image_atomic_imax: - case nir_intrinsic_image_atomic_imin: - case nir_intrinsic_shared_atomic_imax: - case nir_intrinsic_shared_atomic_imin: - case nir_intrinsic_ssbo_atomic_imax: - case nir_intrinsic_ssbo_atomic_imin: - isFloat = false; - isSigned = true; + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_global_atomic: + case nir_intrinsic_image_atomic: + case nir_intrinsic_shared_atomic: + case nir_intrinsic_ssbo_atomic: { + nir_alu_type type = nir_atomic_op_type(nir_intrinsic_atomic_op(insn)); + isFloat = type == nir_type_float; + isSigned = type == nir_type_int; break; + } default: isFloat = false; isSigned = false; @@ -564,30 +554,10 @@ Converter::getOperation(nir_intrinsic_op op) return OP_EMIT; case nir_intrinsic_end_primitive: return OP_RESTART; - case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_and: - case nir_intrinsic_image_atomic_and: - case nir_intrinsic_bindless_image_atomic_comp_swap: - case nir_intrinsic_image_atomic_comp_swap: - case nir_intrinsic_bindless_image_atomic_exchange: - case nir_intrinsic_image_atomic_exchange: - case nir_intrinsic_bindless_image_atomic_imax: - case nir_intrinsic_image_atomic_imax: - case nir_intrinsic_bindless_image_atomic_umax: - case nir_intrinsic_image_atomic_umax: - case nir_intrinsic_bindless_image_atomic_imin: - case nir_intrinsic_image_atomic_imin: - case nir_intrinsic_bindless_image_atomic_umin: - case nir_intrinsic_image_atomic_umin: - case nir_intrinsic_bindless_image_atomic_or: - case nir_intrinsic_image_atomic_or: - case nir_intrinsic_bindless_image_atomic_xor: - case nir_intrinsic_image_atomic_xor: - case nir_intrinsic_bindless_image_atomic_inc_wrap: - case nir_intrinsic_image_atomic_inc_wrap: - case nir_intrinsic_bindless_image_atomic_dec_wrap: - case nir_intrinsic_image_atomic_dec_wrap: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: + case nir_intrinsic_image_atomic_swap: return OP_SUREDP; case nir_intrinsic_bindless_image_load: case nir_intrinsic_image_load: @@ -635,81 +605,44 @@ Converter::getSubOp(nir_op op) } } +int +Converter::getAtomicSubOp(nir_atomic_op op) +{ + switch (op) { + case nir_atomic_op_fadd: + case nir_atomic_op_iadd: + return NV50_IR_SUBOP_ATOM_ADD; + case nir_atomic_op_iand: + return NV50_IR_SUBOP_ATOM_AND; + case nir_atomic_op_cmpxchg: + return NV50_IR_SUBOP_ATOM_CAS; + case nir_atomic_op_imax: + case nir_atomic_op_umax: + return NV50_IR_SUBOP_ATOM_MAX; + case nir_atomic_op_imin: + case nir_atomic_op_umin: + return NV50_IR_SUBOP_ATOM_MIN; + case nir_atomic_op_xchg: + return NV50_IR_SUBOP_ATOM_EXCH; + case nir_atomic_op_ior: + return NV50_IR_SUBOP_ATOM_OR; + case nir_atomic_op_ixor: + return NV50_IR_SUBOP_ATOM_XOR; + case nir_atomic_op_dec_wrap: + return NV50_IR_SUBOP_ATOM_DEC; + case nir_atomic_op_inc_wrap: + return NV50_IR_SUBOP_ATOM_INC; + default: + ERROR("couldn't get SubOp for atomic\n"); + assert(false); + return 0; + } +} + int Converter::getSubOp(nir_intrinsic_op op) { switch (op) { - case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_global_atomic_add: - case nir_intrinsic_image_atomic_add: - case nir_intrinsic_shared_atomic_add: - case nir_intrinsic_ssbo_atomic_add: - return NV50_IR_SUBOP_ATOM_ADD; - case nir_intrinsic_bindless_image_atomic_fadd: - case nir_intrinsic_global_atomic_fadd: - case nir_intrinsic_image_atomic_fadd: - case nir_intrinsic_shared_atomic_fadd: - case nir_intrinsic_ssbo_atomic_fadd: - return NV50_IR_SUBOP_ATOM_ADD; - case nir_intrinsic_bindless_image_atomic_and: - case nir_intrinsic_global_atomic_and: - case nir_intrinsic_image_atomic_and: - case nir_intrinsic_shared_atomic_and: - case nir_intrinsic_ssbo_atomic_and: - return NV50_IR_SUBOP_ATOM_AND; - case nir_intrinsic_bindless_image_atomic_comp_swap: - case nir_intrinsic_global_atomic_comp_swap: - case nir_intrinsic_image_atomic_comp_swap: - case nir_intrinsic_shared_atomic_comp_swap: - case nir_intrinsic_ssbo_atomic_comp_swap: - return NV50_IR_SUBOP_ATOM_CAS; - case nir_intrinsic_bindless_image_atomic_exchange: - case nir_intrinsic_global_atomic_exchange: - case nir_intrinsic_image_atomic_exchange: - case nir_intrinsic_shared_atomic_exchange: - case nir_intrinsic_ssbo_atomic_exchange: - return NV50_IR_SUBOP_ATOM_EXCH; - case nir_intrinsic_bindless_image_atomic_or: - case nir_intrinsic_global_atomic_or: - case nir_intrinsic_image_atomic_or: - case nir_intrinsic_shared_atomic_or: - case nir_intrinsic_ssbo_atomic_or: - return NV50_IR_SUBOP_ATOM_OR; - case nir_intrinsic_bindless_image_atomic_imax: - case nir_intrinsic_bindless_image_atomic_umax: - case nir_intrinsic_global_atomic_imax: - case nir_intrinsic_global_atomic_umax: - case nir_intrinsic_image_atomic_imax: - case nir_intrinsic_image_atomic_umax: - case nir_intrinsic_shared_atomic_imax: - case nir_intrinsic_shared_atomic_umax: - case nir_intrinsic_ssbo_atomic_imax: - case nir_intrinsic_ssbo_atomic_umax: - return NV50_IR_SUBOP_ATOM_MAX; - case nir_intrinsic_bindless_image_atomic_imin: - case nir_intrinsic_bindless_image_atomic_umin: - case nir_intrinsic_global_atomic_imin: - case nir_intrinsic_global_atomic_umin: - case nir_intrinsic_image_atomic_imin: - case nir_intrinsic_image_atomic_umin: - case nir_intrinsic_shared_atomic_imin: - case nir_intrinsic_shared_atomic_umin: - case nir_intrinsic_ssbo_atomic_imin: - case nir_intrinsic_ssbo_atomic_umin: - return NV50_IR_SUBOP_ATOM_MIN; - case nir_intrinsic_bindless_image_atomic_xor: - case nir_intrinsic_global_atomic_xor: - case nir_intrinsic_image_atomic_xor: - case nir_intrinsic_shared_atomic_xor: - case nir_intrinsic_ssbo_atomic_xor: - return NV50_IR_SUBOP_ATOM_XOR; - case nir_intrinsic_bindless_image_atomic_inc_wrap: - case nir_intrinsic_image_atomic_inc_wrap: - return NV50_IR_SUBOP_ATOM_INC; - case nir_intrinsic_bindless_image_atomic_dec_wrap: - case nir_intrinsic_image_atomic_dec_wrap: - return NV50_IR_SUBOP_ATOM_DEC; - case nir_intrinsic_memory_barrier: case nir_intrinsic_memory_barrier_buffer: case nir_intrinsic_memory_barrier_image: @@ -2117,40 +2050,22 @@ Converter::visit(nir_intrinsic_instr *insn) info_out->io.globalAccess |= 0x1; break; } - case nir_intrinsic_shared_atomic_add: - case nir_intrinsic_shared_atomic_fadd: - case nir_intrinsic_shared_atomic_and: - case nir_intrinsic_shared_atomic_comp_swap: - case nir_intrinsic_shared_atomic_exchange: - case nir_intrinsic_shared_atomic_or: - case nir_intrinsic_shared_atomic_imax: - case nir_intrinsic_shared_atomic_imin: - case nir_intrinsic_shared_atomic_umax: - case nir_intrinsic_shared_atomic_umin: - case nir_intrinsic_shared_atomic_xor: { + case nir_intrinsic_shared_atomic: + case nir_intrinsic_shared_atomic_swap: { const DataType dType = getDType(insn); LValues &newDefs = convert(&insn->dest); Value *indirectOffset; uint32_t offset = getIndirect(&insn->src[0], 0, indirectOffset); Symbol *sym = mkSymbol(FILE_MEMORY_SHARED, 0, dType, offset); Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[1], 0)); - if (op == nir_intrinsic_shared_atomic_comp_swap) + if (op == nir_intrinsic_shared_atomic_swap) atom->setSrc(2, getSrc(&insn->src[2], 0)); atom->setIndirect(0, 0, indirectOffset); - atom->subOp = getSubOp(op); + atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn)); break; } - case nir_intrinsic_ssbo_atomic_add: - case nir_intrinsic_ssbo_atomic_fadd: - case nir_intrinsic_ssbo_atomic_and: - case nir_intrinsic_ssbo_atomic_comp_swap: - case nir_intrinsic_ssbo_atomic_exchange: - case nir_intrinsic_ssbo_atomic_or: - case nir_intrinsic_ssbo_atomic_imax: - case nir_intrinsic_ssbo_atomic_imin: - case nir_intrinsic_ssbo_atomic_umax: - case nir_intrinsic_ssbo_atomic_umin: - case nir_intrinsic_ssbo_atomic_xor: { + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: { const DataType dType = getDType(insn); LValues &newDefs = convert(&insn->dest); Value *indirectBuffer; @@ -2161,26 +2076,17 @@ Converter::visit(nir_intrinsic_instr *insn) Symbol *sym = mkSymbol(FILE_MEMORY_BUFFER, buffer, dType, offset); Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[2], 0)); - if (op == nir_intrinsic_ssbo_atomic_comp_swap) + if (op == nir_intrinsic_ssbo_atomic_swap) atom->setSrc(2, getSrc(&insn->src[3], 0)); atom->setIndirect(0, 0, indirectOffset); atom->setIndirect(0, 1, indirectBuffer); - atom->subOp = getSubOp(op); + atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn)); info_out->io.globalAccess |= 0x2; break; } - case nir_intrinsic_global_atomic_add: - case nir_intrinsic_global_atomic_fadd: - case nir_intrinsic_global_atomic_and: - case nir_intrinsic_global_atomic_comp_swap: - case nir_intrinsic_global_atomic_exchange: - case nir_intrinsic_global_atomic_or: - case nir_intrinsic_global_atomic_imax: - case nir_intrinsic_global_atomic_imin: - case nir_intrinsic_global_atomic_umax: - case nir_intrinsic_global_atomic_umin: - case nir_intrinsic_global_atomic_xor: { + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: { const DataType dType = getDType(insn); LValues &newDefs = convert(&insn->dest); Value *address; @@ -2189,44 +2095,22 @@ Converter::visit(nir_intrinsic_instr *insn) Symbol *sym = mkSymbol(FILE_MEMORY_GLOBAL, 0, dType, offset); Instruction *atom = mkOp2(OP_ATOM, dType, newDefs[0], sym, getSrc(&insn->src[1], 0)); - if (op == nir_intrinsic_global_atomic_comp_swap) + if (op == nir_intrinsic_global_atomic_swap) atom->setSrc(2, getSrc(&insn->src[2], 0)); atom->setIndirect(0, 0, address); - atom->subOp = getSubOp(op); + atom->subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn)); info_out->io.globalAccess |= 0x2; break; } - case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_fadd: - case nir_intrinsic_bindless_image_atomic_and: - case nir_intrinsic_bindless_image_atomic_comp_swap: - case nir_intrinsic_bindless_image_atomic_exchange: - case nir_intrinsic_bindless_image_atomic_imax: - case nir_intrinsic_bindless_image_atomic_umax: - case nir_intrinsic_bindless_image_atomic_imin: - case nir_intrinsic_bindless_image_atomic_umin: - case nir_intrinsic_bindless_image_atomic_or: - case nir_intrinsic_bindless_image_atomic_xor: - case nir_intrinsic_bindless_image_atomic_inc_wrap: - case nir_intrinsic_bindless_image_atomic_dec_wrap: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_samples: case nir_intrinsic_bindless_image_size: case nir_intrinsic_bindless_image_store: - case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_fadd: - case nir_intrinsic_image_atomic_and: - case nir_intrinsic_image_atomic_comp_swap: - case nir_intrinsic_image_atomic_exchange: - case nir_intrinsic_image_atomic_imax: - case nir_intrinsic_image_atomic_umax: - case nir_intrinsic_image_atomic_imin: - case nir_intrinsic_image_atomic_umin: - case nir_intrinsic_image_atomic_or: - case nir_intrinsic_image_atomic_xor: - case nir_intrinsic_image_atomic_inc_wrap: - case nir_intrinsic_image_atomic_dec_wrap: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: case nir_intrinsic_image_load: case nir_intrinsic_image_samples: case nir_intrinsic_image_size: @@ -2234,6 +2118,7 @@ Converter::visit(nir_intrinsic_instr *insn) std::vector srcs, defs; Value *indirect; DataType ty; + int subOp = 0; uint32_t mask = 0; TexInstruction::Target target = @@ -2252,41 +2137,21 @@ Converter::visit(nir_intrinsic_instr *insn) int lod_src = -1; bool bindless = false; switch (op) { - case nir_intrinsic_bindless_image_atomic_add: - case nir_intrinsic_bindless_image_atomic_fadd: - case nir_intrinsic_bindless_image_atomic_and: - case nir_intrinsic_bindless_image_atomic_comp_swap: - case nir_intrinsic_bindless_image_atomic_exchange: - case nir_intrinsic_bindless_image_atomic_imax: - case nir_intrinsic_bindless_image_atomic_umax: - case nir_intrinsic_bindless_image_atomic_imin: - case nir_intrinsic_bindless_image_atomic_umin: - case nir_intrinsic_bindless_image_atomic_or: - case nir_intrinsic_bindless_image_atomic_xor: - case nir_intrinsic_bindless_image_atomic_inc_wrap: - case nir_intrinsic_bindless_image_atomic_dec_wrap: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: ty = getDType(insn); bindless = true; info_out->io.globalAccess |= 0x2; mask = 0x1; + subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn)); break; - case nir_intrinsic_image_atomic_add: - case nir_intrinsic_image_atomic_fadd: - case nir_intrinsic_image_atomic_and: - case nir_intrinsic_image_atomic_comp_swap: - case nir_intrinsic_image_atomic_exchange: - case nir_intrinsic_image_atomic_imax: - case nir_intrinsic_image_atomic_umax: - case nir_intrinsic_image_atomic_imin: - case nir_intrinsic_image_atomic_umin: - case nir_intrinsic_image_atomic_or: - case nir_intrinsic_image_atomic_xor: - case nir_intrinsic_image_atomic_inc_wrap: - case nir_intrinsic_image_atomic_dec_wrap: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: ty = getDType(insn); bindless = false; info_out->io.globalAccess |= 0x2; mask = 0x1; + subOp = getAtomicSubOp(nir_intrinsic_atomic_op(insn)); break; case nir_intrinsic_bindless_image_load: case nir_intrinsic_image_load: @@ -2361,7 +2226,7 @@ Converter::visit(nir_intrinsic_instr *insn) texi->tex.mask = mask; texi->cache = convert(nir_intrinsic_access(insn)); texi->setType(ty); - texi->subOp = getSubOp(op); + texi->subOp = subOp; if (indirect) texi->setIndirectR(indirect); @@ -3341,6 +3206,7 @@ Converter::run() type_size, (nir_lower_io_options)0); NIR_PASS_V(nir, nir_lower_subgroups, &subgroup_options); + NIR_PASS_V(nir, nir_lower_legacy_atomics); struct nir_lower_tex_options tex_options = {}; tex_options.lower_txp = ~0;