nak: Implement image atomics

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24998>

src/nouveau/compiler/nak_encode_sm75.rs

@@ -1035,6 +1035,23 @@ impl SM75Instr {
         self.set_field(72..76, op.mask);
     }
 
+    fn encode_suatom(&mut self, op: &OpSuAtom) {
+        self.set_opcode(0x394);
+
+        self.set_dst(op.dst);
+        self.set_reg_src(24..32, op.coord);
+        self.set_reg_src(32..40, op.data);
+        self.set_reg_src(64..72, op.handle);
+        self.set_pred_dst(81..84, op.resident);
+
+        self.set_image_dim(61..64, op.image_dim);
+        self.set_mem_order_scope(&op.mem_order, &op.mem_scope);
+
+        self.set_bit(72, false); /* .BA */
+        self.set_atom_type(73..76, op.atom_type);
+        self.set_atom_op(87..91, op.atom_op);
+    }
+
     fn set_mem_type(&mut self, range: Range<usize>, mem_type: MemType) {
         assert!(range.len() == 3);
         self.set_field(
@@ -1463,6 +1480,7 @@ impl SM75Instr {
             Op::Txq(op) => si.encode_txq(&op),
             Op::SuLd(op) => si.encode_suld(&op),
             Op::SuSt(op) => si.encode_sust(&op),
+            Op::SuAtom(op) => si.encode_suatom(&op),
             Op::Ld(op) => si.encode_ld(&op),
             Op::St(op) => si.encode_st(&op),
             Op::Atom(op) => si.encode_atom(&op),

src/nouveau/compiler/nak_from_nir.rs

@@ -860,6 +860,30 @@ impl<'a> ShaderFromNir<'a> {
     fn parse_intrinsic(&mut self, intrin: &nir_intrinsic_instr) {
         let srcs = intrin.srcs_as_slice();
         match intrin.intrinsic {
+            nir_intrinsic_bindless_image_atomic => {
+                let handle = self.get_src(&srcs[0]);
+                let dim = self.get_image_dim(intrin);
+                let coord = self.get_image_coord(intrin, dim);
+                /* let sample = self.get_src(&srcs[2]); */
+                let data = self.get_src(&srcs[3]);
+                let atom_type = self.get_atomic_type(intrin);
+                let atom_op = self.get_atomic_op(intrin);
+                let dst = self.get_dst(&intrin.def);
+
+                let atom = OpSuAtom {
+                    dst: dst,
+                    resident: Dst::None,
+                    handle: handle,
+                    coord: coord,
+                    data: data,
+                    atom_op: atom_op,
+                    atom_type: atom_type,
+                    image_dim: dim,
+                    mem_order: MemOrder::Strong,
+                    mem_scope: MemScope::System,
+                };
+                self.instrs.push(atom.into());
+            }
             nir_intrinsic_bindless_image_load => {
                 let handle = self.get_src(&srcs[0]);
                 let dim = self.get_image_dim(intrin);

src/nouveau/compiler/nak_ir.rs

@@ -2308,6 +2308,47 @@ impl fmt::Display for OpSuSt {
     }
 }
 
+#[repr(C)]
+#[derive(SrcsAsSlice, DstsAsSlice)]
+pub struct OpSuAtom {
+    pub dst: Dst,
+    pub resident: Dst,
+
+    pub image_dim: ImageDim,
+
+    pub atom_op: AtomOp,
+    pub atom_type: AtomType,
+
+    pub mem_order: MemOrder,
+    pub mem_scope: MemScope,
+
+    #[src_type(GPR)]
+    pub handle: Src,
+
+    #[src_type(SSA)]
+    pub coord: Src,
+
+    #[src_type(SSA)]
+    pub data: Src,
+}
+
+impl fmt::Display for OpSuAtom {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "SUATOM.P.{}.{}.{}.{}.{} [{}] {} {}",
+            self.image_dim,
+            self.atom_op,
+            self.atom_type,
+            self.mem_order,
+            self.mem_scope,
+            self.coord,
+            self.data,
+            self.handle,
+        )
+    }
+}
+
 #[repr(C)]
 #[derive(SrcsAsSlice, DstsAsSlice)]
 pub struct OpLd {
@@ -2923,6 +2964,7 @@ pub enum Op {
     Txq(OpTxq),
     SuLd(OpSuLd),
     SuSt(OpSuSt),
+    SuAtom(OpSuAtom),
     Ld(OpLd),
     St(OpSt),
     Atom(OpAtom),
@@ -3393,6 +3435,7 @@ impl Instr {
         match self.op {
             Op::ASt(_)
             | Op::SuSt(_)
+            | Op::SuAtom(_)
             | Op::St(_)
             | Op::Atom(_)
             | Op::AtomCas(_)
@@ -3438,6 +3481,7 @@ impl Instr {
             Op::Txq(_) => None,
             Op::SuLd(_) => None,
             Op::SuSt(_) => None,
+            Op::SuAtom(_) => None,
             Op::Ld(_) => None,
             Op::St(_) => None,
             Op::Atom(_) => None,