nak: Implement nir_op_[iu]mul[_high]

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24998>

src/nouveau/compiler/nak_encode_sm75.rs

@@ -542,6 +542,29 @@ impl SM75Instr {
         );
     }
 
+    fn encode_imad(&mut self, op: &OpIMad) {
+        self.encode_alu(
+            0x024,
+            Some(op.dst),
+            ALUSrc::from_src(&op.srcs[0]),
+            ALUSrc::from_src(&op.srcs[1]),
+            ALUSrc::from_src(&op.srcs[2]),
+        );
+        self.set_bit(73, op.signed);
+    }
+
+    fn encode_imad64(&mut self, op: &OpIMad64) {
+        self.encode_alu(
+            0x025,
+            Some(op.dst),
+            ALUSrc::from_src(&op.srcs[0]),
+            ALUSrc::from_src(&op.srcs[1]),
+            ALUSrc::from_src(&op.srcs[2]),
+        );
+        self.set_field(81..84, 0x7_u8); /* TODO: Pred? */
+        self.set_bit(73, op.signed);
+    }
+
     fn encode_imnmx(&mut self, op: &OpIMnMx) {
         self.encode_alu(
             0x017,
@@ -897,6 +920,8 @@ impl SM75Instr {
             Op::FSetP(op) => si.encode_fsetp(&op),
             Op::MuFu(op) => si.encode_mufu(&op),
             Op::IAdd3(op) => si.encode_iadd3(&op),
+            Op::IMad(op) => si.encode_imad(&op),
+            Op::IMad64(op) => si.encode_imad64(&op),
             Op::IMnMx(op) => si.encode_imnmx(&op),
             Op::ISetP(op) => si.encode_isetp(&op),
             Op::Lop3(op) => si.encode_lop3(&op),

src/nouveau/compiler/nak_from_nir.rs

@@ -424,6 +424,23 @@ impl<'a> ShaderFromNir<'a> {
                     min: min.into(),
                 })));
             }
+            nir_op_imul => {
+                self.instrs.push(Instr::new(Op::IMad(OpIMad {
+                    dst: dst,
+                    srcs: [srcs[0], srcs[1], Src::new_zero()],
+                    signed: false,
+                })));
+            }
+            nir_op_imul_high | nir_op_umul_high => {
+                let dst64 = self.alloc_ssa(RegFile::GPR, 2);
+                self.instrs.push(Instr::new(Op::IMad64(OpIMad64 {
+                    dst: dst64.into(),
+                    srcs: [srcs[0], srcs[1], Src::new_zero()],
+                    signed: alu.op == nir_op_imul_high,
+                })));
+                self.instrs
+                    .push(Instr::new_split(&[Dst::None, dst], dst64.into()));
+            }
             nir_op_ineg => {
                 self.instrs.push(Instr::new(Op::IMov(OpIMov {
                     dst: dst,

src/nouveau/compiler/nak_ir.rs

@@ -1270,6 +1270,42 @@ impl fmt::Display for OpIAdd3 {
     }
 }
 
+#[repr(C)]
+#[derive(SrcsAsSlice, DstsAsSlice)]
+pub struct OpIMad {
+    pub dst: Dst,
+    pub srcs: [Src; 3],
+    pub signed: bool,
+}
+
+impl fmt::Display for OpIMad {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "IMAD {} {{ {}, {}, {} }}",
+            self.dst, self.srcs[0], self.srcs[1], self.srcs[2],
+        )
+    }
+}
+
+#[repr(C)]
+#[derive(SrcsAsSlice, DstsAsSlice)]
+pub struct OpIMad64 {
+    pub dst: Dst,
+    pub srcs: [Src; 3],
+    pub signed: bool,
+}
+
+impl fmt::Display for OpIMad64 {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "IMAD64 {} {{ {}, {}, {} }}",
+            self.dst, self.srcs[0], self.srcs[1], self.srcs[2],
+        )
+    }
+}
+
 #[repr(C)]
 #[derive(SrcsAsSlice, DstsAsSlice)]
 pub struct OpIMnMx {
@@ -1906,6 +1942,8 @@ pub enum Op {
     FSet(OpFSet),
     FSetP(OpFSetP),
     IAdd3(OpIAdd3),
+    IMad(OpIMad),
+    IMad64(OpIMad64),
     IMnMx(OpIMnMx),
     ISetP(OpISetP),
     Lop3(OpLop3),
@@ -2368,6 +2406,8 @@ impl Instr {
             | Op::FSetP(_)
             | Op::MuFu(_)
             | Op::IAdd3(_)
+            | Op::IMad(_)
+            | Op::IMad64(_)
             | Op::IMnMx(_)
             | Op::Lop3(_)
             | Op::PLop3(_)

src/nouveau/compiler/nak_legalize.rs

@@ -95,6 +95,12 @@ impl<'a> LegalizeInstr<'a> {
                 self.mov_src_if_not_reg(src0, RegFile::GPR);
                 self.mov_src_if_not_reg(src2, RegFile::GPR);
             }
+            Op::IMad(op) => {
+                let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
+                self.swap_srcs_if_not_reg(src0, src1);
+                self.mov_src_if_not_reg(src0, RegFile::GPR);
+                self.mov_src_if_not_reg(src2, RegFile::GPR);
+            }
             Op::Lop3(op) => {
                 /* Fold constants if we can */
                 op.op = LogicOp::new_lut(&|mut x, mut y, mut z| {