diff --git a/src/nouveau/compiler/nak_calc_instr_deps.rs b/src/nouveau/compiler/nak_calc_instr_deps.rs index a1073d7d33b..94e446bc7c5 100644 --- a/src/nouveau/compiler/nak_calc_instr_deps.rs +++ b/src/nouveau/compiler/nak_calc_instr_deps.rs @@ -38,6 +38,7 @@ impl RegTracker { RegFile::UGPR => &self.ureg[range], RegFile::Pred => &self.pred[range], RegFile::UPred => &self.upred[range], + RegFile::Mem => panic!("Not a register"), } } @@ -53,6 +54,7 @@ impl RegTracker { RegFile::UGPR => &mut self.ureg[range], RegFile::Pred => &mut self.pred[range], RegFile::UPred => &mut self.upred[range], + RegFile::Mem => panic!("Not a register"), } } } diff --git a/src/nouveau/compiler/nak_ir.rs b/src/nouveau/compiler/nak_ir.rs index be5db659f55..6606c5e771d 100644 --- a/src/nouveau/compiler/nak_ir.rs +++ b/src/nouveau/compiler/nak_ir.rs @@ -39,15 +39,21 @@ pub enum RegFile { /// /// Uniform predicate registers are 1 bit and uniform across a wave. UPred = 3, + + /// The memory register file + /// + /// This is a virtual register file for things which will get spilled to + /// local memory. Each memory location is 32 bits per SIMT channel. + Mem = 4, } -const NUM_REG_FILES: usize = 4; +const NUM_REG_FILES: usize = 5; impl RegFile { /// Returns true if the register file is uniform across a wave pub fn is_uniform(&self) -> bool { match self { - RegFile::GPR | RegFile::Pred => false, + RegFile::GPR | RegFile::Pred | RegFile::Mem => false, RegFile::UGPR | RegFile::UPred => true, } } @@ -56,14 +62,14 @@ impl RegFile { pub fn is_gpr(&self) -> bool { match self { RegFile::GPR | RegFile::UGPR => true, - RegFile::Pred | RegFile::UPred => false, + RegFile::Pred | RegFile::UPred | RegFile::Mem => false, } } /// Returns true if the register file is a predicate register file pub fn is_predicate(&self) -> bool { match self { - RegFile::GPR | RegFile::UGPR => false, + RegFile::GPR | RegFile::UGPR | RegFile::Mem => false, RegFile::Pred | RegFile::UPred => true, } } @@ -86,6 +92,7 @@ impl RegFile { 0 } } + RegFile::Mem => 1 << 24, } } } @@ -105,6 +112,7 @@ impl TryFrom for RegFile { 1 => Ok(RegFile::UGPR), 2 => Ok(RegFile::Pred), 3 => Ok(RegFile::UPred), + 4 => Ok(RegFile::Mem), _ => Err("Invalid register file number"), } } @@ -224,6 +232,7 @@ impl PerRegFile { f(RegFile::UGPR), f(RegFile::Pred), f(RegFile::UPred), + f(RegFile::Mem), ], } } @@ -285,17 +294,16 @@ impl SSAValue { /// Returns an SSA value with the given register file and index pub fn new(file: RegFile, idx: u32) -> SSAValue { - /* Reserve 2 numbers for use for SSARef::comps() */ - assert!(idx > 0 && idx < (1 << 30) - 2); + assert!(idx > 0 && idx < (1 << 29) - 2); let mut packed = idx; - assert!(u8::from(file) < 4); - packed |= u32::from(u8::from(file)) << 30; + assert!(u8::from(file) < 8); + packed |= u32::from(u8::from(file)) << 29; SSAValue { packed: packed } } /// Returns the index of this SSA value pub fn idx(&self) -> u32 { - self.packed & 0x3fffffff + self.packed & 0x1fffffff } /// Returns true if this SSA value is equal to SSAValue::NONE @@ -308,7 +316,7 @@ impl SSAValue { impl HasRegFile for SSAValue { /// Returns the register file of this SSA value fn file(&self) -> RegFile { - RegFile::try_from(self.packed >> 30).unwrap() + RegFile::try_from(self.packed >> 29).unwrap() } } @@ -319,6 +327,7 @@ impl fmt::Display for SSAValue { RegFile::UGPR => write!(f, "US")?, RegFile::Pred => write!(f, "PS")?, RegFile::UPred => write!(f, "UPS")?, + RegFile::Mem => write!(f, "MS")?, } write!(f, "{}", self.idx()) } @@ -479,6 +488,7 @@ impl RegRef { RegFile::UGPR => 63, RegFile::Pred => 7, RegFile::UPred => 7, + RegFile::Mem => panic!("Mem has no zero index"), } } @@ -524,6 +534,7 @@ impl fmt::Display for RegRef { RegFile::UGPR => write!(f, "UR")?, RegFile::Pred => write!(f, "P")?, RegFile::UPred => write!(f, "UP")?, + RegFile::Mem => write!(f, "M")?, } write!(f, "{}", self.base_idx())?; if self.comps() >= 1 { diff --git a/src/nouveau/compiler/nak_lower_copy_swap.rs b/src/nouveau/compiler/nak_lower_copy_swap.rs index 52cedcdf29a..9ed1e425ad1 100644 --- a/src/nouveau/compiler/nak_lower_copy_swap.rs +++ b/src/nouveau/compiler/nak_lower_copy_swap.rs @@ -3,11 +3,19 @@ use crate::nak_ir::*; -struct LowerCopySwap {} +use std::cmp::max; + +struct LowerCopySwap { + tls_start: u32, + tls_size: u32, +} impl LowerCopySwap { - fn new() -> Self { - Self {} + fn new(tls_size: u32) -> Self { + Self { + tls_start: tls_size, + tls_size: tls_size, + } } fn lower_copy(&mut self, b: &mut impl Builder, copy: OpCopy) { @@ -35,6 +43,23 @@ impl LowerCopySwap { quad_lanes: 0xf, }); } + RegFile::Mem => { + let access = MemAccess { + addr_type: MemAddrType::A32, + mem_type: MemType::B32, + space: MemSpace::Local, + order: MemOrder::Strong, + scope: MemScope::CTA, + }; + let addr = self.tls_start + src_reg.base_idx() * 4; + self.tls_size = max(self.tls_size, addr + 4); + b.push_op(OpLd { + dst: copy.dst, + addr: Src::new_zero(), + offset: addr.try_into().unwrap(), + access: access, + }); + } _ => panic!("Cannot copy to GPR"), }, SrcRef::SSA(_) => panic!("Should be run after RA"), @@ -72,6 +97,29 @@ impl LowerCopySwap { }, SrcRef::SSA(_) => panic!("Should be run after RA"), }, + RegFile::Mem => match copy.src.src_ref { + SrcRef::Reg(src_reg) => match src_reg.file() { + RegFile::GPR => { + let access = MemAccess { + addr_type: MemAddrType::A32, + mem_type: MemType::B32, + space: MemSpace::Local, + order: MemOrder::Strong, + scope: MemScope::CTA, + }; + let addr = self.tls_start + dst_reg.base_idx() * 4; + self.tls_size = max(self.tls_size, addr + 4); + b.push_op(OpSt { + addr: Src::new_zero(), + data: copy.src, + offset: addr.try_into().unwrap(), + access: access, + }); + } + _ => panic!("Cannot copy to Mem"), + }, + _ => panic!("Cannot copy to Mem"), + }, _ => panic!("Unhandled register file"), } } @@ -81,6 +129,7 @@ impl LowerCopySwap { let y = *swap.dsts[1].as_reg().unwrap(); assert!(x.file() == y.file()); + assert!(x.file() != RegFile::Mem); assert!(x.comps() == 1 && y.comps() == 1); assert!(swap.srcs[0].src_mod.is_none()); assert!(*swap.srcs[0].src_ref.as_reg().unwrap() == y); @@ -129,7 +178,8 @@ impl LowerCopySwap { impl Shader { pub fn lower_copy_swap(&mut self) { - let mut pass = LowerCopySwap::new(); + let mut pass = LowerCopySwap::new(self.tls_size); pass.run(self); + self.tls_size = pass.tls_size; } } diff --git a/src/nouveau/compiler/nak_opt_lop.rs b/src/nouveau/compiler/nak_opt_lop.rs index beee5d8b3e6..c24af8d9572 100644 --- a/src/nouveau/compiler/nak_opt_lop.rs +++ b/src/nouveau/compiler/nak_opt_lop.rs @@ -154,6 +154,7 @@ impl LopPass { srcs[src_idx] = match ssa.file() { RegFile::GPR | RegFile::UGPR => SrcRef::Zero.into(), RegFile::Pred | RegFile::UPred => SrcRef::True.into(), + RegFile::Mem => panic!("Not a register"), }; for i in 0..3 {