diff --git a/src/nouveau/compiler/nak_assign_regs.rs b/src/nouveau/compiler/nak_assign_regs.rs index 831c0e0c19d..2b2aec3fee1 100644 --- a/src/nouveau/compiler/nak_assign_regs.rs +++ b/src/nouveau/compiler/nak_assign_regs.rs @@ -6,7 +6,6 @@ #![allow(unstable_name_collisions)] use crate::bitset::BitSet; -use crate::nak_cfg::CFG; use crate::nak_ir::*; use crate::nak_liveness::{BlockLiveness, Liveness, SimpleLiveness}; use crate::util::NextMultipleOf; @@ -1019,9 +1018,8 @@ impl AssignRegs { assert!(s.functions.len() == 1); let f = &mut s.functions[0]; - let cfg = CFG::for_function(f); - let live = SimpleLiveness::for_function(f, &cfg); - let max_live = live.calc_max_live(f, &cfg); + let live = SimpleLiveness::for_function(f); + let max_live = live.calc_max_live(f); let num_regs = PerRegFile::new_with(|file| { let num_regs = file.num_regs(self.sm); @@ -1041,26 +1039,31 @@ impl AssignRegs { s.num_gprs = num_regs[RegFile::GPR]; - for b in &mut f.blocks { - let bl = live.block_live(b.id); - - let pred = cfg.block_predecessors(b.id); + for b_idx in 0..f.blocks.len() { + let pred = f.blocks.pred_indices(b_idx); let pred_ra = if pred.is_empty() { None } else { /* Start with the previous block's. */ - Some(&self.blocks.get(&pred[0]).unwrap().ra) + let pred_id = f.blocks[pred[0]].id; + Some(&self.blocks.get(&pred_id).unwrap().ra) }; + let b = &f.blocks[b_idx]; + let bl = live.block_live(b.id); + let mut arb = AssignRegsBlock::new(&num_regs); - arb.first_pass(b, bl, pred_ra); - self.blocks.insert(b.id, arb); + arb.first_pass(&mut f.blocks[b_idx], bl, pred_ra); + self.blocks.insert(f.blocks[b_idx].id, arb); } - for b in &mut f.blocks { - let arb = self.blocks.get(&b.id).unwrap(); - for sb_id in cfg.block_successors(b.id) { + for b_idx in 0..f.blocks.len() { + let b_id = f.blocks[b_idx].id; + let arb = self.blocks.get(&b_id).unwrap(); + for sb_idx in f.blocks.succ_indices(b_idx).to_vec() { + let sb_id = f.blocks[sb_idx].id; let target = self.blocks.get(&sb_id).unwrap(); + let b = &mut f.blocks[b_idx]; arb.second_pass(target, b); } } diff --git a/src/nouveau/compiler/nak_cfg.rs b/src/nouveau/compiler/nak_cfg.rs index 4f3b95cc306..f2adfc0f29e 100644 --- a/src/nouveau/compiler/nak_cfg.rs +++ b/src/nouveau/compiler/nak_cfg.rs @@ -4,7 +4,6 @@ */ use crate::bitset::BitSet; -use crate::nak_ir::*; use std::collections::HashMap; use std::hash::Hash; @@ -148,11 +147,11 @@ fn calc_dominance(nodes: &mut Vec>) { } } -pub struct CFG2 { +pub struct CFG { nodes: Vec>, } -impl CFG2 { +impl CFG { pub fn from_blocks_edges( nodes: impl IntoIterator, edges: impl IntoIterator, @@ -172,7 +171,7 @@ impl CFG2 { rev_post_order_sort(&mut nodes); calc_dominance(&mut nodes); - CFG2 { nodes: nodes } + CFG { nodes: nodes } } pub fn get(&self, idx: usize) -> Option<&N> { @@ -204,7 +203,7 @@ impl CFG2 { } } -impl Index for CFG2 { +impl Index for CFG { type Output = N; fn index(&self, idx: usize) -> &N { @@ -212,13 +211,13 @@ impl Index for CFG2 { } } -impl IndexMut for CFG2 { +impl IndexMut for CFG { fn index_mut(&mut self, idx: usize) -> &mut N { &mut self.nodes[idx].node } } -impl<'a, N> IntoIterator for &'a CFG2 { +impl<'a, N> IntoIterator for &'a CFG { type Item = &'a CFGNode; type IntoIter = slice::Iter<'a, CFGNode>; @@ -227,7 +226,7 @@ impl<'a, N> IntoIterator for &'a CFG2 { } } -impl<'a, N> IntoIterator for &'a mut CFG2 { +impl<'a, N> IntoIterator for &'a mut CFG { type Item = &'a mut CFGNode; type IntoIter = slice::IterMut<'a, CFGNode>; @@ -262,13 +261,13 @@ impl CFGBuilder { self.edges.push((s, p)); } - pub fn as_cfg(mut self) -> CFG2 { + pub fn as_cfg(mut self) -> CFG { let edges = self.edges.drain(..).map(|(s, p)| { let s = *self.key_map.get(&s).unwrap(); let p = *self.key_map.get(&p).unwrap(); (s, p) }); - CFG2::from_blocks_edges(self.nodes, edges) + CFG::from_blocks_edges(self.nodes, edges) } } @@ -277,75 +276,3 @@ impl Default for CFGBuilder { CFGBuilder::new() } } - -struct CFGBlock { - pred: Vec, - num_succ: u8, - succ: [u32; 2], -} - -pub struct CFG { - block_map: HashMap, -} - -impl CFG { - fn block_mut(&mut self, id: u32) -> &mut CFGBlock { - self.block_map.entry(id).or_insert_with(|| CFGBlock { - pred: Vec::new(), - num_succ: 0, - succ: [0_u32; 2], - }) - } - - fn block(&self, id: u32) -> &CFGBlock { - self.block_map.get(&id).unwrap() - } - - pub fn block_predecessors(&self, id: u32) -> &[u32] { - &self.block(id).pred - } - - pub fn block_successors(&self, id: u32) -> &[u32] { - let b = self.block(id); - let num_succ = usize::try_from(b.num_succ).unwrap(); - &b.succ[0..num_succ] - } - - pub fn for_function(f: &Function) -> CFG { - let mut cfg = CFG { - block_map: HashMap::new(), - }; - - for (i, bb) in f.blocks.iter().enumerate() { - let mut succ = [0_u32; 2]; - let mut num_succ = 0_usize; - - if bb.falls_through() { - succ[num_succ] = f.blocks[i + 1].id; - num_succ += 1; - } - - if let Some(br) = bb.branch() { - match &br.op { - Op::Bra(bra) => { - succ[num_succ] = bra.target; - num_succ += 1; - } - Op::Exit(_) => (), - _ => panic!("Unhandled branch op"), - } - } - - for si in 0..num_succ { - cfg.block_mut(succ[si]).pred.push(bb.id); - } - - let cb = cfg.block_mut(bb.id); - assert!(cb.num_succ == 0); - cb.num_succ = num_succ.try_into().unwrap(); - cb.succ = succ; - } - - cfg - } -} diff --git a/src/nouveau/compiler/nak_from_nir.rs b/src/nouveau/compiler/nak_from_nir.rs index 6ca23509b36..06bcdc56f1d 100644 --- a/src/nouveau/compiler/nak_from_nir.rs +++ b/src/nouveau/compiler/nak_from_nir.rs @@ -6,6 +6,7 @@ #![allow(non_upper_case_globals)] #![allow(unstable_name_collisions)] +use crate::nak_cfg::CFGBuilder; use crate::nak_ir::*; use crate::nir::*; use crate::util::DivCeil; @@ -32,7 +33,7 @@ fn alloc_ssa_for_nir(b: &mut impl SSABuilder, ssa: &nir_def) -> Vec { struct ShaderFromNir<'a> { nir: &'a nir_shader, - blocks: Vec, + cfg: CFGBuilder, fs_out_regs: Vec, end_block_id: u32, ssa_map: HashMap>, @@ -51,7 +52,7 @@ impl<'a> ShaderFromNir<'a> { Self { nir: nir, - blocks: Vec::new(), + cfg: CFGBuilder::new(), fs_out_regs: fs_out_regs, end_block_id: 0, ssa_map: HashMap::new(), @@ -1622,6 +1623,10 @@ impl<'a> ShaderFromNir<'a> { } if let Some(ni) = nb.following_if() { + /* The fall-through edge has to come first */ + self.cfg.add_edge(nb.index, ni.first_then_block().index); + self.cfg.add_edge(nb.index, ni.first_else_block().index); + let mut bra = Instr::new_boxed(OpBra { target: ni.first_else_block().index, }); @@ -1638,13 +1643,14 @@ impl<'a> ShaderFromNir<'a> { if s0.index == self.end_block_id { b.push_op(OpExit {}); } else { + self.cfg.add_edge(nb.index, s0.index); b.push_op(OpBra { target: s0.index }); } } let mut bb = BasicBlock::new(nb.index); bb.instrs.append(&mut b.as_vec()); - self.blocks.push(bb); + self.cfg.add_node(bb.id, bb); } fn parse_if(&mut self, alloc: &mut SSAValueAllocator, ni: &nir_if) { @@ -1678,24 +1684,35 @@ impl<'a> ShaderFromNir<'a> { } pub fn parse_function_impl(&mut self, nfi: &nir_function_impl) -> Function { - let mut f = Function::new(0); + let mut ssa_alloc = SSAValueAllocator::new(); self.end_block_id = nfi.end_block().index; - self.parse_cf_list(&mut f.ssa_alloc, nfi.iter_body()); + self.parse_cf_list(&mut ssa_alloc, nfi.iter_body()); - let end_block = self.blocks.last_mut().unwrap(); + let mut cfg = std::mem::take(&mut self.cfg).as_cfg(); + assert!(cfg.len() > 0); + for i in 0..cfg.len() { + if cfg[i].falls_through() { + assert!(cfg.succ_indices(i)[0] == i + 1); + } + } if self.nir.info.stage() == MESA_SHADER_FRAGMENT && nfi.function().is_entrypoint { + let end_block_idx = cfg.len() - 1; + let end_block = &mut cfg[end_block_idx]; + let fs_out = Instr::new_boxed(OpFSOut { srcs: std::mem::replace(&mut self.fs_out_regs, Vec::new()), }); end_block.instrs.insert(end_block.instrs.len() - 1, fs_out); } - f.blocks.append(&mut self.blocks); - f + Function { + ssa_alloc: ssa_alloc, + blocks: cfg, + } } pub fn parse_shader(&mut self, sm: u8) -> Shader { diff --git a/src/nouveau/compiler/nak_ir.rs b/src/nouveau/compiler/nak_ir.rs index 3a7e1d966a9..e016144d399 100644 --- a/src/nouveau/compiler/nak_ir.rs +++ b/src/nouveau/compiler/nak_ir.rs @@ -8,6 +8,7 @@ extern crate nak_ir_proc; pub use crate::nak_builder::{ Builder, InstrBuilder, SSABuilder, SSAInstrBuilder, }; +use crate::nak_cfg::CFG; use nak_ir_proc::*; use std::fmt; use std::iter::Zip; @@ -3995,29 +3996,12 @@ impl BasicBlock { } } -impl fmt::Display for BasicBlock { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "block {} {{\n", self.id)?; - for i in &self.instrs { - write!(f, " {}\n", i)?; - } - write!(f, "}}\n") - } -} - pub struct Function { pub ssa_alloc: SSAValueAllocator, - pub blocks: Vec, + pub blocks: CFG, } impl Function { - pub fn new(_id: u32) -> Function { - Function { - ssa_alloc: SSAValueAllocator::new(), - blocks: Vec::new(), - } - } - pub fn map_instrs< F: Fn(Box, &mut SSAValueAllocator) -> MappedInstrs, >( @@ -4032,8 +4016,28 @@ impl Function { impl fmt::Display for Function { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for b in &self.blocks { - write!(f, "{}", b)?; + for i in 0..self.blocks.len() { + write!(f, "block {}(id={}) [", i, self.blocks[i].id)?; + for (pi, p) in self.blocks.pred_indices(i).iter().enumerate() { + if pi > 0 { + write!(f, ", ")?; + } + write!(f, "{}", p)?; + } + write!(f, "] -> {{\n")?; + + for i in &self.blocks[i].instrs { + write!(f, " {}\n", i)?; + } + + write!(f, "}} -> [")?; + for (si, s) in self.blocks.succ_indices(i).iter().enumerate() { + if si > 0 { + write!(f, ", ")?; + } + write!(f, "{}", s)?; + } + write!(f, "]\n")?; } Ok(()) } diff --git a/src/nouveau/compiler/nak_liveness.rs b/src/nouveau/compiler/nak_liveness.rs index 3a30428b4e2..61954a31809 100644 --- a/src/nouveau/compiler/nak_liveness.rs +++ b/src/nouveau/compiler/nak_liveness.rs @@ -4,7 +4,6 @@ */ use crate::bitset::BitSet; -use crate::nak_cfg::CFG; use crate::nak_ir::*; use std::cell::RefCell; @@ -70,11 +69,11 @@ pub trait Liveness { fn block_live(&self, id: u32) -> &Self::PerBlock; - fn calc_max_live(&self, f: &Function, cfg: &CFG) -> PerRegFile { + fn calc_max_live(&self, f: &Function) -> PerRegFile { let mut max_live: PerRegFile = Default::default(); let mut block_live_out: HashMap = HashMap::new(); - for bb in &f.blocks { + for (bb_idx, bb) in f.blocks.iter().enumerate() { let bl = self.block_live(bb.id); let mut live = LiveSet::new(); @@ -82,7 +81,8 @@ pub trait Liveness { /* Predecessors are added block order so we can just grab the first * one (if any) and it will be a block we've processed. */ - if let Some(pred_id) = cfg.block_predecessors(bb.id).first() { + if let Some(pred_idx) = f.blocks.pred_indices(bb_idx).first() { + let pred_id = f.blocks[*pred_idx].id; let pred_out = block_live_out.get(&pred_id).unwrap(); for ssa in pred_out.iter() { if bl.is_live_in(ssa) { @@ -218,13 +218,13 @@ pub struct SimpleLiveness { } impl SimpleLiveness { - pub fn for_function(func: &Function, cfg: &CFG) -> SimpleLiveness { + pub fn for_function(func: &Function) -> SimpleLiveness { let mut l = SimpleLiveness { blocks: HashMap::new(), }; let mut live_in = HashMap::new(); - for b in &func.blocks { + for b in func.blocks.iter() { let bl = SimpleBlockLiveness::for_block(b); l.blocks.insert(b.id, bl); live_in.insert(b.id, BitSet::new()); @@ -233,11 +233,12 @@ impl SimpleLiveness { let mut to_do = true; while to_do { to_do = false; - for b in func.blocks.iter().rev() { + for (b_idx, b) in func.blocks.iter().enumerate().rev() { let bl = l.blocks.get_mut(&b.id).unwrap(); /* Compute live-out */ - for sb_id in cfg.block_successors(b.id) { + for sb_idx in func.blocks.succ_indices(b_idx) { + let sb_id = func.blocks[*sb_idx].id; let s_live_in = live_in.get(&sb_id).unwrap(); to_do |= bl.live_out.union_with(s_live_in); } @@ -380,7 +381,7 @@ pub struct NextUseLiveness { } impl NextUseLiveness { - pub fn for_function(func: &Function, cfg: &CFG) -> NextUseLiveness { + pub fn for_function(func: &Function) -> NextUseLiveness { let mut blocks = HashMap::new(); for b in &func.blocks { let bl = NextUseBlockLiveness::for_block(b); @@ -390,13 +391,14 @@ impl NextUseLiveness { let mut to_do = true; while to_do { to_do = false; - for b in func.blocks.iter().rev() { + for (b_idx, b) in func.blocks.iter().enumerate().rev() { let num_instrs = b.instrs.len(); let mut bl = blocks.get(&b.id).unwrap().borrow_mut(); /* Compute live-out */ - for sb_id in cfg.block_successors(b.id) { - if *sb_id == b.id { + for sb_idx in func.blocks.succ_indices(b_idx) { + let sb_id = func.blocks[*sb_idx].id; + if sb_id == b.id { for entry in bl.ssa_map.values_mut() { if entry.defined { continue;