nak: Use the builder for the legalize pass

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24998>
This commit is contained in:
Faith Ekstrand 2023-08-28 19:40:04 -05:00 committed by Marge Bot
parent 4c798afb25
commit fc35dd6aba
2 changed files with 225 additions and 249 deletions

View file

@ -3759,15 +3759,6 @@ impl Instr {
Box::new(Instr::new(op))
}
pub fn new_mov(dst: Dst, src: Src) -> Instr {
OpMov {
dst: dst,
src: src,
quad_lanes: 0xf,
}
.into()
}
pub fn dsts(&self) -> &[Dst] {
self.op.dsts_as_slice()
}

View file

@ -7,11 +7,6 @@ use crate::nak_ir::*;
use std::collections::{HashMap, HashSet};
struct LegalizeInstr<'a> {
ssa_alloc: &'a mut SSAValueAllocator,
instrs: Vec<Box<Instr>>,
}
fn src_is_reg(src: &Src) -> bool {
match src.src_ref {
SrcRef::Zero | SrcRef::True | SrcRef::False | SrcRef::SSA(_) => true,
@ -45,275 +40,265 @@ fn fold_lop_src(src: &Src, x: &mut u8) {
}
}
impl<'a> LegalizeInstr<'a> {
pub fn new(ssa_alloc: &'a mut SSAValueAllocator) -> LegalizeInstr {
LegalizeInstr {
ssa_alloc: ssa_alloc,
instrs: Vec::new(),
fn mov_src(b: &mut impl SSABuilder, src: &mut Src, file: RegFile) {
let val = b.alloc_ssa(file, 1);
b.mov_to(val.into(), src.src_ref.into());
src.src_ref = val.into();
}
fn mov_src_if_not_reg(b: &mut impl SSABuilder, src: &mut Src, file: RegFile) {
if !src_is_reg(&src) {
mov_src(b, src, file);
}
}
fn swap_srcs_if_not_reg(x: &mut Src, y: &mut Src) {
if !src_is_reg(x) && src_is_reg(y) {
std::mem::swap(x, y);
}
}
fn legalize_instr(b: &mut impl SSABuilder, instr: &mut Instr) {
match &mut instr.op {
Op::FAdd(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
}
pub fn mov_src(&mut self, src: &mut Src, file: RegFile) {
let val = self.ssa_alloc.alloc(file);
self.instrs
.push(Instr::new_mov(val.into(), src.src_ref.into()).into());
src.src_ref = val.into();
}
pub fn mov_src_if_not_reg(&mut self, src: &mut Src, file: RegFile) {
if !src_is_reg(&src) {
self.mov_src(src, file);
Op::FFma(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
mov_src_if_not_reg(b, src2, RegFile::GPR);
}
}
pub fn swap_srcs_if_not_reg(&mut self, x: &mut Src, y: &mut Src) {
if !src_is_reg(x) && src_is_reg(y) {
std::mem::swap(x, y);
Op::FMnMx(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
}
Op::FMul(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::FSet(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cmp_op = op.cmp_op.flip();
}
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::FSetP(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cmp_op = op.cmp_op.flip();
}
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::MuFu(_) => (), /* Nothing to do */
Op::DAdd(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::IAbs(_) | Op::INeg(_) => (), /* Nothing to do */
Op::IAdd3(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
swap_srcs_if_not_reg(src2, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
mov_src_if_not_reg(b, src2, RegFile::GPR);
}
Op::IMad(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
mov_src_if_not_reg(b, src2, RegFile::GPR);
}
Op::IMad64(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
mov_src_if_not_reg(b, src2, RegFile::GPR);
}
Op::IMnMx(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
swap_srcs_if_not_reg(src0, src1);
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::ISetP(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cmp_op = op.cmp_op.flip();
}
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::Lop3(op) => {
/* Fold constants if we can */
op.op = LogicOp::new_lut(&|mut x, mut y, mut z| {
fold_lop_src(&op.srcs[0], &mut x);
fold_lop_src(&op.srcs[1], &mut y);
fold_lop_src(&op.srcs[2], &mut z);
op.op.eval(x, y, z)
});
for src in &mut op.srcs {
src.src_mod = SrcMod::None;
if src_as_lop_imm(src).is_some() {
src.src_ref = SrcRef::Zero;
}
}
pub fn map(&mut self, mut instr: Box<Instr>) -> MappedInstrs {
match &mut instr.op {
Op::FAdd(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.op = LogicOp::new_lut(&|x, y, z| op.op.eval(y, x, z))
}
Op::FFma(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
self.mov_src_if_not_reg(src2, RegFile::GPR);
if !src_is_reg(src2) && src_is_reg(src1) {
std::mem::swap(src2, src1);
op.op = LogicOp::new_lut(&|x, y, z| op.op.eval(x, z, y))
}
Op::FMnMx(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
mov_src_if_not_reg(b, src0, RegFile::GPR);
mov_src_if_not_reg(b, src2, RegFile::GPR);
}
Op::Shf(op) => {
mov_src_if_not_reg(b, &mut op.low, RegFile::GPR);
mov_src_if_not_reg(b, &mut op.high, RegFile::GPR);
}
Op::F2F(_)
| Op::F2I(_)
| Op::I2F(_)
| Op::Mov(_)
| Op::FRnd(_)
| Op::PopC(_)
| Op::Brev(_)
| Op::BFind(_)
| Op::Prmt(_) => (),
Op::Sel(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cond.src_mod = op.cond.src_mod.bnot();
}
Op::FMul(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::FSet(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cmp_op = op.cmp_op.flip();
}
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::FSetP(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cmp_op = op.cmp_op.flip();
}
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::MuFu(_) => (), /* Nothing to do */
Op::DAdd(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::IAbs(_) | Op::INeg(_) => (), /* Nothing to do */
Op::IAdd3(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.swap_srcs_if_not_reg(src2, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
self.mov_src_if_not_reg(src2, RegFile::GPR);
}
Op::IMad(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
self.mov_src_if_not_reg(src2, RegFile::GPR);
}
Op::IMad64(op) => {
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
self.mov_src_if_not_reg(src2, RegFile::GPR);
}
Op::IMnMx(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
self.swap_srcs_if_not_reg(src0, src1);
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::ISetP(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cmp_op = op.cmp_op.flip();
}
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::Lop3(op) => {
/* Fold constants if we can */
op.op = LogicOp::new_lut(&|mut x, mut y, mut z| {
mov_src_if_not_reg(b, src0, RegFile::GPR);
}
Op::PLop3(op) => {
/* Fold constants if we can */
for lop in &mut op.ops {
*lop = LogicOp::new_lut(&|mut x, mut y, mut z| {
fold_lop_src(&op.srcs[0], &mut x);
fold_lop_src(&op.srcs[1], &mut y);
fold_lop_src(&op.srcs[2], &mut z);
op.op.eval(x, y, z)
lop.eval(x, y, z)
});
for src in &mut op.srcs {
src.src_mod = SrcMod::None;
if src_as_lop_imm(src).is_some() {
src.src_ref = SrcRef::Zero;
}
}
for src in &mut op.srcs {
src.src_mod = SrcMod::None;
if src_as_lop_imm(src).is_some() {
src.src_ref = SrcRef::True;
}
}
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.op = LogicOp::new_lut(&|x, y, z| op.op.eval(y, x, z))
}
if !src_is_reg(src2) && src_is_reg(src1) {
std::mem::swap(src2, src1);
op.op = LogicOp::new_lut(&|x, y, z| op.op.eval(x, z, y))
}
self.mov_src_if_not_reg(src0, RegFile::GPR);
self.mov_src_if_not_reg(src2, RegFile::GPR);
}
Op::Shf(op) => {
self.mov_src_if_not_reg(&mut op.low, RegFile::GPR);
self.mov_src_if_not_reg(&mut op.high, RegFile::GPR);
}
Op::F2F(_)
| Op::F2I(_)
| Op::I2F(_)
| Op::Mov(_)
| Op::FRnd(_)
| Op::PopC(_)
| Op::Brev(_)
| Op::BFind(_)
| Op::Prmt(_) => (),
Op::Sel(op) => {
let [ref mut src0, ref mut src1] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
op.cond.src_mod = op.cond.src_mod.bnot();
}
self.mov_src_if_not_reg(src0, RegFile::GPR);
}
Op::PLop3(op) => {
/* Fold constants if we can */
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
for lop in &mut op.ops {
*lop = LogicOp::new_lut(&|mut x, mut y, mut z| {
fold_lop_src(&op.srcs[0], &mut x);
fold_lop_src(&op.srcs[1], &mut y);
fold_lop_src(&op.srcs[2], &mut z);
lop.eval(x, y, z)
});
*lop = LogicOp::new_lut(&|x, y, z| lop.eval(y, x, z));
}
for src in &mut op.srcs {
src.src_mod = SrcMod::None;
if src_as_lop_imm(src).is_some() {
src.src_ref = SrcRef::True;
}
}
let [ref mut src0, ref mut src1, ref mut src2] = op.srcs;
if !src_is_reg(src0) && src_is_reg(src1) {
std::mem::swap(src0, src1);
for lop in &mut op.ops {
*lop = LogicOp::new_lut(&|x, y, z| lop.eval(y, x, z));
}
}
if !src_is_reg(src2) && src_is_reg(src1) {
std::mem::swap(src2, src1);
for lop in &mut op.ops {
*lop = LogicOp::new_lut(&|x, y, z| lop.eval(x, z, y));
}
}
self.mov_src_if_not_reg(src0, RegFile::GPR);
self.mov_src_if_not_reg(src2, RegFile::GPR);
}
Op::Ldc(_) => (), // Nothing to do
Op::Copy(_) => (), // Nothing to do
_ => {
let src_types = instr.src_types();
for (i, src) in instr.srcs_mut().iter_mut().enumerate() {
match src_types[i] {
SrcType::SSA => {
if src.as_ssa().is_none() {
self.mov_src(src, RegFile::GPR);
}
}
SrcType::GPR => {
self.mov_src_if_not_reg(src, RegFile::GPR);
}
SrcType::ALU
| SrcType::F32
| SrcType::F64
| SrcType::I32 => {
panic!("ALU srcs must be legalized explicitly");
}
SrcType::Pred => {
panic!("Predicates must be legalized explicitly");
if !src_is_reg(src2) && src_is_reg(src1) {
std::mem::swap(src2, src1);
for lop in &mut op.ops {
*lop = LogicOp::new_lut(&|x, y, z| lop.eval(x, z, y));
}
}
mov_src_if_not_reg(b, src0, RegFile::GPR);
mov_src_if_not_reg(b, src2, RegFile::GPR);
}
Op::Ldc(_) => (), // Nothing to do
Op::Copy(_) => (), // Nothing to do
_ => {
let src_types = instr.src_types();
for (i, src) in instr.srcs_mut().iter_mut().enumerate() {
match src_types[i] {
SrcType::SSA => {
if src.as_ssa().is_none() {
mov_src(b, src, RegFile::GPR);
}
}
SrcType::GPR => {
mov_src_if_not_reg(b, src, RegFile::GPR);
}
SrcType::ALU
| SrcType::F32
| SrcType::F64
| SrcType::I32 => {
panic!("ALU srcs must be legalized explicitly");
}
SrcType::Pred => {
panic!("Predicates must be legalized explicitly");
}
}
}
}
}
let mut vec_src_map: HashMap<SSARef, SSARef> = HashMap::new();
let mut vec_comps = HashSet::new();
let mut pcopy = OpParCopy::new();
for src in instr.srcs_mut() {
if let SrcRef::SSA(vec) = &src.src_ref {
if vec.comps() == 1 {
continue;
}
let mut vec_src_map: HashMap<SSARef, SSARef> = HashMap::new();
let mut vec_comps = HashSet::new();
let mut pcopy = OpParCopy::new();
for src in instr.srcs_mut() {
if let SrcRef::SSA(vec) = &src.src_ref {
if vec.comps() == 1 {
continue;
}
/* If the same vector shows up twice in one instruction, that's
* okay. Just make it look the same as the previous source we
* fixed up.
/* If the same vector shows up twice in one instruction, that's
* okay. Just make it look the same as the previous source we
* fixed up.
*/
if let Some(new_vec) = vec_src_map.get(&vec) {
src.src_ref = (*new_vec).into();
continue;
}
let mut new_vec = *vec;
for c in 0..vec.comps() {
let ssa = vec[usize::from(c)];
/* If the same SSA value shows up in multiple non-identical
* vector sources or as multiple components in the same
* source, we need to make a copy so it can get assigned to
* multiple different registers.
*/
if let Some(new_vec) = vec_src_map.get(&vec) {
src.src_ref = (*new_vec).into();
continue;
if vec_comps.get(&ssa).is_some() {
let copy = b.alloc_ssa(ssa.file(), 1)[0];
pcopy.push(copy.into(), ssa.into());
new_vec[usize::from(c)] = copy;
} else {
vec_comps.insert(ssa);
}
let mut new_vec = *vec;
for c in 0..vec.comps() {
let ssa = vec[usize::from(c)];
/* If the same SSA value shows up in multiple non-identical
* vector sources or as multiple components in the same
* source, we need to make a copy so it can get assigned to
* multiple different registers.
*/
if vec_comps.get(&ssa).is_some() {
let copy = self.ssa_alloc.alloc(ssa.file());
pcopy.push(copy.into(), ssa.into());
new_vec[usize::from(c)] = copy;
} else {
vec_comps.insert(ssa);
}
}
vec_src_map.insert(*vec, new_vec);
src.src_ref = new_vec.into();
}
}
if !pcopy.is_empty() {
self.instrs.push(Instr::new_boxed(Op::ParCopy(pcopy)));
vec_src_map.insert(*vec, new_vec);
src.src_ref = new_vec.into();
}
}
self.instrs.push(instr);
MappedInstrs::Many(std::mem::replace(&mut self.instrs, Vec::new()))
if !pcopy.is_empty() {
b.push_op(pcopy);
}
}
impl Shader {
pub fn legalize(&mut self) {
self.map_instrs(|instr, ssa_alloc| -> MappedInstrs {
LegalizeInstr::new(ssa_alloc).map(instr)
self.map_instrs(|mut instr, ssa_alloc| -> MappedInstrs {
let mut b = SSAInstrBuilder::new(ssa_alloc);
legalize_instr(&mut b, &mut instr);
b.push_instr(instr);
b.as_mapped_instrs()
});
}
}