/* * Copyright © 2019 Valve Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Rhys Perry (pendingchaos02@gmail.com) * */ #include #include "aco_ir.h" #include "aco_builder.h" #include namespace aco { struct ssa_state { bool needs_init; uint64_t cur_undef_operands; unsigned phi_block_idx; unsigned loop_nest_depth; std::map writes; std::vector latest; }; Operand get_ssa(Program *program, unsigned block_idx, ssa_state *state, bool before_write) { if (!before_write) { auto it = state->writes.find(block_idx); if (it != state->writes.end()) return Operand(Temp(it->second, program->lane_mask)); if (state->latest[block_idx]) return Operand(Temp(state->latest[block_idx], program->lane_mask)); } Block& block = program->blocks[block_idx]; size_t pred = block.linear_preds.size(); if (pred == 0 || block.loop_nest_depth < state->loop_nest_depth) { return Operand(program->lane_mask); } else if (block.loop_nest_depth > state->loop_nest_depth) { Operand op = get_ssa(program, block_idx - 1, state, false); assert(!state->latest[block_idx]); state->latest[block_idx] = op.tempId(); return op; } else if (pred == 1 || block.kind & block_kind_loop_exit) { Operand op = get_ssa(program, block.linear_preds[0], state, false); assert(!state->latest[block_idx]); state->latest[block_idx] = op.tempId(); return op; } else if (block.kind & block_kind_loop_header && !(program->blocks[state->phi_block_idx].kind & block_kind_loop_exit)) { return Operand(program->lane_mask); } else { unsigned res = program->allocateId(); assert(!state->latest[block_idx]); state->latest[block_idx] = res; aco_ptr phi{create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, pred, 1)}; for (unsigned i = 0; i < pred; i++) phi->operands[i] = get_ssa(program, block.linear_preds[i], state, false); phi->definitions[0] = Definition(Temp{res, program->lane_mask}); block.instructions.emplace(block.instructions.begin(), std::move(phi)); return Operand(Temp(res, program->lane_mask)); } } void insert_before_logical_end(Block *block, aco_ptr instr) { auto IsLogicalEnd = [] (const aco_ptr& instr) -> bool { return instr->opcode == aco_opcode::p_logical_end; }; auto it = std::find_if(block->instructions.crbegin(), block->instructions.crend(), IsLogicalEnd); if (it == block->instructions.crend()) { assert(block->instructions.back()->format == Format::PSEUDO_BRANCH); block->instructions.insert(std::prev(block->instructions.end()), std::move(instr)); } else block->instructions.insert(std::prev(it.base()), std::move(instr)); } void lower_divergent_bool_phi(Program *program, ssa_state *state, Block *block, aco_ptr& phi) { Builder bld(program); state->latest.resize(program->blocks.size()); uint64_t undef_operands = 0; for (unsigned i = 0; i < phi->operands.size(); i++) undef_operands |= phi->operands[i].isUndefined() << i; if (state->needs_init || undef_operands != state->cur_undef_operands || block->logical_preds.size() > 64) { /* this only has to be done once per block unless the set of predecessors * which are undefined changes */ state->cur_undef_operands = undef_operands; state->phi_block_idx = block->index; state->loop_nest_depth = block->loop_nest_depth; if (block->kind & block_kind_loop_exit) { state->loop_nest_depth += 1; } state->writes.clear(); state->needs_init = false; } std::fill(state->latest.begin(), state->latest.end(), 0); for (unsigned i = 0; i < phi->operands.size(); i++) { if (phi->operands[i].isUndefined()) continue; state->writes[block->logical_preds[i]] = program->allocateId(); } for (unsigned i = 0; i < phi->operands.size(); i++) { Block *pred = &program->blocks[block->logical_preds[i]]; if (phi->operands[i].isUndefined()) continue; Operand cur = get_ssa(program, pred->index, state, true); assert(cur.regClass() == bld.lm); Temp new_cur = {state->writes.at(pred->index), program->lane_mask}; assert(new_cur.regClass() == bld.lm); if (cur.isUndefined()) { insert_before_logical_end(pred, bld.sop1(aco_opcode::s_mov_b64, Definition(new_cur), phi->operands[i]).get_ptr()); } else { Temp tmp1 = bld.tmp(bld.lm), tmp2 = bld.tmp(bld.lm); insert_before_logical_end(pred, bld.sop2(Builder::s_andn2, Definition(tmp1), bld.def(s1, scc), cur, Operand(exec, bld.lm)).get_ptr()); insert_before_logical_end(pred, bld.sop2(Builder::s_and, Definition(tmp2), bld.def(s1, scc), phi->operands[i].getTemp(), Operand(exec, bld.lm)).get_ptr()); insert_before_logical_end(pred, bld.sop2(Builder::s_or, Definition(new_cur), bld.def(s1, scc), tmp1, tmp2).get_ptr()); } } unsigned num_preds = block->linear_preds.size(); if (phi->operands.size() != num_preds) { Pseudo_instruction* new_phi{create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, num_preds, 1)}; new_phi->definitions[0] = phi->definitions[0]; phi.reset(new_phi); } else { phi->opcode = aco_opcode::p_linear_phi; } assert(phi->operands.size() == num_preds); for (unsigned i = 0; i < num_preds; i++) phi->operands[i] = get_ssa(program, block->linear_preds[i], state, false); return; } void lower_subdword_phis(Program *program, Block *block, aco_ptr& phi) { Builder bld(program); for (unsigned i = 0; i < phi->operands.size(); i++) { if (phi->operands[i].isUndefined()) continue; if (phi->operands[i].regClass() == phi->definitions[0].regClass()) continue; assert(phi->operands[i].isTemp()); Block *pred = &program->blocks[block->logical_preds[i]]; Temp phi_src = phi->operands[i].getTemp(); assert(phi_src.regClass().type() == RegType::sgpr); Temp tmp = bld.tmp(RegClass(RegType::vgpr, phi_src.size())); insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), phi_src).get_ptr()); Temp new_phi_src = bld.tmp(phi->definitions[0].regClass()); insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_extract_vector, Definition(new_phi_src), tmp, Operand(0u)).get_ptr()); phi->operands[i].setTemp(new_phi_src); } return; } void lower_phis(Program* program) { ssa_state state; for (Block& block : program->blocks) { state.needs_init = true; for (aco_ptr& phi : block.instructions) { if (phi->opcode == aco_opcode::p_phi) { assert(program->wave_size == 64 ? phi->definitions[0].regClass() != s1 : phi->definitions[0].regClass() != s2); if (phi->definitions[0].regClass() == program->lane_mask) lower_divergent_bool_phi(program, &state, &block, phi); else if (phi->definitions[0].regClass().is_subdword()) lower_subdword_phis(program, &block, phi); } else if (!is_phi(phi)) { break; } } } } }