mesa/src/amd/compiler/aco_lower_phis.cpp

/*
 * Copyright © 2019 Valve Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Rhys Perry (pendingchaos02@gmail.com)
 *
 */

#include <map>

#include "aco_ir.h"
#include "aco_builder.h"
#include <algorithm>


namespace aco {

struct ssa_state {
   bool needs_init;
   uint64_t cur_undef_operands;

   unsigned phi_block_idx;
   unsigned loop_nest_depth;
   std::map<unsigned, unsigned> writes;
   std::vector<unsigned> latest;
};

Operand get_ssa(Program *program, unsigned block_idx, ssa_state *state, bool before_write)
{
   if (!before_write) {
      auto it = state->writes.find(block_idx);
      if (it != state->writes.end())
         return Operand(Temp(it->second, program->lane_mask));
      if (state->latest[block_idx])
         return Operand(Temp(state->latest[block_idx], program->lane_mask));
   }

   Block& block = program->blocks[block_idx];
   size_t pred = block.linear_preds.size();
   if (pred == 0 || block.loop_nest_depth < state->loop_nest_depth) {
      return Operand(program->lane_mask);
   } else if (block.loop_nest_depth > state->loop_nest_depth) {
      Operand op = get_ssa(program, block_idx - 1, state, false);
      assert(!state->latest[block_idx]);
      state->latest[block_idx] = op.tempId();
      return op;
   } else if (pred == 1 || block.kind & block_kind_loop_exit) {
      Operand op = get_ssa(program, block.linear_preds[0], state, false);
      assert(!state->latest[block_idx]);
      state->latest[block_idx] = op.tempId();
      return op;
   } else if (block.kind & block_kind_loop_header &&
              !(program->blocks[state->phi_block_idx].kind & block_kind_loop_exit)) {
      return Operand(program->lane_mask);
   } else {
      unsigned res = program->allocateId();
      assert(!state->latest[block_idx]);
      state->latest[block_idx] = res;

      aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, pred, 1)};
      for (unsigned i = 0; i < pred; i++)
         phi->operands[i] = get_ssa(program, block.linear_preds[i], state, false);
      phi->definitions[0] = Definition(Temp{res, program->lane_mask});
      block.instructions.emplace(block.instructions.begin(), std::move(phi));

      return Operand(Temp(res, program->lane_mask));
   }
}

void insert_before_logical_end(Block *block, aco_ptr<Instruction> instr)
{
   auto IsLogicalEnd = [] (const aco_ptr<Instruction>& instr) -> bool {
      return instr->opcode == aco_opcode::p_logical_end;
   };
   auto it = std::find_if(block->instructions.crbegin(), block->instructions.crend(), IsLogicalEnd);

   if (it == block->instructions.crend()) {
      assert(block->instructions.back()->format == Format::PSEUDO_BRANCH);
      block->instructions.insert(std::prev(block->instructions.end()), std::move(instr));
   }
   else
      block->instructions.insert(std::prev(it.base()), std::move(instr));
}

void lower_divergent_bool_phi(Program *program, ssa_state *state, Block *block, aco_ptr<Instruction>& phi)
{
   Builder bld(program);

   state->latest.resize(program->blocks.size());

   uint64_t undef_operands = 0;
   for (unsigned i = 0; i < phi->operands.size(); i++)
      undef_operands |= phi->operands[i].isUndefined() << i;

   if (state->needs_init || undef_operands != state->cur_undef_operands ||
       block->logical_preds.size() > 64) {
      /* this only has to be done once per block unless the set of predecessors
       * which are undefined changes */
      state->cur_undef_operands = undef_operands;
      state->phi_block_idx = block->index;
      state->loop_nest_depth = block->loop_nest_depth;
      if (block->kind & block_kind_loop_exit) {
         state->loop_nest_depth += 1;
      }
      state->writes.clear();
      state->needs_init = false;
   }
   std::fill(state->latest.begin(), state->latest.end(), 0);

   for (unsigned i = 0; i < phi->operands.size(); i++) {
      if (phi->operands[i].isUndefined())
         continue;

      state->writes[block->logical_preds[i]] = program->allocateId();
   }

   for (unsigned i = 0; i < phi->operands.size(); i++) {
      Block *pred = &program->blocks[block->logical_preds[i]];

      if (phi->operands[i].isUndefined())
         continue;

      Operand cur = get_ssa(program, pred->index, state, true);
      assert(cur.regClass() == bld.lm);
      Temp new_cur = {state->writes.at(pred->index), program->lane_mask};
      assert(new_cur.regClass() == bld.lm);

      if (cur.isUndefined()) {
         insert_before_logical_end(pred, bld.sop1(aco_opcode::s_mov_b64, Definition(new_cur), phi->operands[i]).get_ptr());
      } else {
         Temp tmp1 = bld.tmp(bld.lm), tmp2 = bld.tmp(bld.lm);
         insert_before_logical_end(pred,
            bld.sop2(Builder::s_andn2, Definition(tmp1), bld.def(s1, scc),
                     cur, Operand(exec, bld.lm)).get_ptr());
         insert_before_logical_end(pred,
            bld.sop2(Builder::s_and, Definition(tmp2), bld.def(s1, scc),
                     phi->operands[i].getTemp(), Operand(exec, bld.lm)).get_ptr());
         insert_before_logical_end(pred,
            bld.sop2(Builder::s_or, Definition(new_cur), bld.def(s1, scc),
                     tmp1, tmp2).get_ptr());
      }
   }

   unsigned num_preds = block->linear_preds.size();
   if (phi->operands.size() != num_preds) {
      Pseudo_instruction* new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_preds, 1)};
      new_phi->definitions[0] = phi->definitions[0];
      phi.reset(new_phi);
   } else {
      phi->opcode = aco_opcode::p_linear_phi;
   }
   assert(phi->operands.size() == num_preds);

   for (unsigned i = 0; i < num_preds; i++)
      phi->operands[i] = get_ssa(program, block->linear_preds[i], state, false);

   return;
}

void lower_subdword_phis(Program *program, Block *block, aco_ptr<Instruction>& phi)
{
   Builder bld(program);
   for (unsigned i = 0; i < phi->operands.size(); i++) {
      if (phi->operands[i].isUndefined())
         continue;
      if (phi->operands[i].regClass() == phi->definitions[0].regClass())
         continue;

      assert(phi->operands[i].isTemp());
      Block *pred = &program->blocks[block->logical_preds[i]];
      Temp phi_src = phi->operands[i].getTemp();

      assert(phi_src.regClass().type() == RegType::sgpr);
      Temp tmp = bld.tmp(RegClass(RegType::vgpr, phi_src.size()));
      insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), phi_src).get_ptr());
      Temp new_phi_src = bld.tmp(phi->definitions[0].regClass());
      insert_before_logical_end(pred, bld.pseudo(aco_opcode::p_extract_vector, Definition(new_phi_src), tmp, Operand(0u)).get_ptr());

      phi->operands[i].setTemp(new_phi_src);
   }
   return;
}

void lower_phis(Program* program)
{
   ssa_state state;

   for (Block& block : program->blocks) {
      state.needs_init = true;
      for (aco_ptr<Instruction>& phi : block.instructions) {
         if (phi->opcode == aco_opcode::p_phi) {
            assert(program->wave_size == 64 ? phi->definitions[0].regClass() != s1 : phi->definitions[0].regClass() != s2);
            if (phi->definitions[0].regClass() == program->lane_mask)
               lower_divergent_bool_phi(program, &state, &block, phi);
            else if (phi->definitions[0].regClass().is_subdword())
               lower_subdword_phis(program, &block, phi);
         } else if (!is_phi(phi)) {
            break;
         }
      }
   }
}

}