aco: split selection_control_remove into rarely_taken and never_taken

No fossil-db changes.

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>
Reviewed-by: Daniel Schürmann <daniel@schuermann.dev>
Backport-to: 24.1
Backport-to: 24.2
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30321>

src/amd/compiler/aco_insert_exec_mask.cpp

@@ -672,7 +672,7 @@ add_branch_code(exec_ctx& ctx, Block* block)
       assert(block->linear_succs.size() == 2);
       assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z);
       Temp cond = block->instructions.back()->operands[0].getTemp();
-      const bool sel_ctrl = block->instructions.back()->branch().selection_control_remove;
+      aco_ptr<Instruction> branch = std::move(block->instructions.back());
       block->instructions.pop_back();
 
       uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
@@ -690,14 +690,15 @@ add_branch_code(exec_ctx& ctx, Block* block)
 
       Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
                                      block->linear_succs[1], block->linear_succs[0]);
-      r->branch().selection_control_remove = sel_ctrl;
+      r->branch().rarely_taken = branch->branch().rarely_taken;
+      r->branch().never_taken = branch->branch().never_taken;
       return;
    }
 
    if (block->kind & block_kind_invert) {
       // exec = s_andn2_b64 (original_exec, exec)
       assert(block->instructions.back()->opcode == aco_opcode::p_branch);
-      const bool sel_ctrl = block->instructions.back()->branch().selection_control_remove;
+      aco_ptr<Instruction> branch = std::move(block->instructions.back());
       block->instructions.pop_back();
       assert(ctx.info[idx].exec.size() >= 2);
       Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first;
@@ -706,7 +707,8 @@ add_branch_code(exec_ctx& ctx, Block* block)
 
       Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
                                      block->linear_succs[1], block->linear_succs[0]);
-      r->branch().selection_control_remove = sel_ctrl;
+      r->branch().rarely_taken = branch->branch().rarely_taken;
+      r->branch().never_taken = branch->branch().never_taken;
       return;
    }
 

src/amd/compiler/aco_instruction_selection.cpp

@@ -10615,9 +10615,12 @@ begin_divergent_if_then(isel_context* ctx, if_context* ic, Temp cond,
    branch.reset(create_instruction(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 1));
    branch->definitions[0] = Definition(ctx->program->allocateTmp(s2));
    branch->operands[0] = Operand(cond);
-   branch->branch().selection_control_remove =
-      sel_ctrl == nir_selection_control_flatten ||
-      sel_ctrl == nir_selection_control_divergent_always_taken;
+   bool never_taken =
+      sel_ctrl == nir_selection_control_divergent_always_taken &&
+      !(ctx->cf_info.exec.potentially_empty_discard || ctx->cf_info.exec.potentially_empty_break ||
+        ctx->cf_info.exec.potentially_empty_continue);
+   branch->branch().rarely_taken = sel_ctrl == nir_selection_control_flatten || never_taken;
+   branch->branch().never_taken = never_taken;
    ctx->block->instructions.push_back(std::move(branch));
 
    ic->BB_if_idx = ctx->block->index;
@@ -10680,9 +10683,12 @@ begin_divergent_if_else(isel_context* ctx, if_context* ic,
    /* branch to linear else block (skip else) */
    branch.reset(create_instruction(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 1));
    branch->definitions[0] = Definition(ctx->program->allocateTmp(s2));
-   branch->branch().selection_control_remove =
-      sel_ctrl == nir_selection_control_flatten ||
-      sel_ctrl == nir_selection_control_divergent_always_taken;
+   bool never_taken =
+      sel_ctrl == nir_selection_control_divergent_always_taken &&
+      !(ctx->cf_info.exec.potentially_empty_discard || ctx->cf_info.exec.potentially_empty_break ||
+        ctx->cf_info.exec.potentially_empty_continue);
+   branch->branch().rarely_taken = sel_ctrl == nir_selection_control_flatten || never_taken;
+   branch->branch().never_taken = never_taken;
    ctx->block->instructions.push_back(std::move(branch));
 
    ic->exec_old.combine(ctx->cf_info.exec);

src/amd/compiler/aco_ir.h

@@ -1674,10 +1674,11 @@ struct Pseudo_branch_instruction : public Instruction {
     */
    uint32_t target[2];
 
-   /* Indicates that selection control prefers to remove this instruction if possible.
-    * This is set when the branch is divergent and always taken, or flattened.
-    */
-   bool selection_control_remove;
+   /* Indicates that this rarely or never jumps to target[0]. */
+   bool rarely_taken;
+   bool never_taken;
+
+   uint16_t padding;
 };
 static_assert(sizeof(Pseudo_branch_instruction) == sizeof(Instruction) + 12, "Unexpected padding");
 

src/amd/compiler/aco_lower_to_hw_instr.cpp

@@ -2832,8 +2832,7 @@ lower_to_hw_instr(Program* program)
              * - The application prefers to remove control flow
              * - The compiler stack knows that it's a divergent branch always taken
              */
-            const bool prefer_remove =
-               branch->selection_control_remove && ctx.program->gfx_level >= GFX10;
+            const bool prefer_remove = branch->rarely_taken && ctx.program->gfx_level >= GFX10;
             bool can_remove = block->index < target;
             unsigned num_scalar = 0;
             unsigned num_vector = 0;

src/amd/compiler/aco_print_ir.cpp

@@ -601,6 +601,10 @@ print_instr_format_specific(enum amd_gfx_level gfx_level, const Instruction* ins
          fprintf(output, " BB%d", branch.target[0]);
       if (branch.target[1] != 0)
          fprintf(output, ", BB%d", branch.target[1]);
+      if (branch.rarely_taken)
+         fprintf(output, " rarely_taken");
+      if (branch.never_taken)
+         fprintf(output, " never_taken");
       break;
    }
    case Format::PSEUDO_REDUCTION: {