diff --git a/src/panfrost/bifrost/bi_opt_dce.c b/src/panfrost/bifrost/bi_opt_dce.c index 84ea9882c42..aa20f146d33 100644 --- a/src/panfrost/bifrost/bi_opt_dce.c +++ b/src/panfrost/bifrost/bi_opt_dce.c @@ -25,54 +25,65 @@ #include "compiler.h" #include "util/u_memory.h" -/* A simple liveness-based dead code elimination pass. */ +/* A simple SSA-based mark-and-sweep dead code elimination pass. */ void bi_opt_dead_code_eliminate(bi_context *ctx) { - unsigned temp_count = bi_max_temp(ctx); + /* Mark live values */ + BITSET_WORD *mark = calloc(sizeof(BITSET_WORD), BITSET_WORDS(ctx->ssa_alloc)); - bi_compute_liveness(ctx); + u_worklist worklist; + u_worklist_init(&worklist, ctx->num_blocks, NULL); - bi_foreach_block_rev(ctx, block) { - uint8_t *live = rzalloc_array(block, uint8_t, temp_count); - - bi_foreach_successor(block, succ) { - for (unsigned i = 0; i < temp_count; ++i) - live[i] |= succ->live_in[i]; - } - - bi_foreach_instr_in_block_safe_rev(block, ins) { - bool all_null = true; - - bi_foreach_dest(ins, d) { - /* Destination required */ - if (ins->op == BI_OPCODE_AXCHG_I32 || - ins->op == BI_OPCODE_ACMPXCHG_I32 || - ins->op == BI_OPCODE_ATOM_RETURN_I32 || - ins->op == BI_OPCODE_ATOM1_RETURN_I32 || - ins->op == BI_OPCODE_BLEND || - ins->op == BI_OPCODE_ATEST || - ins->op == BI_OPCODE_ZS_EMIT) - continue; - - unsigned index = bi_get_node(ins->dest[d]); - - if (index >= temp_count) - all_null = false; - else if (live[index] & bi_writemask(ins, d)) - all_null = false; - } - - if (all_null && !bi_side_effects(ins)) - bi_remove_instruction(ins); - else - bi_liveness_ins_update(live, ins, temp_count); - } - - ralloc_free(block->live_in); - block->live_in = live; + bi_foreach_block(ctx, block) { + bi_worklist_push_head(&worklist, block); } + + while(!u_worklist_is_empty(&worklist)) { + /* Pop in reverse order for backwards pass */ + bi_block *blk = bi_worklist_pop_head(&worklist); + + bool progress = false; + + bi_foreach_instr_in_block_rev(blk, I) { + bool needed = bi_side_effects(I); + + bi_foreach_dest(I, d) + needed |= BITSET_TEST(mark, I->dest[d].value); + + if (!needed) + continue; + + bi_foreach_src(I, s) { + if (bi_is_ssa(I->src[s])) { + progress |= !BITSET_TEST(mark, I->src[s].value); + BITSET_SET(mark, I->src[s].value); + } + } + } + + /* XXX: slow */ + if (progress) { + bi_foreach_block(ctx, block) + bi_worklist_push_head(&worklist, block); + } + } + + u_worklist_fini(&worklist); + + /* Sweep */ + bi_foreach_instr_global_safe(ctx, I) { + bool needed = bi_side_effects(I); + + bi_foreach_dest(I, d) + needed |= BITSET_TEST(mark, I->dest[d].value); + + if (!needed) + bi_remove_instruction(I); + } + + free(mark); } /* Post-RA liveness-based dead code analysis to clean up results of bundling */ diff --git a/src/panfrost/bifrost/bi_pressure_schedule.c b/src/panfrost/bifrost/bi_pressure_schedule.c index ff8a2aebc08..3735a3933ab 100644 --- a/src/panfrost/bifrost/bi_pressure_schedule.c +++ b/src/panfrost/bifrost/bi_pressure_schedule.c @@ -34,10 +34,7 @@ struct sched_ctx { struct dag *dag; /* Live set */ - uint8_t *live; - - /* Size of the live set */ - unsigned max; + BITSET_WORD *live; }; struct sched_node { @@ -227,35 +224,33 @@ create_dag(bi_context *ctx, bi_block *block, void *memctx) * live_in = (live_out - KILL) + GEN */ static signed -calculate_pressure_delta(bi_instr *I, uint8_t *live, unsigned max) +calculate_pressure_delta(bi_instr *I, BITSET_WORD *live) { signed delta = 0; /* Destinations must be unique */ bi_foreach_dest(I, d) { - unsigned node = bi_get_node(I->dest[d]); - assert(node < max); + assert(I->dest[d].type == BI_INDEX_NORMAL); - if (live[node]) + if (BITSET_TEST(live, I->dest[d].value)) delta -= bi_count_write_registers(I, d); } bi_foreach_src(I, src) { - unsigned node = bi_get_node(I->src[src]); - if (node >= max) + if (I->src[src].type != BI_INDEX_NORMAL) continue; /* Filter duplicates */ bool dupe = false; for (unsigned i = 0; i < src; ++i) { - if (bi_get_node(I->src[i]) == node) { + if (bi_is_equiv(I->src[i], I->src[src])) { dupe = true; break; } } - if (!dupe && !live[node]) + if (!dupe && !BITSET_TEST(live, I->src[src].value)) delta += bi_count_read_registers(I, src); } @@ -273,7 +268,7 @@ choose_instr(struct sched_ctx *s) struct sched_node *best = NULL; list_for_each_entry(struct sched_node, n, &s->dag->heads, dag.link) { - int32_t delta = calculate_pressure_delta(n->instr, s->live, s->max); + int32_t delta = calculate_pressure_delta(n->instr, s->live); if (delta < min_delta) { best = n; @@ -292,16 +287,16 @@ pressure_schedule_block(bi_context *ctx, bi_block *block, struct sched_ctx *s) signed orig_max_pressure = 0; unsigned nr_ins = 0; - memcpy(s->live, block->live_out, s->max); + memcpy(s->live, block->ssa_live_out, BITSET_WORDS(ctx->ssa_alloc) * sizeof(BITSET_WORD)); bi_foreach_instr_in_block_rev(block, I) { - pressure += calculate_pressure_delta(I, s->live, s->max); + pressure += calculate_pressure_delta(I, s->live); orig_max_pressure = MAX2(pressure, orig_max_pressure); - bi_liveness_ins_update(s->live, I, s->max); + bi_liveness_ins_update_ssa(s->live, I); nr_ins++; } - memcpy(s->live, block->live_out, s->max); + memcpy(s->live, block->ssa_live_out, BITSET_WORDS(ctx->ssa_alloc) * sizeof(BITSET_WORD)); /* off by a constant, that's ok */ signed max_pressure = 0; @@ -312,12 +307,12 @@ pressure_schedule_block(bi_context *ctx, bi_block *block, struct sched_ctx *s) while (!list_is_empty(&s->dag->heads)) { struct sched_node *node = choose_instr(s); - pressure += calculate_pressure_delta(node->instr, s->live, s->max); + pressure += calculate_pressure_delta(node->instr, s->live); max_pressure = MAX2(pressure, max_pressure); dag_prune_head(s->dag, &node->dag); schedule[nr_ins++] = node; - bi_liveness_ins_update(s->live, node->instr, s->max); + bi_liveness_ins_update_ssa(s->live, node->instr); } /* Bail if it looks like it's worse */ @@ -338,15 +333,13 @@ pressure_schedule_block(bi_context *ctx, bi_block *block, struct sched_ctx *s) void bi_pressure_schedule(bi_context *ctx) { - bi_compute_liveness(ctx); - unsigned temp_count = bi_max_temp(ctx); + bi_compute_liveness_ssa(ctx); void *memctx = ralloc_context(ctx); - uint8_t *live = ralloc_array(memctx, uint8_t, temp_count); + BITSET_WORD *live = ralloc_array(memctx, BITSET_WORD, BITSET_WORDS(ctx->ssa_alloc)); bi_foreach_block(ctx, block) { struct sched_ctx sctx = { .dag = create_dag(ctx, block, memctx), - .max = temp_count, .live = live }; diff --git a/src/panfrost/bifrost/bi_validate.c b/src/panfrost/bifrost/bi_validate.c index 3a2c51de236..7d10ff130ca 100644 --- a/src/panfrost/bifrost/bi_validate.c +++ b/src/panfrost/bifrost/bi_validate.c @@ -39,15 +39,14 @@ bi_validate_initialization(bi_context *ctx) /* Calculate the live set */ bi_block *entry = bi_entry_block(ctx); - unsigned temp_count = bi_max_temp(ctx); - bi_compute_liveness(ctx); + bi_compute_liveness_ssa(ctx); /* Validate that the live set is indeed empty */ - for (unsigned i = 0; i < temp_count; ++i) { - if (entry->live_in[i] == 0) continue; - - fprintf(stderr, "%s%u\n", (i & PAN_IS_REG) ? "r" : "", i >> 1); - success = false; + for (unsigned i = 0; i < ctx->ssa_alloc; ++i) { + if (BITSET_TEST(entry->ssa_live_in, i)) { + fprintf(stderr, "%u\n", i); + success = false; + } } return success; diff --git a/src/panfrost/bifrost/bifrost_compile.c b/src/panfrost/bifrost/bifrost_compile.c index c9e8e90dd6e..fd35dd7d0df 100644 --- a/src/panfrost/bifrost/bifrost_compile.c +++ b/src/panfrost/bifrost/bifrost_compile.c @@ -5021,11 +5021,6 @@ bi_compile_variant_nir(nir_shader *nir, } } - /* We can only go out-of-SSA after speciailizing IDVS, as opt_dead_cf - * doesn't know how to deal with nir_register. - */ - NIR_PASS_V(nir, nir_convert_from_ssa, true); - /* If nothing is pushed, all UBOs need to be uploaded */ ctx->ubo_mask = ~0;