From f064b7a42c86703b22d90240094f2ec3e8ee7797 Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Fri, 29 Jan 2021 19:33:19 -0600 Subject: [PATCH] nir: Add some ssa-only fast-paths for nir_src rewrite Basically every pass in NIR uses nir_ssa_def_rewrite_uses which calls nir_instr_rewrite_src which is fairly complex because it handles all sorts of non-SSA cases. Since we already know a priori that every source written by nir_ssa_def_rewrite_uses is SSA, we can check new_src once at the top of the function and cut out all that complexity. While we're at it, we expose a new SSA-only nir_ssa_def_rewrite_uses_ssa helper which takes an SSA def which avoids the one SSA check. It's also more convenient 90% of the time. Compile time as tested by Rhys Perry Difference at 95.0% confidence -797.166 +/- 418.649 -0.566174% +/- 0.296441% (Student's t, pooled s = 325.459) Reviewed-by: Alyssa Rosenzweig Reviewed-by: Rhys Perry Reviewed-by: Connor Abbott Part-of: --- src/compiler/nir/nir.c | 23 ++++++++++++++++++----- src/compiler/nir/nir.h | 25 +++++++++++++++++++++++++ src/compiler/nir/nir_instr_set.c | 2 +- src/compiler/nir/nir_search.c | 2 +- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c index 176d9c15145..a5e0733e342 100644 --- a/src/compiler/nir/nir.c +++ b/src/compiler/nir/nir.c @@ -1604,15 +1604,28 @@ nir_ssa_dest_init(nir_instr *instr, nir_dest *dest, } void -nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src) +nir_ssa_def_rewrite_uses_ssa(nir_ssa_def *def, nir_ssa_def *new_ssa) { - assert(!new_src.is_ssa || def != new_src.ssa); - + assert(def != new_ssa); nir_foreach_use_safe(use_src, def) - nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src); + nir_instr_rewrite_src_ssa(use_src->parent_instr, use_src, new_ssa); nir_foreach_if_use_safe(use_src, def) - nir_if_rewrite_condition(use_src->parent_if, new_src); + nir_if_rewrite_condition_ssa(use_src->parent_if, use_src, new_ssa); +} + +void +nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src) +{ + if (new_src.is_ssa) { + nir_ssa_def_rewrite_uses_ssa(def, new_src.ssa); + } else { + nir_foreach_use_safe(use_src, def) + nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src); + + nir_foreach_if_use_safe(use_src, def) + nir_if_rewrite_condition(use_src->parent_if, new_src); + } } static bool diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index d70e91b980f..c716a5bdf4f 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -3862,8 +3862,32 @@ NIR_SRC_AS_(deref, nir_deref_instr, nir_instr_type_deref, nir_instr_as_deref) bool nir_src_is_dynamically_uniform(nir_src src); bool nir_srcs_equal(nir_src src1, nir_src src2); bool nir_instrs_equal(const nir_instr *instr1, const nir_instr *instr2); + +static inline void +nir_instr_rewrite_src_ssa(ASSERTED nir_instr *instr, + nir_src *src, nir_ssa_def *new_ssa) +{ + assert(src->parent_instr == instr); + assert(src->is_ssa && src->ssa); + list_del(&src->use_link); + src->ssa = new_ssa; + list_addtail(&src->use_link, &new_ssa->uses); +} + void nir_instr_rewrite_src(nir_instr *instr, nir_src *src, nir_src new_src); void nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src); + +static inline void +nir_if_rewrite_condition_ssa(ASSERTED nir_if *if_stmt, + nir_src *src, nir_ssa_def *new_ssa) +{ + assert(src->parent_if == if_stmt); + assert(src->is_ssa && src->ssa); + list_del(&src->use_link); + src->ssa = new_ssa; + list_addtail(&src->use_link, &new_ssa->if_uses); +} + void nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src); void nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest); @@ -3884,6 +3908,7 @@ nir_ssa_dest_init_for_type(nir_instr *instr, nir_dest *dest, glsl_get_bit_size(type), name); } void nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src); +void nir_ssa_def_rewrite_uses_ssa(nir_ssa_def *def, nir_ssa_def *new_ssa); void nir_ssa_def_rewrite_uses_after(nir_ssa_def *def, nir_src new_src, nir_instr *after_me); diff --git a/src/compiler/nir/nir_instr_set.c b/src/compiler/nir/nir_instr_set.c index 2d652503301..9220ed49bf5 100644 --- a/src/compiler/nir/nir_instr_set.c +++ b/src/compiler/nir/nir_instr_set.c @@ -821,7 +821,7 @@ nir_instr_set_add_or_rewrite(struct set *instr_set, nir_instr *instr) if (instr->type == nir_instr_type_alu && nir_instr_as_alu(instr)->exact) nir_instr_as_alu(match)->exact = true; - nir_ssa_def_rewrite_uses(def, nir_src_for_ssa(new_def)); + nir_ssa_def_rewrite_uses_ssa(def, new_def); return true; } diff --git a/src/compiler/nir/nir_search.c b/src/compiler/nir/nir_search.c index 04ad2490a3f..1408917fde4 100644 --- a/src/compiler/nir/nir_search.c +++ b/src/compiler/nir/nir_search.c @@ -793,7 +793,7 @@ nir_replace_instr(nir_builder *build, nir_alu_instr *instr, /* Rewrite the uses of the old SSA value to the new one, and recurse * through the uses updating the automaton's state. */ - nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa, nir_src_for_ssa(ssa_val)); + nir_ssa_def_rewrite_uses_ssa(&instr->dest.dest.ssa, ssa_val); nir_algebraic_update_automaton(ssa_val->parent_instr, algebraic_worklist, states, pass_op_table);