From e8dbed2be4b232f69c5197a42cbf620cd6b44e11 Mon Sep 17 00:00:00 2001 From: Job Noorman Date: Fri, 21 Nov 2025 14:11:56 +0100 Subject: [PATCH] ir3: don't use list_head for rpt groups To link together instructions in a rpt group we currently (ab)use list_head. This is a bit of a hack because we don't actually have a list_head that points to the first instruction without being embedded in an instruction itself (the way list_head is supposed to be used). Instead, the list_head embedded in the first instruction of a rpt group also serves as the one pointing to the list. In order make a distinction between the first and last instruction (for which the main list_head would usually be used), we rely on the fact that (currently) instructions in a rpt group are emitted in order which means that later instructions have a larger serialno than earlier ones. In order to make all this less hacky, and to lift the restriction of needing instructions to be emitted in order, replace the list_head with explicit rpt_next/rpt_prev pointers which link the instructions together in a doubly but non-circular linked list. Signed-off-by: Job Noorman Part-of: --- src/freedreno/ir3/ir3.c | 43 ++++++++++++++++++++++--------------- src/freedreno/ir3/ir3.h | 36 +++++++++++-------------------- src/freedreno/ir3/ir3_rpt.c | 32 ++++++++++++++------------- 3 files changed, 55 insertions(+), 56 deletions(-) diff --git a/src/freedreno/ir3/ir3.c b/src/freedreno/ir3/ir3.c index 086f4cb16a9..61c89a0ea09 100644 --- a/src/freedreno/ir3/ir3.c +++ b/src/freedreno/ir3/ir3.c @@ -896,7 +896,6 @@ instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc) instr->srcs_max = nsrc; #endif - list_inithead(&instr->rpt_node); return instr; } @@ -975,7 +974,8 @@ ir3_instr_clone(struct ir3_instruction *instr) new_instr->dsts = dsts; new_instr->srcs = srcs; new_instr->uses = NULL; - list_inithead(&new_instr->rpt_node); + new_instr->rpt_prev = NULL; + new_instr->rpt_next = NULL; insert_instr(ir3_before_terminator(instr->block), new_instr); @@ -1020,7 +1020,14 @@ void ir3_instr_remove(struct ir3_instruction *instr) { list_delinit(&instr->node); - list_delinit(&instr->rpt_node); + + if (instr->rpt_prev) { + instr->rpt_prev->rpt_next = instr->rpt_next; + } + + if (instr->rpt_next) { + instr->rpt_next->rpt_prev = instr->rpt_prev; + } } void @@ -1029,28 +1036,25 @@ ir3_instr_create_rpt(struct ir3_instruction **instrs, unsigned n) assert(n > 0 && !ir3_instr_is_rpt(instrs[0])); for (unsigned i = 1; i < n; ++i) { - assert(!ir3_instr_is_rpt(instrs[i])); - assert(instrs[i]->serialno > instrs[i - 1]->serialno); + struct ir3_instruction *instr = instrs[i]; + struct ir3_instruction *prev = instrs[i - 1]; + assert(!ir3_instr_is_rpt(instr)); - list_addtail(&instrs[i]->rpt_node, &instrs[0]->rpt_node); + prev->rpt_next = instr; + instr->rpt_prev = prev; } } bool ir3_instr_is_rpt(const struct ir3_instruction *instr) { - return !list_is_empty(&instr->rpt_node); + return instr->rpt_prev || instr->rpt_next; } bool ir3_instr_is_first_rpt(const struct ir3_instruction *instr) { - if (!ir3_instr_is_rpt(instr)) - return false; - - struct ir3_instruction *prev_rpt = - list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node); - return prev_rpt->serialno > instr->serialno; + return instr->rpt_next && !instr->rpt_prev; } struct ir3_instruction * @@ -1058,9 +1062,7 @@ ir3_instr_prev_rpt(const struct ir3_instruction *instr) { assert(ir3_instr_is_rpt(instr)); - if (ir3_instr_is_first_rpt(instr)) - return NULL; - return list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node); + return instr->rpt_prev; } struct ir3_instruction * @@ -1081,7 +1083,14 @@ ir3_instr_rpt_length(const struct ir3_instruction *instr) { assert(ir3_instr_is_first_rpt(instr)); - return list_length(&instr->rpt_node) + 1; + unsigned length = 1; + + while (instr->rpt_next) { + length++; + instr = instr->rpt_next; + } + + return length; } struct ir3_register * diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index 864642f9dc7..9b9a09b8bf2 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -609,22 +609,12 @@ struct ir3_instruction { /* List of this instruction's repeat group. Vectorized NIR instructions are * emitted as multiple scalar instructions that are linked together using - * this field. After RA, the ir3_combine_rpt pass iterates these groups and, - * if the register assignment allows it, merges them into a (rptN) + * these fields. After RA, the ir3_combine_rpt pass iterates these groups + * and, if the register assignment allows it, merges them into a (rptN) * instruction. - * - * NOTE: this is not a typical list as there is no empty list head. The list - * head is stored in the first instruction of the repeat group so also refers - * to a list entry. In order to distinguish the list's first entry, we use - * serialno: instructions in a repeat group are always emitted consecutively - * so the first will have the lowest serialno. - * - * As this is not a typical list, we have to be careful with using the - * existing list helper. For example, using list_length on the first - * instruction will yield one less than the number of instructions in its - * group. */ - struct list_head rpt_node; + struct ir3_instruction *rpt_prev; + struct ir3_instruction *rpt_next; uint32_t serialno; @@ -2043,22 +2033,20 @@ __ssa_srcp_n(struct ir3_instruction *instr, unsigned n) /* Iterate over all instructions in a repeat group. */ #define foreach_instr_rpt(__rpt, __instr) \ if (assert(ir3_instr_is_first_rpt(__instr)), true) \ - for (struct ir3_instruction *__rpt = __instr, *__first = __instr; \ - __first || __rpt != __instr; \ - __first = NULL, __rpt = \ - list_entry(__rpt->rpt_node.next, \ - struct ir3_instruction, rpt_node)) + for (struct ir3_instruction *__rpt = __instr; __rpt; \ + __rpt = __rpt->rpt_next) /* Iterate over all instructions except the first one in a repeat group. */ #define foreach_instr_rpt_excl(__rpt, __instr) \ if (assert(ir3_instr_is_first_rpt(__instr)), true) \ - list_for_each_entry (struct ir3_instruction, __rpt, &__instr->rpt_node, \ - rpt_node) + for (struct ir3_instruction *__rpt = __instr->rpt_next; __rpt; \ + __rpt = __rpt->rpt_next) #define foreach_instr_rpt_excl_safe(__rpt, __instr) \ - if (assert(ir3_instr_is_first_rpt(__instr)), true) \ - list_for_each_entry_safe (struct ir3_instruction, __rpt, \ - &__instr->rpt_node, rpt_node) + if (assert(ir3_instr_is_first_rpt(__instr)), __instr->rpt_next) \ + for (struct ir3_instruction *__rpt = __instr->rpt_next, \ + *__next = __rpt->rpt_next; \ + __rpt; __rpt = __next, __next = __next ? __next->rpt_next : NULL) /* iterators for blocks: */ #define foreach_block(__block, __list) \ diff --git a/src/freedreno/ir3/ir3_rpt.c b/src/freedreno/ir3/ir3_rpt.c index fc0ffdfa535..294393dfd43 100644 --- a/src/freedreno/ir3/ir3_rpt.c +++ b/src/freedreno/ir3/ir3_rpt.c @@ -49,16 +49,6 @@ ir3_nir_vectorize_filter(const nir_instr *instr, const void *data) return 4; } -static void -rpt_list_split(struct list_head *list, struct list_head *at) -{ - struct list_head *new_last = at->prev; - new_last->next = list; - at->prev = list->prev; - list->prev->next = at; - list->prev = new_last; -} - static enum ir3_register_flags rpt_compatible_src_flags(struct ir3_register *src) { @@ -150,7 +140,8 @@ cleanup_rpt_instr(struct ir3_instruction *instr) unsigned rpt_n = 1; foreach_instr_rpt_excl (rpt, instr) { if (!can_rpt(instr, rpt, rpt_n++)) { - rpt_list_split(&instr->rpt_node, &rpt->rpt_node); + rpt->rpt_prev->rpt_next = NULL; + rpt->rpt_prev = NULL; /* We have to do this recursively since later repetitions might come * before the first in the instruction list. @@ -282,8 +273,16 @@ merge_instr(struct ir3_instruction *instr) * with the following instructions (if any) once we encounter it in * ir3_combine_rpt. */ - if (!try_merge(instr, rpt, rpt_n)) + if (!try_merge(instr, rpt, rpt_n)) { + /* Unlink rpt from this rpt group so that it becomes the first of a new + * one. Note that we don't need to unlink rpt when the merge succeeds + * since it will be removed in that case. + */ + assert(rpt->rpt_prev); + rpt->rpt_prev->rpt_next = NULL; + rpt->rpt_prev = NULL; break; + } instr->repeat++; @@ -299,12 +298,15 @@ merge_instr(struct ir3_instruction *instr) * remove it in ir3_combine_rpt when we encounter it. */ rpt->flags |= IR3_INSTR_MARK; - list_delinit(&rpt->rpt_node); ++rpt_n; progress = true; } - list_delinit(&instr->rpt_node); + if (instr->rpt_prev) + instr->rpt_prev->rpt_next = NULL; + + instr->rpt_prev = NULL; + instr->rpt_next = NULL; return progress; } @@ -321,7 +323,7 @@ ir3_merge_rpt(struct ir3 *ir, struct ir3_shader_variant *v) foreach_block (block, &ir->block_list) { foreach_instr_safe (instr, &block->instr_list) { if (instr->flags & IR3_INSTR_MARK) { - list_delinit(&instr->node); + ir3_instr_remove(instr); continue; }