ir3: don't use list_head for rpt groups

To link together instructions in a rpt group we currently (ab)use
list_head. This is a bit of a hack because we don't actually have a
list_head that points to the first instruction without being embedded in
an instruction itself (the way list_head is supposed to be used).
Instead, the list_head embedded in the first instruction of a rpt group
also serves as the one pointing to the list. In order make a distinction
between the first and last instruction (for which the main list_head
would usually be used), we rely on the fact that (currently)
instructions in a rpt group are emitted in order which means that later
instructions have a larger serialno than earlier ones.

In order to make all this less hacky, and to lift the restriction of
needing instructions to be emitted in order, replace the list_head with
explicit rpt_next/rpt_prev pointers which link the instructions together
in a doubly but non-circular linked list.

Signed-off-by: Job Noorman <jnoorman@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/38576>

src/freedreno/ir3/ir3.c

@@ -896,7 +896,6 @@ instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc)
    instr->srcs_max = nsrc;
 #endif
 
-   list_inithead(&instr->rpt_node);
    return instr;
 }
 
@@ -975,7 +974,8 @@ ir3_instr_clone(struct ir3_instruction *instr)
    new_instr->dsts = dsts;
    new_instr->srcs = srcs;
    new_instr->uses = NULL;
-   list_inithead(&new_instr->rpt_node);
+   new_instr->rpt_prev = NULL;
+   new_instr->rpt_next = NULL;
 
    insert_instr(ir3_before_terminator(instr->block), new_instr);
 
@@ -1020,7 +1020,14 @@ void
 ir3_instr_remove(struct ir3_instruction *instr)
 {
    list_delinit(&instr->node);
-   list_delinit(&instr->rpt_node);
+
+   if (instr->rpt_prev) {
+      instr->rpt_prev->rpt_next = instr->rpt_next;
+   }
+
+   if (instr->rpt_next) {
+      instr->rpt_next->rpt_prev = instr->rpt_prev;
+   }
 }
 
 void
@@ -1029,28 +1036,25 @@ ir3_instr_create_rpt(struct ir3_instruction **instrs, unsigned n)
    assert(n > 0 && !ir3_instr_is_rpt(instrs[0]));
 
    for (unsigned i = 1; i < n; ++i) {
-      assert(!ir3_instr_is_rpt(instrs[i]));
-      assert(instrs[i]->serialno > instrs[i - 1]->serialno);
+      struct ir3_instruction *instr = instrs[i];
+      struct ir3_instruction *prev = instrs[i - 1];
+      assert(!ir3_instr_is_rpt(instr));
 
-      list_addtail(&instrs[i]->rpt_node, &instrs[0]->rpt_node);
+      prev->rpt_next = instr;
+      instr->rpt_prev = prev;
    }
 }
 
 bool
 ir3_instr_is_rpt(const struct ir3_instruction *instr)
 {
-   return !list_is_empty(&instr->rpt_node);
+   return instr->rpt_prev || instr->rpt_next;
 }
 
 bool
 ir3_instr_is_first_rpt(const struct ir3_instruction *instr)
 {
-   if (!ir3_instr_is_rpt(instr))
-      return false;
-
-   struct ir3_instruction *prev_rpt =
-      list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node);
-   return prev_rpt->serialno > instr->serialno;
+   return instr->rpt_next && !instr->rpt_prev;
 }
 
 struct ir3_instruction *
@@ -1058,9 +1062,7 @@ ir3_instr_prev_rpt(const struct ir3_instruction *instr)
 {
    assert(ir3_instr_is_rpt(instr));
 
-   if (ir3_instr_is_first_rpt(instr))
-      return NULL;
-   return list_entry(instr->rpt_node.prev, struct ir3_instruction, rpt_node);
+   return instr->rpt_prev;
 }
 
 struct ir3_instruction *
@@ -1081,7 +1083,14 @@ ir3_instr_rpt_length(const struct ir3_instruction *instr)
 {
    assert(ir3_instr_is_first_rpt(instr));
 
-   return list_length(&instr->rpt_node) + 1;
+   unsigned length = 1;
+
+   while (instr->rpt_next) {
+      length++;
+      instr = instr->rpt_next;
+   }
+
+   return length;
 }
 
 struct ir3_register *

src/freedreno/ir3/ir3.h

@@ -609,22 +609,12 @@ struct ir3_instruction {
 
    /* List of this instruction's repeat group. Vectorized NIR instructions are
     * emitted as multiple scalar instructions that are linked together using
-    * this field. After RA, the ir3_combine_rpt pass iterates these groups and,
-    * if the register assignment allows it, merges them into a (rptN)
+    * these fields. After RA, the ir3_combine_rpt pass iterates these groups
+    * and, if the register assignment allows it, merges them into a (rptN)
     * instruction.
-    *
-    * NOTE: this is not a typical list as there is no empty list head. The list
-    * head is stored in the first instruction of the repeat group so also refers
-    * to a list entry. In order to distinguish the list's first entry, we use
-    * serialno: instructions in a repeat group are always emitted consecutively
-    * so the first will have the lowest serialno.
-    *
-    * As this is not a typical list, we have to be careful with using the
-    * existing list helper. For example, using list_length on the first
-    * instruction will yield one less than the number of instructions in its
-    * group.
     */
-   struct list_head rpt_node;
+   struct ir3_instruction *rpt_prev;
+   struct ir3_instruction *rpt_next;
 
    uint32_t serialno;
 
@@ -2043,22 +2033,20 @@ __ssa_srcp_n(struct ir3_instruction *instr, unsigned n)
 /* Iterate over all instructions in a repeat group. */
 #define foreach_instr_rpt(__rpt, __instr)                                      \
    if (assert(ir3_instr_is_first_rpt(__instr)), true)                          \
-      for (struct ir3_instruction *__rpt = __instr, *__first = __instr;        \
-           __first || __rpt != __instr;                                        \
-           __first = NULL, __rpt =                                             \
-                              list_entry(__rpt->rpt_node.next,                 \
-                                         struct ir3_instruction, rpt_node))
+      for (struct ir3_instruction *__rpt = __instr; __rpt;                     \
+           __rpt = __rpt->rpt_next)
 
 /* Iterate over all instructions except the first one in a repeat group. */
 #define foreach_instr_rpt_excl(__rpt, __instr)                                 \
    if (assert(ir3_instr_is_first_rpt(__instr)), true)                          \
-      list_for_each_entry (struct ir3_instruction, __rpt, &__instr->rpt_node,  \
-                           rpt_node)
+      for (struct ir3_instruction *__rpt = __instr->rpt_next; __rpt;           \
+           __rpt = __rpt->rpt_next)
 
 #define foreach_instr_rpt_excl_safe(__rpt, __instr)                            \
-   if (assert(ir3_instr_is_first_rpt(__instr)), true)                          \
-      list_for_each_entry_safe (struct ir3_instruction, __rpt,                 \
-                                &__instr->rpt_node, rpt_node)
+   if (assert(ir3_instr_is_first_rpt(__instr)), __instr->rpt_next)             \
+      for (struct ir3_instruction *__rpt = __instr->rpt_next,                  \
+                                  *__next = __rpt->rpt_next;                   \
+           __rpt; __rpt = __next, __next = __next ? __next->rpt_next : NULL)
 
 /* iterators for blocks: */
 #define foreach_block(__block, __list)                                         \

src/freedreno/ir3/ir3_rpt.c

@@ -49,16 +49,6 @@ ir3_nir_vectorize_filter(const nir_instr *instr, const void *data)
    return 4;
 }
 
-static void
-rpt_list_split(struct list_head *list, struct list_head *at)
-{
-   struct list_head *new_last = at->prev;
-   new_last->next = list;
-   at->prev = list->prev;
-   list->prev->next = at;
-   list->prev = new_last;
-}
-
 static enum ir3_register_flags
 rpt_compatible_src_flags(struct ir3_register *src)
 {
@@ -150,7 +140,8 @@ cleanup_rpt_instr(struct ir3_instruction *instr)
    unsigned rpt_n = 1;
    foreach_instr_rpt_excl (rpt, instr) {
       if (!can_rpt(instr, rpt, rpt_n++)) {
-         rpt_list_split(&instr->rpt_node, &rpt->rpt_node);
+         rpt->rpt_prev->rpt_next = NULL;
+         rpt->rpt_prev = NULL;
 
          /* We have to do this recursively since later repetitions might come
           * before the first in the instruction list.
@@ -282,8 +273,16 @@ merge_instr(struct ir3_instruction *instr)
        * with the following instructions (if any) once we encounter it in
        * ir3_combine_rpt.
        */
-      if (!try_merge(instr, rpt, rpt_n))
+      if (!try_merge(instr, rpt, rpt_n)) {
+         /* Unlink rpt from this rpt group so that it becomes the first of a new
+          * one. Note that we don't need to unlink rpt when the merge succeeds
+          * since it will be removed in that case.
+          */
+         assert(rpt->rpt_prev);
+         rpt->rpt_prev->rpt_next = NULL;
+         rpt->rpt_prev = NULL;
          break;
+      }
 
       instr->repeat++;
 
@@ -299,12 +298,15 @@ merge_instr(struct ir3_instruction *instr)
        * remove it in ir3_combine_rpt when we encounter it.
        */
       rpt->flags |= IR3_INSTR_MARK;
-      list_delinit(&rpt->rpt_node);
       ++rpt_n;
       progress = true;
    }
 
-   list_delinit(&instr->rpt_node);
+   if (instr->rpt_prev)
+      instr->rpt_prev->rpt_next = NULL;
+
+   instr->rpt_prev = NULL;
+   instr->rpt_next = NULL;
    return progress;
 }
 
@@ -321,7 +323,7 @@ ir3_merge_rpt(struct ir3 *ir, struct ir3_shader_variant *v)
    foreach_block (block, &ir->block_list) {
       foreach_instr_safe (instr, &block->instr_list) {
          if (instr->flags & IR3_INSTR_MARK) {
-            list_delinit(&instr->node);
+            ir3_instr_remove(instr);
             continue;
          }