ir3: Make tied sources/destinations part of the IR

Previously this was hard-coded for a6xx atomic instructions. However
we'll need a way for array destinations to point to the source with the
previous value of the array when we split them up. This is conceptually
the same as tied source/destinations for a6xx atomics, except that array
writes sometimes won't have a previous value to point to. So move this
into the IR so that it can be more dynamic. As a bonus we can move the
knowledge of a6xx atomics out of RA, where it's out-of-place, and into
the a6xx-specific code that creates them.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11469>

src/freedreno/ir3/ir3.h

@@ -185,6 +185,12 @@ struct ir3_register {
 	 */
 	struct ir3_register *def;
 
+	/* Pointer to another register in the instruction that must share the same
+	 * physical register. Each destination can be tied with one source, and
+	 * they must have "tied" pointing to each other.
+	 */
+	struct ir3_register *tied;
+
 	unsigned merge_set_offset;
 	struct ir3_merge_set *merge_set;
 	unsigned interval_start, interval_end;
@@ -603,6 +609,12 @@ struct ir3_register * ir3_reg_create(struct ir3_instruction *instr,
 struct ir3_register * ir3_reg_clone(struct ir3 *shader,
 		struct ir3_register *reg);
 
+static inline void ir3_reg_tie(struct ir3_register *dst, struct ir3_register *src)
+{
+	dst->tied = src;
+	src->tied = dst;
+}
+
 void ir3_instr_set_address(struct ir3_instruction *instr,
 		struct ir3_instruction *addr);
 

src/freedreno/ir3/ir3_a6xx.c

@@ -194,6 +194,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
 	array_insert(b, b->keeps, atomic);
 
 	atomic->regs[0]->wrmask = src1->regs[0]->wrmask;
+	ir3_reg_tie(atomic->regs[0], atomic->regs[3]);
 	struct ir3_instruction *split;
 	ir3_split_dest(b, &split, atomic, 0, 1);
 	return split;
@@ -345,6 +346,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
 	array_insert(b, b->keeps, atomic);
 
 	atomic->regs[0]->wrmask = src1->regs[0]->wrmask;
+	ir3_reg_tie(atomic->regs[0], atomic->regs[3]);
 	struct ir3_instruction *split;
 	ir3_split_dest(b, &split, atomic, 0, 1);
 	return split;

src/freedreno/ir3/ir3_print.c

@@ -203,6 +203,13 @@ static void print_reg_name(struct log_stream *stream, struct ir3_instruction *in
 	if (reg->flags & IR3_REG_R)
 		mesa_log_stream_printf(stream, "(r)");
 
+	/* Right now all instructions that use tied registers only have one
+	 * destination register, so we can just print (tied) as if it's a flag,
+	 * although it's more convenient for RA if it's a pointer.
+	 */
+	if (reg->tied)
+		printf("(tied)");
+
 	if (reg->flags & IR3_REG_SHARED)
 		mesa_log_stream_printf(stream, "s");
 	if (reg->flags & IR3_REG_HALF)

src/freedreno/ir3/ir3_ra.c

@@ -1097,7 +1097,7 @@ allocate_dst(struct ra_ctx *ctx, struct ir3_register *dst)
 {
 	struct ra_file *file = ra_get_file(ctx, dst);
 
-	struct ir3_register *tied = ra_dst_get_tied_src(ctx->compiler, dst);
+	struct ir3_register *tied = dst->tied;
 	if (tied) {
 		struct ra_interval *tied_interval = &ctx->intervals[tied->def->name];
 		struct ra_interval *dst_interval = &ctx->intervals[dst->name];
@@ -1138,7 +1138,7 @@ assign_src(struct ra_ctx *ctx, struct ir3_instruction *instr, struct ir3_registe
 
 	bool array_rmw = ra_reg_is_array_rmw(src);
 
-	struct ir3_register *tied = ra_src_get_tied_dst(ctx->compiler, instr, src);
+	struct ir3_register *tied = src->tied;
 	physreg_t physreg;
 	if (tied) {
 		struct ra_interval *tied_interval = &ctx->intervals[tied->name];

src/freedreno/ir3/ir3_ra.h

@@ -109,20 +109,6 @@ ra_reg_is_dst(const struct ir3_register *reg)
 		((reg->flags & IR3_REG_ARRAY) || reg->wrmask);
 }
 
-static inline struct ir3_register *
-ra_dst_get_tied_src(const struct ir3_compiler *compiler, struct ir3_register *dst)
-{
-	/* With the a6xx new cat6 encoding, the same register is used for the
-	 * value and destination of atomic operations.
-	 */
-	if (compiler->gpu_id >= 600 && is_atomic(dst->instr->opc) &&
-		(dst->instr->flags & IR3_INSTR_G)) {
-		return dst->instr->regs[3];
-	}
-
-	return NULL;
-}
-
 /* Iterators for sources and destinations which:
  * - Don't include fake sources (irrelevant for RA)
  * - Don't include non-SSA sources (immediates and constants, also irrelevant)
@@ -144,19 +130,6 @@ ra_dst_get_tied_src(const struct ir3_compiler *compiler, struct ir3_register *ds
 		for (unsigned __cnt = (__instr)->regs_count, __i = 0; __i < __cnt; __i++) \
 			if (ra_reg_is_dst((__srcreg = (__instr)->regs[__i])))
 
-static inline struct ir3_register *
-ra_src_get_tied_dst(const struct ir3_compiler *compiler,
-					struct ir3_instruction *instr,
-				    struct ir3_register *src)
-{
-	if (compiler->gpu_id >= 600 && is_atomic(instr->opc) &&
-		(instr->flags & IR3_INSTR_G) && src == instr->regs[3]) {
-		return instr->regs[0];
-	}
-
-	return NULL;
-}
-
 
 #define RA_HALF_SIZE (4 * 48)
 #define RA_FULL_SIZE (4 * 48 * 2)

src/freedreno/ir3/ir3_ra_validate.c

@@ -139,6 +139,8 @@ validate_simple(struct ra_val_ctx *ctx, struct ir3_instruction *instr)
 	ra_foreach_dst (dst, instr) {
 		unsigned dst_max = ra_reg_get_physreg(dst) + reg_size(dst);
 		validate_assert(ctx, dst_max <= get_file_size(ctx, dst));
+		if (dst->tied)
+			validate_assert(ctx, ra_reg_get_num(dst) == ra_reg_get_num(dst->tied));
 	}
 
 	ra_foreach_src (src, instr) {

src/freedreno/ir3/ir3_spill.c

@@ -251,8 +251,7 @@ handle_instr(struct ra_spill_ctx *ctx, struct ir3_instruction *instr)
 
 	ra_foreach_dst(dst, instr) {
 		if (!ra_reg_is_array_rmw(dst)) {
-			struct ir3_register *tied_src =
-				ra_dst_get_tied_src(ctx->compiler, dst);
+			struct ir3_register *tied_src = dst->tied;
 			if (tied_src && !(tied_src->flags & IR3_REG_FIRST_KILL))
 				insert_dst(ctx, dst);
 		}

src/freedreno/ir3/ir3_validate.c

@@ -98,6 +98,31 @@ validate_phi(struct ir3_validate_ctx *ctx, struct ir3_instruction *phi)
 	validate_assert(ctx, writes_gpr(phi));
 }
 
+static void
+validate_reg(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr,
+			 struct ir3_register *reg)
+{
+	if (reg->tied) {
+		validate_assert(ctx, reg->tied->tied == reg);
+		validate_assert(ctx, (reg->tied->flags & IR3_REG_DEST) !=
+						(reg->flags & IR3_REG_DEST));
+		validate_assert(ctx, reg_class_flags(reg->tied) == reg_class_flags(reg));
+		validate_assert(ctx, reg->tied->wrmask == reg->wrmask);
+		if (reg->flags & IR3_REG_ARRAY) {
+			validate_assert(ctx, reg->tied->array.base == reg->array.base);
+			validate_assert(ctx, reg->tied->size == reg->size);
+		}
+		bool found = false;
+		for (unsigned i = 0; i < instr->regs_count; i++) {
+			if (instr->regs[i] == reg->tied) {
+				found = true;
+				break;
+			}
+		}
+		validate_assert(ctx, found && "tied register not in the same instruction");
+	}
+}
+
 #define validate_reg_size(ctx, reg, type) \
 	validate_assert(ctx, type_size(type) == (((reg)->flags & IR3_REG_HALF) ? 16 : 32))
 
@@ -142,6 +167,12 @@ validate_instr(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr)
 
 		last_reg = reg;
 	}
+	
+	for (unsigned i = 0; i < instr->regs_count; i++) {
+		struct ir3_register *reg = instr->regs[i];
+
+		validate_reg(ctx, instr, reg);
+	}
 
 	_mesa_set_add(ctx->defs, instr);