From fda91b49d71dcb7b5368920c3bc97faad0848eb2 Mon Sep 17 00:00:00 2001 From: Job Noorman Date: Sat, 30 Nov 2024 17:42:25 +0100 Subject: [PATCH] ir3: refactor builders to use ir3_builder API All functions that used to take an ir3_block as argument to append instructions to now take an ir3_builder as argument. Add an ir3_builder field to ir3_context and replace all uses of ir3_context::block for creating instructions with ir3_context::build. Signed-off-by: Job Noorman Part-of: --- src/freedreno/ir3/ir3.h | 108 +++--- src/freedreno/ir3/ir3_a4xx.c | 26 +- src/freedreno/ir3/ir3_a6xx.c | 34 +- src/freedreno/ir3/ir3_array_to_ssa.c | 4 +- src/freedreno/ir3/ir3_compiler_nir.c | 393 +++++++++++---------- src/freedreno/ir3/ir3_context.c | 64 ++-- src/freedreno/ir3/ir3_context.h | 16 +- src/freedreno/ir3/ir3_image.c | 6 +- src/freedreno/ir3/ir3_legalize.c | 54 +-- src/freedreno/ir3/ir3_lower_parallelcopy.c | 31 +- src/freedreno/ir3/ir3_lower_shared_phi.c | 6 +- src/freedreno/ir3/ir3_lower_spill.c | 8 +- src/freedreno/ir3/ir3_lower_subgroups.c | 17 +- src/freedreno/ir3/ir3_merge_regs.c | 2 +- src/freedreno/ir3/ir3_ra.c | 8 +- src/freedreno/ir3/ir3_shared_folding.c | 10 +- src/freedreno/ir3/ir3_shared_ra.c | 37 +- src/freedreno/ir3/ir3_spill.c | 3 +- 18 files changed, 406 insertions(+), 421 deletions(-) diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index 7f22b368e00..71192a3d167 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -2387,12 +2387,12 @@ type_flags(type_t type) } static inline struct ir3_instruction * -create_immed_typed_shared(struct ir3_block *block, uint32_t val, type_t type, bool shared) +create_immed_typed_shared(struct ir3_builder *build, uint32_t val, type_t type, bool shared) { struct ir3_instruction *mov; ir3_register_flags flags = type_flags(type); - mov = ir3_instr_create(block, OPC_MOV, 1, 1); + mov = ir3_build_instr(build, OPC_MOV, 1, 1); mov->cat1.src_type = type; mov->cat1.dst_type = type; __ssa_dst(mov)->flags |= flags | (shared ? IR3_REG_SHARED : 0); @@ -2402,30 +2402,30 @@ create_immed_typed_shared(struct ir3_block *block, uint32_t val, type_t type, bo } static inline struct ir3_instruction * -create_immed_typed(struct ir3_block *block, uint32_t val, type_t type) +create_immed_typed(struct ir3_builder *build, uint32_t val, type_t type) { - return create_immed_typed_shared(block, val, type, false); + return create_immed_typed_shared(build, val, type, false); } static inline struct ir3_instruction * -create_immed_shared(struct ir3_block *block, uint32_t val, bool shared) +create_immed_shared(struct ir3_builder *build, uint32_t val, bool shared) { - return create_immed_typed_shared(block, val, TYPE_U32, shared); + return create_immed_typed_shared(build, val, TYPE_U32, shared); } static inline struct ir3_instruction * -create_immed(struct ir3_block *block, uint32_t val) +create_immed(struct ir3_builder *build, uint32_t val) { - return create_immed_shared(block, val, false); + return create_immed_shared(build, val, false); } static inline struct ir3_instruction * -create_uniform_typed(struct ir3_block *block, unsigned n, type_t type) +create_uniform_typed(struct ir3_builder *build, unsigned n, type_t type) { struct ir3_instruction *mov; ir3_register_flags flags = type_flags(type); - mov = ir3_instr_create(block, OPC_MOV, 1, 1); + mov = ir3_build_instr(build, OPC_MOV, 1, 1); mov->cat1.src_type = type; mov->cat1.dst_type = type; __ssa_dst(mov)->flags |= flags; @@ -2435,18 +2435,18 @@ create_uniform_typed(struct ir3_block *block, unsigned n, type_t type) } static inline struct ir3_instruction * -create_uniform(struct ir3_block *block, unsigned n) +create_uniform(struct ir3_builder *build, unsigned n) { - return create_uniform_typed(block, n, TYPE_F32); + return create_uniform_typed(build, n, TYPE_F32); } static inline struct ir3_instruction * -create_uniform_indirect(struct ir3_block *block, int n, type_t type, +create_uniform_indirect(struct ir3_builder *build, int n, type_t type, struct ir3_instruction *address) { struct ir3_instruction *mov; - mov = ir3_instr_create(block, OPC_MOV, 1, 1); + mov = ir3_build_instr(build, OPC_MOV, 1, 1); mov->cat1.src_type = type; mov->cat1.dst_type = type; __ssa_dst(mov); @@ -2458,9 +2458,9 @@ create_uniform_indirect(struct ir3_block *block, int n, type_t type, } static inline struct ir3_instruction * -ir3_MOV(struct ir3_block *block, struct ir3_instruction *src, type_t type) +ir3_MOV(struct ir3_builder *build, struct ir3_instruction *src, type_t type) { - struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV, 1, 1); + struct ir3_instruction *instr = ir3_build_instr(build, OPC_MOV, 1, 1); ir3_register_flags flags = type_flags(type) | (src->dsts[0]->flags & IR3_REG_SHARED); __ssa_dst(instr)->flags |= flags; @@ -2477,24 +2477,24 @@ ir3_MOV(struct ir3_block *block, struct ir3_instruction *src, type_t type) } static inline struct ir3_instruction_rpt -ir3_MOV_rpt(struct ir3_block *block, unsigned nrpt, +ir3_MOV_rpt(struct ir3_builder *build, unsigned nrpt, struct ir3_instruction_rpt src, type_t type) { struct ir3_instruction_rpt dst; assert(nrpt <= ARRAY_SIZE(dst.rpts)); for (unsigned rpt = 0; rpt < nrpt; ++rpt) - dst.rpts[rpt] = ir3_MOV(block, src.rpts[rpt], type); + dst.rpts[rpt] = ir3_MOV(build, src.rpts[rpt], type); ir3_instr_create_rpt(dst.rpts, nrpt); return dst; } static inline struct ir3_instruction * -ir3_COV(struct ir3_block *block, struct ir3_instruction *src, type_t src_type, +ir3_COV(struct ir3_builder *build, struct ir3_instruction *src, type_t src_type, type_t dst_type) { - struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOV, 1, 1); + struct ir3_instruction *instr = ir3_build_instr(build, OPC_MOV, 1, 1); ir3_register_flags dst_flags = type_flags(dst_type) | (src->dsts[0]->flags & IR3_REG_SHARED); ASSERTED ir3_register_flags src_flags = type_flags(src_type); @@ -2509,22 +2509,22 @@ ir3_COV(struct ir3_block *block, struct ir3_instruction *src, type_t src_type, } static inline struct ir3_instruction_rpt -ir3_COV_rpt(struct ir3_block *block, unsigned nrpt, +ir3_COV_rpt(struct ir3_builder *build, unsigned nrpt, struct ir3_instruction_rpt src, type_t src_type, type_t dst_type) { struct ir3_instruction_rpt dst; for (unsigned rpt = 0; rpt < nrpt; ++rpt) - dst.rpts[rpt] = ir3_COV(block, src.rpts[rpt], src_type, dst_type); + dst.rpts[rpt] = ir3_COV(build, src.rpts[rpt], src_type, dst_type); ir3_instr_create_rpt(dst.rpts, nrpt); return dst; } static inline struct ir3_instruction * -ir3_MOVMSK(struct ir3_block *block, unsigned components) +ir3_MOVMSK(struct ir3_builder *build, unsigned components) { - struct ir3_instruction *instr = ir3_instr_create(block, OPC_MOVMSK, 1, 0); + struct ir3_instruction *instr = ir3_build_instr(build, OPC_MOVMSK, 1, 0); struct ir3_register *dst = __ssa_dst(instr); dst->flags |= IR3_REG_SHARED; @@ -2534,11 +2534,11 @@ ir3_MOVMSK(struct ir3_block *block, unsigned components) } static inline struct ir3_instruction * -ir3_BALLOT_MACRO(struct ir3_block *block, struct ir3_instruction *src, +ir3_BALLOT_MACRO(struct ir3_builder *build, struct ir3_instruction *src, unsigned components) { struct ir3_instruction *instr = - ir3_instr_create(block, OPC_BALLOT_MACRO, 1, 1); + ir3_build_instr(build, OPC_BALLOT_MACRO, 1, 1); struct ir3_register *dst = __ssa_dst(instr); dst->flags |= IR3_REG_SHARED; @@ -2551,9 +2551,9 @@ ir3_BALLOT_MACRO(struct ir3_block *block, struct ir3_instruction *src, /* clang-format off */ #define __INSTR0(flag, name, opc) \ -static inline struct ir3_instruction *ir3_##name(struct ir3_block *block) \ +static inline struct ir3_instruction *ir3_##name(struct ir3_builder *build) \ { \ - struct ir3_instruction *instr = ir3_instr_create(block, opc, 1, 0); \ + struct ir3_instruction *instr = ir3_build_instr(build, opc, 1, 0); \ instr->flags |= flag; \ return instr; \ } @@ -2564,10 +2564,10 @@ static inline struct ir3_instruction *ir3_##name(struct ir3_block *block) \ /* clang-format off */ #define __INSTR1(flag, dst_count, name, opc, scalar_alu) \ static inline struct ir3_instruction *ir3_##name( \ - struct ir3_block *block, struct ir3_instruction *a, unsigned aflags) \ + struct ir3_builder *build, struct ir3_instruction *a, unsigned aflags) \ { \ struct ir3_instruction *instr = \ - ir3_instr_create(block, opc, dst_count, 1); \ + ir3_build_instr(build, opc, dst_count, 1); \ unsigned dst_flag = scalar_alu ? (a->dsts[0]->flags & IR3_REG_SHARED) : 0; \ for (unsigned i = 0; i < dst_count; i++) \ __ssa_dst(instr)->flags |= dst_flag; \ @@ -2576,13 +2576,13 @@ static inline struct ir3_instruction *ir3_##name( \ return instr; \ } \ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ - struct ir3_block *block, unsigned nrpt, \ + struct ir3_builder *build, unsigned nrpt, \ struct ir3_instruction_rpt a, unsigned aflags) \ { \ struct ir3_instruction_rpt dst; \ assert(nrpt <= ARRAY_SIZE(dst.rpts)); \ for (unsigned rpt = 0; rpt < nrpt; rpt++) \ - dst.rpts[rpt] = ir3_##name(block, a.rpts[rpt], aflags); \ + dst.rpts[rpt] = ir3_##name(build, a.rpts[rpt], aflags); \ ir3_instr_create_rpt(dst.rpts, nrpt); \ return dst; \ } @@ -2597,10 +2597,10 @@ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ /* clang-format off */ #define __INSTR2(flag, dst_count, name, opc, scalar_alu) \ static inline struct ir3_instruction *ir3_##name( \ - struct ir3_block *block, struct ir3_instruction *a, unsigned aflags, \ + struct ir3_builder *build, struct ir3_instruction *a, unsigned aflags, \ struct ir3_instruction *b, unsigned bflags) \ { \ - struct ir3_instruction *instr = ir3_instr_create(block, opc, dst_count, 2); \ + struct ir3_instruction *instr = ir3_build_instr(build, opc, dst_count, 2); \ unsigned dst_flag = scalar_alu ? (a->dsts[0]->flags & b->dsts[0]->flags & \ IR3_REG_SHARED) : 0; \ for (unsigned i = 0; i < dst_count; i++) \ @@ -2611,14 +2611,14 @@ static inline struct ir3_instruction *ir3_##name( \ return instr; \ } \ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ - struct ir3_block *block, unsigned nrpt, \ + struct ir3_builder *build, unsigned nrpt, \ struct ir3_instruction_rpt a, unsigned aflags, \ struct ir3_instruction_rpt b, unsigned bflags) \ { \ struct ir3_instruction_rpt dst; \ assert(nrpt <= ARRAY_SIZE(dst.rpts)); \ for (unsigned rpt = 0; rpt < nrpt; rpt++) { \ - dst.rpts[rpt] = ir3_##name(block, a.rpts[rpt], aflags, \ + dst.rpts[rpt] = ir3_##name(build, a.rpts[rpt], aflags, \ b.rpts[rpt], bflags); \ } \ ir3_instr_create_rpt(dst.rpts, nrpt); \ @@ -2634,12 +2634,12 @@ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ /* clang-format off */ #define __INSTR3(flag, dst_count, name, opc, scalar_alu) \ static inline struct ir3_instruction *ir3_##name( \ - struct ir3_block *block, struct ir3_instruction *a, unsigned aflags, \ + struct ir3_builder *build, struct ir3_instruction *a, unsigned aflags, \ struct ir3_instruction *b, unsigned bflags, struct ir3_instruction *c, \ unsigned cflags) \ { \ struct ir3_instruction *instr = \ - ir3_instr_create(block, opc, dst_count, 3); \ + ir3_build_instr(build, opc, dst_count, 3); \ unsigned dst_flag = scalar_alu ? (a->dsts[0]->flags & b->dsts[0]->flags & \ c->dsts[0]->flags & IR3_REG_SHARED) : 0; \ for (unsigned i = 0; i < dst_count; i++) \ @@ -2651,7 +2651,7 @@ static inline struct ir3_instruction *ir3_##name( \ return instr; \ } \ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ - struct ir3_block *block, unsigned nrpt, \ + struct ir3_builder *build, unsigned nrpt, \ struct ir3_instruction_rpt a, unsigned aflags, \ struct ir3_instruction_rpt b, unsigned bflags, \ struct ir3_instruction_rpt c, unsigned cflags) \ @@ -2659,7 +2659,7 @@ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ struct ir3_instruction_rpt dst; \ assert(nrpt <= ARRAY_SIZE(dst.rpts)); \ for (unsigned rpt = 0; rpt < nrpt; rpt++) { \ - dst.rpts[rpt] = ir3_##name(block, a.rpts[rpt], aflags, \ + dst.rpts[rpt] = ir3_##name(build, a.rpts[rpt], aflags, \ b.rpts[rpt], bflags, \ c.rpts[rpt], cflags); \ } \ @@ -2676,12 +2676,12 @@ static inline struct ir3_instruction_rpt ir3_##name##_rpt( \ /* clang-format off */ #define __INSTR4(flag, dst_count, name, opc) \ static inline struct ir3_instruction *ir3_##name( \ - struct ir3_block *block, struct ir3_instruction *a, unsigned aflags, \ + struct ir3_builder *build, struct ir3_instruction *a, unsigned aflags, \ struct ir3_instruction *b, unsigned bflags, struct ir3_instruction *c, \ unsigned cflags, struct ir3_instruction *d, unsigned dflags) \ { \ struct ir3_instruction *instr = \ - ir3_instr_create(block, opc, dst_count, 4); \ + ir3_build_instr(build, opc, dst_count, 4); \ for (unsigned i = 0; i < dst_count; i++) \ __ssa_dst(instr); \ __ssa_src(instr, a, aflags); \ @@ -2699,12 +2699,12 @@ static inline struct ir3_instruction *ir3_##name( \ /* clang-format off */ #define __INSTR5(flag, name, opc) \ static inline struct ir3_instruction *ir3_##name( \ - struct ir3_block *block, struct ir3_instruction *a, unsigned aflags, \ + struct ir3_builder *build, struct ir3_instruction *a, unsigned aflags, \ struct ir3_instruction *b, unsigned bflags, struct ir3_instruction *c, \ unsigned cflags, struct ir3_instruction *d, unsigned dflags, \ struct ir3_instruction *e, unsigned eflags) \ { \ - struct ir3_instruction *instr = ir3_instr_create(block, opc, 1, 5); \ + struct ir3_instruction *instr = ir3_build_instr(build, opc, 1, 5); \ __ssa_dst(instr); \ __ssa_src(instr, a, aflags); \ __ssa_src(instr, b, bflags); \ @@ -2721,13 +2721,13 @@ static inline struct ir3_instruction *ir3_##name( \ /* clang-format off */ #define __INSTR6(flag, dst_count, name, opc) \ static inline struct ir3_instruction *ir3_##name( \ - struct ir3_block *block, struct ir3_instruction *a, unsigned aflags, \ + struct ir3_builder *build, struct ir3_instruction *a, unsigned aflags, \ struct ir3_instruction *b, unsigned bflags, struct ir3_instruction *c, \ unsigned cflags, struct ir3_instruction *d, unsigned dflags, \ struct ir3_instruction *e, unsigned eflags, struct ir3_instruction *f, \ unsigned fflags) \ { \ - struct ir3_instruction *instr = ir3_instr_create(block, opc, 1, 6); \ + struct ir3_instruction *instr = ir3_build_instr(build, opc, 1, 6); \ for (unsigned i = 0; i < dst_count; i++) \ __ssa_dst(instr); \ __ssa_src(instr, a, aflags); \ @@ -2773,19 +2773,19 @@ INSTR2(READ_COND_MACRO) INSTR1(READ_GETLAST_MACRO) static inline struct ir3_instruction * -ir3_ELECT_MACRO(struct ir3_block *block) +ir3_ELECT_MACRO(struct ir3_builder *build) { struct ir3_instruction *instr = - ir3_instr_create(block, OPC_ELECT_MACRO, 1, 0); + ir3_build_instr(build, OPC_ELECT_MACRO, 1, 0); __ssa_dst(instr); return instr; } static inline struct ir3_instruction * -ir3_SHPS_MACRO(struct ir3_block *block) +ir3_SHPS_MACRO(struct ir3_builder *build) { struct ir3_instruction *instr = - ir3_instr_create(block, OPC_SHPS_MACRO, 1, 0); + ir3_build_instr(build, OPC_SHPS_MACRO, 1, 0); __ssa_dst(instr); return instr; } @@ -2887,7 +2887,7 @@ INSTR1F(3D, DSY) INSTR1(RGETPOS) static inline struct ir3_instruction * -ir3_SAM(struct ir3_block *block, opc_t opc, type_t type, unsigned wrmask, +ir3_SAM(struct ir3_builder *build, opc_t opc, type_t type, unsigned wrmask, ir3_instruction_flags flags, struct ir3_instruction *samp_tex, struct ir3_instruction *src0, struct ir3_instruction *src1) { @@ -2904,7 +2904,7 @@ ir3_SAM(struct ir3_block *block, opc_t opc, type_t type, unsigned wrmask, nreg++; } - sam = ir3_instr_create(block, opc, 1, nreg); + sam = ir3_build_instr(build, opc, 1, nreg); sam->flags |= flags; __ssa_dst(sam)->wrmask = wrmask; if (flags & IR3_INSTR_S2EN) { @@ -2932,12 +2932,12 @@ ir3_SAM(struct ir3_block *block, opc_t opc, type_t type, unsigned wrmask, * argument (the initial value of rx) and tie it to the destination. */ static inline struct ir3_instruction * -ir3_BRCST_ACTIVE(struct ir3_block *block, unsigned cluster_size, +ir3_BRCST_ACTIVE(struct ir3_builder *build, unsigned cluster_size, struct ir3_instruction *src, struct ir3_instruction *dst_default) { struct ir3_instruction *brcst = - ir3_instr_create(block, OPC_BRCST_ACTIVE, 1, 2); + ir3_build_instr(build, OPC_BRCST_ACTIVE, 1, 2); brcst->cat5.cluster_size = cluster_size; brcst->cat5.type = TYPE_U32; struct ir3_register *brcst_dst = __ssa_dst(brcst); diff --git a/src/freedreno/ir3/ir3_a4xx.c b/src/freedreno/ir3/ir3_a4xx.c index 58731e4b58d..a87735fa3d4 100644 --- a/src/freedreno/ir3/ir3_a4xx.c +++ b/src/freedreno/ir3/ir3_a4xx.c @@ -32,7 +32,7 @@ byte_offset_to_address(struct ir3_context *ctx, nir_src *ssbo, struct ir3_instruction *byte_offset) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; if (ctx->compiler->gen == 4) { uint32_t index = nir_src_as_uint(*ssbo); @@ -52,7 +52,7 @@ static void emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *ldgb, *src0, *src1, *byte_offset, *offset; struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]); @@ -79,7 +79,7 @@ emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr, static void emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *stgb, *src0, *src1, *src2, *byte_offset, *offset; unsigned wrmask = nir_intrinsic_write_mask(intr); unsigned ncomp = ffs(~wrmask) - 1; @@ -105,11 +105,11 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) stgb->barrier_class = IR3_BARRIER_BUFFER_W; stgb->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W; - array_insert(b, b->keeps, stgb); + array_insert(ctx->block, ctx->block->keeps, stgb); } static struct ir3_instruction * -emit_atomic(struct ir3_block *b, +emit_atomic(struct ir3_builder *b, nir_atomic_op op, struct ir3_instruction *bo, struct ir3_instruction *data, @@ -167,7 +167,7 @@ emit_atomic(struct ir3_block *b, static struct ir3_instruction * emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; nir_atomic_op op = nir_intrinsic_atomic_op(intr); type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32; @@ -196,7 +196,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W; /* even if nothing consume the result, we can't DCE the instruction: */ - array_insert(b, b->keeps, atomic); + array_insert(ctx->block, ctx->block->keeps, atomic); return atomic; } @@ -205,7 +205,7 @@ static struct ir3_instruction * get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr, struct ir3_instruction *const *coords, bool byteoff) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *offset; unsigned index = nir_src_as_uint(instr->src[0]); unsigned ncoords = ir3_get_image_coords(instr, NULL); @@ -264,7 +264,7 @@ static void emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]); struct ir3_instruction *ibo = ir3_image_to_ibo(ctx, intr->src[0]); struct ir3_instruction *offset = get_image_offset(ctx, intr, coords, true); @@ -311,7 +311,7 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, static void emit_intrinsic_store_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *stib, *offset; struct ir3_instruction *const *value = ir3_get_src(ctx, &intr->src[3]); struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]); @@ -341,14 +341,14 @@ emit_intrinsic_store_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) stib->barrier_class = IR3_BARRIER_IMAGE_W; stib->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W; - array_insert(b, b->keeps, stib); + array_insert(ctx->block, ctx->block->keeps, stib); } /* src[] = { deref, coord, sample_index, value, compare }. const_index[] = {} */ static struct ir3_instruction * emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *atomic, *src0, *src1, *src2; struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]); struct ir3_instruction *image = ir3_image_to_ibo(ctx, intr->src[0]); @@ -375,7 +375,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W; /* even if nothing consume the result, we can't DCE the instruction: */ - array_insert(b, b->keeps, atomic); + array_insert(ctx->block, ctx->block->keeps, atomic); return atomic; } diff --git a/src/freedreno/ir3/ir3_a6xx.c b/src/freedreno/ir3/ir3_a6xx.c index d1aace1af02..ca456e58290 100644 --- a/src/freedreno/ir3/ir3_a6xx.c +++ b/src/freedreno/ir3/ir3_a6xx.c @@ -38,7 +38,7 @@ static void emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *offset; struct ir3_instruction *ldib; unsigned imm_offset_val; @@ -87,7 +87,7 @@ emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr, static void emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *stib, *val, *offset; unsigned wrmask = nir_intrinsic_write_mask(intr); unsigned ncomp = ffs(~wrmask) - 1; @@ -146,11 +146,11 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) ir3_handle_bindless_cat6(stib, intr->src[1]); ir3_handle_nonuniform(stib, intr); - array_insert(b, b->keeps, stib); + array_insert(ctx->block, ctx->block->keeps, stib); } static struct ir3_instruction * -emit_atomic(struct ir3_block *b, +emit_atomic(struct ir3_builder *b, nir_atomic_op op, struct ir3_instruction *ibo, struct ir3_instruction *src0, @@ -202,7 +202,7 @@ emit_atomic(struct ir3_block *b, static struct ir3_instruction * emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *atomic, *ibo, *src0, *src1, *data, *dummy; nir_atomic_op op = nir_intrinsic_atomic_op(intr); type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32; @@ -261,7 +261,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) ir3_handle_bindless_cat6(atomic, intr->src[0]); /* even if nothing consume the result, we can't DCE the instruction: */ - array_insert(b, b->keeps, atomic); + array_insert(ctx->block, ctx->block->keeps, atomic); atomic->dsts[0]->wrmask = src1->dsts[0]->wrmask; ir3_reg_tie(atomic->dsts[0], atomic->srcs[2]); @@ -278,7 +278,7 @@ static void emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *ldib; struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]); unsigned ncoords = ir3_get_image_coords(intr, NULL); @@ -303,7 +303,7 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, static void emit_intrinsic_store_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *stib; struct ir3_instruction *const *value = ir3_get_src(ctx, &intr->src[3]); struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]); @@ -326,14 +326,14 @@ emit_intrinsic_store_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) ir3_handle_bindless_cat6(stib, intr->src[0]); ir3_handle_nonuniform(stib, intr); - array_insert(b, b->keeps, stib); + array_insert(ctx->block, ctx->block->keeps, stib); } /* src[] = { deref, coord, sample_index, value, compare }. const_index[] = {} */ static struct ir3_instruction * emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *atomic, *ibo, *src0, *src1, *dummy; struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]); struct ir3_instruction *value = ir3_get_src(ctx, &intr->src[3])[0]; @@ -374,7 +374,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) ir3_handle_bindless_cat6(atomic, intr->src[0]); /* even if nothing consume the result, we can't DCE the instruction: */ - array_insert(b, b->keeps, atomic); + array_insert(ctx->block, ctx->block->keeps, atomic); atomic->dsts[0]->wrmask = src1->dsts[0]->wrmask; ir3_reg_tie(atomic->dsts[0], atomic->srcs[2]); @@ -388,7 +388,7 @@ static void emit_intrinsic_image_size(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *ibo = ir3_image_to_ibo(ctx, intr->src[0]); struct ir3_instruction *resinfo = ir3_RESINFO(b, ibo, 0); resinfo->cat6.iim_val = 1; @@ -409,7 +409,7 @@ emit_intrinsic_load_global_ir3(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; unsigned dest_components = nir_intrinsic_dest_components(intr); struct ir3_instruction *addr, *offset; @@ -452,7 +452,7 @@ static void emit_intrinsic_store_global_ir3(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *value, *addr, *offset; unsigned ncomp = nir_intrinsic_src_components(intr, 0); @@ -486,7 +486,7 @@ emit_intrinsic_store_global_ir3(struct ir3_context *ctx, stg->cat6.type = type_uint_size(intr->src[0].ssa->bit_size); stg->cat6.iim_val = 1; - array_insert(b, b->keeps, stg); + array_insert(ctx->block, ctx->block->keeps, stg); stg->barrier_class = IR3_BARRIER_BUFFER_W; stg->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W; @@ -495,7 +495,7 @@ emit_intrinsic_store_global_ir3(struct ir3_context *ctx, static struct ir3_instruction * emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *addr, *atomic, *src1; struct ir3_instruction *value = ir3_get_src(ctx, &intr->src[1])[0]; nir_atomic_op op = nir_intrinsic_atomic_op(intr); @@ -571,7 +571,7 @@ emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->dsts[0]->wrmask = MASK(intr->def.bit_size == 64 ? 2 : 1); /* even if nothing consume the result, we can't DCE the instruction: */ - array_insert(b, b->keeps, atomic); + array_insert(ctx->block, ctx->block->keeps, atomic); return atomic; } diff --git a/src/freedreno/ir3/ir3_array_to_ssa.c b/src/freedreno/ir3/ir3_array_to_ssa.c index cd0a92582e9..dc461bb8d38 100644 --- a/src/freedreno/ir3/ir3_array_to_ssa.c +++ b/src/freedreno/ir3/ir3_array_to_ssa.c @@ -86,9 +86,7 @@ read_value_beginning(struct array_ctx *ctx, struct ir3_block *block, unsigned flags = IR3_REG_ARRAY | (arr->half ? IR3_REG_HALF : 0); struct ir3_instruction *phi = - ir3_instr_create(block, OPC_META_PHI, 1, block->predecessors_count); - list_del(&phi->node); - list_add(&phi->node, &block->instr_list); + ir3_instr_create_at(ir3_before_block(block), OPC_META_PHI, 1, block->predecessors_count); struct ir3_register *dst = __ssa_dst(phi); dst->flags |= flags; diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c index fac4bbba217..f72081c88a8 100644 --- a/src/freedreno/ir3/ir3_compiler_nir.c +++ b/src/freedreno/ir3/ir3_compiler_nir.c @@ -41,30 +41,30 @@ cp_instrs(struct ir3_instruction *dst[], struct ir3_instruction *instrs[], } static struct ir3_instruction_rpt -create_immed_rpt(struct ir3_block *block, unsigned nrpt, unsigned val) +create_immed_rpt(struct ir3_builder *build, unsigned nrpt, unsigned val) { - return rpt_instr(create_immed(block, val), nrpt); + return rpt_instr(create_immed(build, val), nrpt); } static struct ir3_instruction_rpt -create_immed_shared_rpt(struct ir3_block *block, unsigned nrpt, uint32_t val, +create_immed_shared_rpt(struct ir3_builder *build, unsigned nrpt, uint32_t val, bool shared) { - return rpt_instr(create_immed_shared(block, val, shared), nrpt); + return rpt_instr(create_immed_shared(build, val, shared), nrpt); } static struct ir3_instruction_rpt -create_immed_typed_rpt(struct ir3_block *block, unsigned nrpt, unsigned val, +create_immed_typed_rpt(struct ir3_builder *build, unsigned nrpt, unsigned val, type_t type) { - return rpt_instr(create_immed_typed(block, val, type), nrpt); + return rpt_instr(create_immed_typed(build, val, type), nrpt); } static inline struct ir3_instruction_rpt -create_immed_typed_shared_rpt(struct ir3_block *block, unsigned nrpt, +create_immed_typed_shared_rpt(struct ir3_builder *build, unsigned nrpt, uint32_t val, type_t type, bool shared) { - return rpt_instr(create_immed_typed_shared(block, val, type, shared), nrpt); + return rpt_instr(create_immed_typed_shared(build, val, type, shared), nrpt); } static void @@ -124,7 +124,7 @@ create_input(struct ir3_context *ctx, unsigned compmask) { struct ir3_instruction *in; - in = ir3_instr_create(ctx->in_block, OPC_META_INPUT, 1, 0); + in = ir3_instr_create_at(ir3_before_terminator(ctx->in_block), OPC_META_INPUT, 1, 0); in->input.sysval = ~0; __ssa_dst(in)->wrmask = compmask; @@ -137,30 +137,30 @@ static struct ir3_instruction_rpt create_frag_input(struct ir3_context *ctx, struct ir3_instruction *coord, unsigned n, unsigned ncomp) { - struct ir3_block *block = ctx->block; + struct ir3_builder *build = &ctx->build; struct ir3_instruction_rpt instr; /* packed inloc is fixed up later: */ struct ir3_instruction_rpt inloc; for (unsigned i = 0; i < ncomp; i++) - inloc.rpts[i] = create_immed(block, n + i); + inloc.rpts[i] = create_immed(build, n + i); if (coord) { instr = - ir3_BARY_F_rpt(block, ncomp, inloc, 0, rpt_instr(coord, ncomp), 0); + ir3_BARY_F_rpt(build, ncomp, inloc, 0, rpt_instr(coord, ncomp), 0); } else if (ctx->compiler->flat_bypass) { if (ctx->compiler->gen >= 6) { - instr = ir3_FLAT_B_rpt(block, ncomp, inloc, 0, inloc, 0); + instr = ir3_FLAT_B_rpt(build, ncomp, inloc, 0, inloc, 0); } else { for (unsigned i = 0; i < ncomp; i++) { instr.rpts[i] = - ir3_LDLV(block, inloc.rpts[i], 0, create_immed(block, 1), 0); + ir3_LDLV(build, inloc.rpts[i], 0, create_immed(build, 1), 0); instr.rpts[i]->cat6.type = TYPE_U32; instr.rpts[i]->cat6.iim_val = 1; } } } else { - instr = ir3_BARY_F_rpt(block, ncomp, inloc, 0, + instr = ir3_BARY_F_rpt(build, ncomp, inloc, 0, rpt_instr(ctx->ij[IJ_PERSP_PIXEL], ncomp), 0); for (unsigned i = 0; i < ncomp; i++) @@ -178,7 +178,7 @@ create_driver_param(struct ir3_context *ctx, uint32_t dp) const struct ir3_const_state *const_state = ir3_const_state(ctx->so); unsigned n = const_state->offsets.driver_param; unsigned r = regid(n + dp / 4, dp % 4); - return create_uniform(ctx->block, r); + return create_uniform(&ctx->build, r); } static struct ir3_instruction * @@ -189,7 +189,7 @@ create_driver_param_indirect(struct ir3_context *ctx, uint32_t dp, /* NOTE: dp is in scalar, but there can be >4 dp components: */ const struct ir3_const_state *const_state = ir3_const_state(ctx->so); unsigned n = const_state->offsets.driver_param; - return create_uniform_indirect(ctx->block, n * 4 + dp, TYPE_U32, address); + return create_uniform_indirect(&ctx->build, n * 4 + dp, TYPE_U32, address); } /* @@ -347,9 +347,9 @@ create_cov(struct ir3_context *ctx, unsigned nrpt, */ if (src_type == TYPE_U8 && full_type(dst_type) == TYPE_U32) { struct ir3_instruction_rpt mask = - create_immed_typed_rpt(ctx->block, nrpt, 0xff, TYPE_U8); + create_immed_typed_rpt(&ctx->build, nrpt, 0xff, TYPE_U8); struct ir3_instruction_rpt cov = - ir3_AND_B_rpt(ctx->block, nrpt, src, 0, mask, 0); + ir3_AND_B_rpt(&ctx->build, nrpt, src, 0, mask, 0); set_dst_flags(cov.rpts, nrpt, type_flags(dst_type)); return cov; } @@ -365,13 +365,13 @@ create_cov(struct ir3_context *ctx, unsigned nrpt, struct ir3_instruction_rpt cov; if (op == nir_op_u2f16 || op == nir_op_u2f32) { struct ir3_instruction_rpt mask = - create_immed_typed_rpt(ctx->block, nrpt, 0xff, TYPE_U8); - cov = ir3_AND_B_rpt(ctx->block, nrpt, src, 0, mask, 0); + create_immed_typed_rpt(&ctx->build, nrpt, 0xff, TYPE_U8); + cov = ir3_AND_B_rpt(&ctx->build, nrpt, src, 0, mask, 0); set_dst_flags(cov.rpts, nrpt, IR3_REG_HALF); - cov = ir3_COV_rpt(ctx->block, nrpt, cov, TYPE_U16, dst_type); + cov = ir3_COV_rpt(&ctx->build, nrpt, cov, TYPE_U16, dst_type); } else { - cov = ir3_COV_rpt(ctx->block, nrpt, src, TYPE_U8, TYPE_S16); - cov = ir3_COV_rpt(ctx->block, nrpt, cov, TYPE_S16, dst_type); + cov = ir3_COV_rpt(&ctx->build, nrpt, src, TYPE_U8, TYPE_S16); + cov = ir3_COV_rpt(&ctx->build, nrpt, cov, TYPE_S16, dst_type); } return cov; } @@ -385,13 +385,13 @@ create_cov(struct ir3_context *ctx, unsigned nrpt, type_t intermediate_type = op == nir_op_f2u8 ? TYPE_U16 : TYPE_S16; struct ir3_instruction_rpt cov = - ir3_COV_rpt(ctx->block, nrpt, src, src_type, intermediate_type); - cov = ir3_COV_rpt(ctx->block, nrpt, cov, intermediate_type, TYPE_U8); + ir3_COV_rpt(&ctx->build, nrpt, src, src_type, intermediate_type); + cov = ir3_COV_rpt(&ctx->build, nrpt, cov, intermediate_type, TYPE_U8); return cov; } struct ir3_instruction_rpt cov = - ir3_COV_rpt(ctx->block, nrpt, src, src_type, dst_type); + ir3_COV_rpt(&ctx->build, nrpt, src, src_type, dst_type); if (op == nir_op_f2f16_rtne) { set_cat1_round(cov.rpts, nrpt, ROUND_EVEN); @@ -418,9 +418,9 @@ resize_shift_amount(struct ir3_context *ctx, unsigned nrpt, struct ir3_instruction_rpt src, unsigned bs) { if (bs == 16) - return ir3_COV_rpt(ctx->block, nrpt, src, TYPE_U32, TYPE_U16); + return ir3_COV_rpt(&ctx->build, nrpt, src, TYPE_U32, TYPE_U16); else if (bs == 8) - return ir3_COV_rpt(ctx->block, nrpt, src, TYPE_U32, TYPE_U8); + return ir3_COV_rpt(&ctx->build, nrpt, src, TYPE_U32, TYPE_U8); else return src; } @@ -431,7 +431,7 @@ emit_alu_dot_4x8_as_dp4acc(struct ir3_context *ctx, nir_alu_instr *alu, struct ir3_instruction **src) { if (ctx->compiler->has_compliant_dp4acc) { - dst[0] = ir3_DP4ACC(ctx->block, src[0], 0, src[1], 0, src[2], 0); + dst[0] = ir3_DP4ACC(&ctx->build, src[0], 0, src[1], 0, src[2], 0); /* This is actually the LHS signedness attribute. * IR3_SRC_UNSIGNED ~ unsigned LHS (i.e. OpUDot and OpUDotAccSat). @@ -463,12 +463,12 @@ emit_alu_dot_4x8_as_dp4acc(struct ir3_context *ctx, nir_alu_instr *alu, struct ir3_instruction *accumulator = NULL; if (alu->op == nir_op_udot_4x8_uadd_sat) { - accumulator = create_immed(ctx->block, 0); + accumulator = create_immed(&ctx->build, 0); } else { accumulator = src[2]; } - dst[0] = ir3_DP4ACC(ctx->block, src[0], 0, src[1], 0, accumulator, 0); + dst[0] = ir3_DP4ACC(&ctx->build, src[0], 0, src[1], 0, accumulator, 0); if (alu->op == nir_op_udot_4x8_uadd || alu->op == nir_op_udot_4x8_uadd_sat) { @@ -481,7 +481,7 @@ emit_alu_dot_4x8_as_dp4acc(struct ir3_context *ctx, nir_alu_instr *alu, * we have to emulate it. */ if (alu->op == nir_op_udot_4x8_uadd_sat) { - dst[0] = ir3_ADD_U(ctx->block, dst[0], 0, src[2], 0); + dst[0] = ir3_ADD_U(&ctx->build, dst[0], 0, src[2], 0); dst[0]->flags |= IR3_INSTR_SAT; } else if (alu->op == nir_op_sudot_4x8_iadd_sat) { dst[0]->flags |= IR3_INSTR_SAT; @@ -504,24 +504,24 @@ emit_alu_dot_4x8_as_dp2acc(struct ir3_context *ctx, nir_alu_instr *alu, struct ir3_instruction *accumulator = NULL; if (alu->op == nir_op_udot_4x8_uadd_sat || alu->op == nir_op_sudot_4x8_iadd_sat) { - accumulator = create_immed(ctx->block, 0); + accumulator = create_immed(&ctx->build, 0); } else { accumulator = src[2]; } - dst[0] = ir3_DP2ACC(ctx->block, src[0], 0, src[1], 0, accumulator, 0); + dst[0] = ir3_DP2ACC(&ctx->build, src[0], 0, src[1], 0, accumulator, 0); dst[0]->cat3.packed = IR3_SRC_PACKED_LOW; dst[0]->cat3.signedness = signedness; - dst[0] = ir3_DP2ACC(ctx->block, src[0], 0, src[1], 0, dst[0], 0); + dst[0] = ir3_DP2ACC(&ctx->build, src[0], 0, src[1], 0, dst[0], 0); dst[0]->cat3.packed = IR3_SRC_PACKED_HIGH; dst[0]->cat3.signedness = signedness; if (alu->op == nir_op_udot_4x8_uadd_sat) { - dst[0] = ir3_ADD_U(ctx->block, dst[0], 0, src[2], 0); + dst[0] = ir3_ADD_U(&ctx->build, dst[0], 0, src[2], 0); dst[0]->flags |= IR3_INSTR_SAT; } else if (alu->op == nir_op_sudot_4x8_iadd_sat) { - dst[0] = ir3_ADD_S(ctx->block, dst[0], 0, src[2], 0); + dst[0] = ir3_ADD_S(&ctx->build, dst[0], 0, src[2], 0); dst[0]->flags |= IR3_INSTR_SAT; } } @@ -565,7 +565,7 @@ emit_alu(struct ir3_context *ctx, nir_alu_instr *alu) const nir_op_info *info = &nir_op_infos[alu->op]; struct ir3_instruction_rpt dst, src[info->num_inputs]; unsigned bs[info->num_inputs]; /* bit size */ - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; unsigned dst_sz; unsigned dst_bitsize = ir3_bitsize(ctx, alu->def.bit_size); type_t dst_type = type_uint_size(dst_bitsize); @@ -846,21 +846,21 @@ emit_alu(struct ir3_context *ctx, nir_alu_instr *alu) case nir_op_inot: if (bs[0] == 1) { struct ir3_instruction_rpt one = create_immed_typed_shared_rpt( - ctx->block, dst_sz, 1, ctx->compiler->bool_type, use_shared); + b, dst_sz, 1, ctx->compiler->bool_type, use_shared); dst = ir3_SUB_U_rpt(b, dst_sz, one, 0, src[0], 0); } else { - dst = ir3_NOT_B_rpt(ctx->block, dst_sz, src[0], 0); + dst = ir3_NOT_B_rpt(b, dst_sz, src[0], 0); } break; case nir_op_ior: dst = ir3_OR_B_rpt(b, dst_sz, src[0], 0, src[1], 0); break; case nir_op_ishl: - dst = ir3_SHL_B_rpt(ctx->block, dst_sz, src[0], 0, + dst = ir3_SHL_B_rpt(b, dst_sz, src[0], 0, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0); break; case nir_op_ishr: - dst = ir3_ASHR_B_rpt(ctx->block, dst_sz, src[0], 0, + dst = ir3_ASHR_B_rpt(b, dst_sz, src[0], 0, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0); break; case nir_op_isub: @@ -870,31 +870,31 @@ emit_alu(struct ir3_context *ctx, nir_alu_instr *alu) dst = ir3_XOR_B_rpt(b, dst_sz, src[0], 0, src[1], 0); break; case nir_op_ushr: - dst = ir3_SHR_B_rpt(ctx->block, dst_sz, src[0], 0, + dst = ir3_SHR_B_rpt(b, dst_sz, src[0], 0, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0); break; case nir_op_shrm_ir3: - dst = ir3_SHRM_rpt(ctx->block, dst_sz, + dst = ir3_SHRM_rpt(b, dst_sz, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0, src[0], 0, src[2], 0); break; case nir_op_shlm_ir3: - dst = ir3_SHLM_rpt(ctx->block, dst_sz, + dst = ir3_SHLM_rpt(b, dst_sz, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0, src[0], 0, src[2], 0); break; case nir_op_shrg_ir3: - dst = ir3_SHRG_rpt(ctx->block, dst_sz, + dst = ir3_SHRG_rpt(b, dst_sz, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0, src[0], 0, src[2], 0); break; case nir_op_shlg_ir3: - dst = ir3_SHLG_rpt(ctx->block, dst_sz, + dst = ir3_SHLG_rpt(b, dst_sz, resize_shift_amount(ctx, dst_sz, src[1], bs[0]), 0, src[0], 0, src[2], 0); break; case nir_op_andg_ir3: - dst = ir3_ANDG_rpt(ctx->block, dst_sz, src[0], 0, src[1], 0, src[2], 0); + dst = ir3_ANDG_rpt(b, dst_sz, src[0], 0, src[1], 0, src[2], 0); break; case nir_op_ilt: dst = ir3_CMPS_S_rpt(b, dst_sz, src[0], 0, src[1], 0); @@ -1123,7 +1123,7 @@ static void emit_intrinsic_load_ubo_ldc(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; /* This is only generated for us by nir_lower_ubo_vec4, which leaves base = * 0. @@ -1157,7 +1157,7 @@ static void emit_intrinsic_copy_ubo_to_uniform(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; unsigned base = nir_intrinsic_base(intr); unsigned size = nir_intrinsic_range(intr); @@ -1182,14 +1182,14 @@ emit_intrinsic_copy_ubo_to_uniform(struct ir3_context *ctx, ctx->so->constlen = MAX2(ctx->so->constlen, DIV_ROUND_UP(base + size * 4, 4)); - array_insert(b, b->keeps, ldc); + array_insert(ctx->block, ctx->block->keeps, ldc); } static void emit_intrinsic_copy_global_to_uniform(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; unsigned size = nir_intrinsic_range(intr); unsigned dst = nir_intrinsic_range_base(intr); @@ -1221,7 +1221,7 @@ emit_intrinsic_copy_global_to_uniform(struct ir3_context *ctx, ctx->so->constlen = MAX2(ctx->so->constlen, DIV_ROUND_UP(dst + size * 4, 4)); - array_insert(b, b->keeps, ldg); + array_insert(ctx->block, ctx->block->keeps, ldg); } @@ -1230,7 +1230,7 @@ static void emit_intrinsic_load_ubo(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *base_lo, *base_hi, *addr, *src0, *src1; const struct ir3_const_state *const_state = ir3_const_state(ctx->so); unsigned ubo = regid(const_state->offsets.ubo, 0); @@ -1311,7 +1311,7 @@ emit_intrinsic_load_kernel_input(struct ir3_context *ctx, struct ir3_instruction **dst) { const struct ir3_const_state *const_state = ir3_const_state(ctx->so); - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; unsigned offset = nir_intrinsic_base(intr); unsigned p = regid(const_state->offsets.kernel_params, 0); @@ -1344,7 +1344,7 @@ static void emit_intrinsic_ssbo_size(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *ibo = ir3_ssbo_to_ibo(ctx, intr->src[0]); struct ir3_instruction *resinfo = ir3_RESINFO(b, ibo, 0); resinfo->cat6.iim_val = 1; @@ -1371,7 +1371,7 @@ static void emit_intrinsic_load_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *ldl, *offset; unsigned base; @@ -1394,7 +1394,7 @@ emit_intrinsic_load_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr, static void emit_intrinsic_store_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *stl, *offset; struct ir3_instruction *const *value; unsigned base, wrmask, ncomp; @@ -1415,7 +1415,7 @@ emit_intrinsic_store_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr) stl->barrier_class = IR3_BARRIER_SHARED_W; stl->barrier_conflict = IR3_BARRIER_SHARED_R | IR3_BARRIER_SHARED_W; - array_insert(b, b->keeps, stl); + array_insert(ctx->block, ctx->block->keeps, stl); } /* src[] = { offset }. const_index[] = { base } */ @@ -1424,7 +1424,7 @@ emit_intrinsic_load_shared_ir3(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *load, *offset; unsigned base; @@ -1452,7 +1452,7 @@ static void emit_intrinsic_store_shared_ir3(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *store, *offset; struct ir3_instruction *const *value; @@ -1473,7 +1473,7 @@ emit_intrinsic_store_shared_ir3(struct ir3_context *ctx, store->barrier_class = IR3_BARRIER_SHARED_W; store->barrier_conflict = IR3_BARRIER_SHARED_R | IR3_BARRIER_SHARED_W; - array_insert(b, b->keeps, store); + array_insert(ctx->block, ctx->block->keeps, store); } /* @@ -1495,7 +1495,7 @@ emit_intrinsic_store_shared_ir3(struct ir3_context *ctx, static struct ir3_instruction * emit_intrinsic_atomic_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *atomic, *src0, *src1; type_t type = TYPE_U32; @@ -1548,7 +1548,7 @@ emit_intrinsic_atomic_shared(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->barrier_conflict = IR3_BARRIER_SHARED_R | IR3_BARRIER_SHARED_W; /* even if nothing consume the result, we can't DCE the instruction: */ - array_insert(b, b->keeps, atomic); + array_insert(ctx->block, ctx->block->keeps, atomic); return atomic; } @@ -1557,7 +1557,7 @@ static void stp_ldp_offset(struct ir3_context *ctx, nir_src *src, struct ir3_instruction **offset, int32_t *base) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; if (nir_src_is_const(*src)) { unsigned src_offset = nir_src_as_uint(*src); @@ -1580,7 +1580,7 @@ static void emit_intrinsic_load_scratch(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *ldp, *offset; int32_t base; @@ -1602,7 +1602,7 @@ emit_intrinsic_load_scratch(struct ir3_context *ctx, nir_intrinsic_instr *intr, static void emit_intrinsic_store_scratch(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *stp, *offset; struct ir3_instruction *const *value; unsigned wrmask, ncomp; @@ -1624,7 +1624,7 @@ emit_intrinsic_store_scratch(struct ir3_context *ctx, nir_intrinsic_instr *intr) stp->barrier_class = IR3_BARRIER_PRIVATE_W; stp->barrier_conflict = IR3_BARRIER_PRIVATE_R | IR3_BARRIER_PRIVATE_W; - array_insert(b, b->keeps, stp); + array_insert(ctx->block, ctx->block->keeps, stp); } struct tex_src_info { @@ -1641,7 +1641,7 @@ struct tex_src_info { static struct tex_src_info get_image_ssbo_samp_tex_src(struct ir3_context *ctx, nir_src *src, bool image) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct tex_src_info info = {0}; nir_intrinsic_instr *bindless_tex = ir3_bindless_resource(*src); @@ -1695,8 +1695,8 @@ get_image_ssbo_samp_tex_src(struct ir3_context *ctx, nir_src *src, bool image) ctx->so->num_samp = MAX2(ctx->so->num_samp, tex_idx + 1); - texture = create_immed_typed(ctx->block, tex_idx, TYPE_U16); - sampler = create_immed_typed(ctx->block, tex_idx, TYPE_U16); + texture = create_immed_typed(b, tex_idx, TYPE_U16); + sampler = create_immed_typed(b, tex_idx, TYPE_U16); info.samp_tex = ir3_collect(b, texture, sampler); } @@ -1713,7 +1713,7 @@ emit_sam(struct ir3_context *ctx, opc_t opc, struct tex_src_info info, if (info.flags & IR3_INSTR_A1EN) { addr = ir3_get_addr1(ctx, info.a1_val); } - sam = ir3_SAM(ctx->block, opc, type, wrmask, info.flags, info.samp_tex, src0, + sam = ir3_SAM(&ctx->build, opc, type, wrmask, info.flags, info.samp_tex, src0, src1); if (info.flags & IR3_INSTR_A1EN) { ir3_instr_set_address(sam, addr); @@ -1749,7 +1749,7 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, return; } - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct tex_src_info info = get_image_ssbo_samp_tex_src(ctx, &intr->src[0], true); struct ir3_instruction *sam; struct ir3_instruction *const *src0 = ir3_get_src(ctx, &intr->src[1]); @@ -1791,7 +1791,7 @@ emit_intrinsic_image_size_tex(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct tex_src_info info = get_image_ssbo_samp_tex_src(ctx, &intr->src[0], true); struct ir3_instruction *sam, *lod; unsigned flags, ncoords = ir3_get_image_coords(intr, &flags); @@ -1832,7 +1832,7 @@ static struct tex_src_info get_bindless_samp_src(struct ir3_context *ctx, nir_src *tex, nir_src *samp) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct tex_src_info info = {0}; info.flags |= IR3_INSTR_B; @@ -1944,7 +1944,7 @@ emit_intrinsic_load_ssbo(struct ir3_context *ctx, return; } - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; nir_src *offset_src = &intr->src[2]; struct ir3_instruction *coords = NULL; unsigned imm_offset = 0; @@ -1992,14 +1992,14 @@ emit_control_barrier(struct ir3_context *ctx) if (ctx->so->type == MESA_SHADER_TESS_CTRL) return; - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *barrier = ir3_BAR(b); barrier->cat7.g = true; if (ctx->compiler->gen < 6) barrier->cat7.l = true; barrier->flags = IR3_INSTR_SS | IR3_INSTR_SY; barrier->barrier_class = IR3_BARRIER_EVERYTHING; - array_insert(b, b->keeps, barrier); + array_insert(ctx->block, ctx->block->keeps, barrier); ctx->so->has_barrier = true; } @@ -2007,7 +2007,7 @@ emit_control_barrier(struct ir3_context *ctx) static void emit_intrinsic_barrier(struct ir3_context *ctx, nir_intrinsic_instr *intr) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction *barrier; /* TODO: find out why there is a major difference of .l usage @@ -2080,7 +2080,7 @@ emit_intrinsic_barrier(struct ir3_context *ctx, nir_intrinsic_instr *intr) } /* make sure barrier doesn't get DCE'd */ - array_insert(b, b->keeps, barrier); + array_insert(ctx->block, ctx->block->keeps, barrier); if (ctx->compiler->gen >= 7 && mem_scope > SCOPE_WORKGROUP && modes & (nir_var_mem_ssbo | nir_var_image) && @@ -2097,7 +2097,7 @@ emit_intrinsic_barrier(struct ir3_context *ctx, nir_intrinsic_instr *intr) */ ccinv->barrier_class = barrier->barrier_class; ccinv->barrier_conflict = barrier->barrier_conflict; - array_insert(b, b->keeps, ccinv); + array_insert(ctx->block, ctx->block->keeps, ccinv); } } @@ -2156,12 +2156,14 @@ get_barycentric(struct ir3_context *ctx, enum ir3_bary bary) if (!ctx->ij[bary]) { struct ir3_instruction *xy[2]; struct ir3_instruction *ij; + struct ir3_builder build = + ir3_builder_at(ir3_before_terminator(ctx->in_block)); ij = create_sysval_input(ctx, SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL + bary, 0x3); - ir3_split_dest(ctx->in_block, xy, ij, 0, 2); + ir3_split_dest(&build, xy, ij, 0, 2); - ctx->ij[bary] = ir3_create_collect(ctx->in_block, xy, 2); + ctx->ij[bary] = ir3_create_collect(&build, xy, 2); } return ctx->ij[bary]; @@ -2231,19 +2233,20 @@ emit_intrinsic_barycentric(struct ir3_context *ctx, nir_intrinsic_instr *intr, enum ir3_bary bary = sysval - SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL; struct ir3_instruction *ij = get_barycentric(ctx, bary); - ir3_split_dest(ctx->block, dst, ij, 0, 2); + ir3_split_dest(&ctx->build, dst, ij, 0, 2); } static struct ir3_instruction * get_frag_coord(struct ir3_context *ctx, nir_intrinsic_instr *intr) { if (!ctx->frag_coord) { - struct ir3_block *b = ir3_after_preamble(ctx->ir); + struct ir3_block *block = ir3_after_preamble(ctx->ir); + struct ir3_builder b = ir3_builder_at(ir3_before_terminator(block)); struct ir3_instruction_rpt xyzw; struct ir3_instruction *hw_frag_coord; hw_frag_coord = create_sysval_input(ctx, SYSTEM_VALUE_FRAG_COORD, 0xf); - ir3_split_dest(b, xyzw.rpts, hw_frag_coord, 0, 4); + ir3_split_dest(&b, xyzw.rpts, hw_frag_coord, 0, 4); /* for frag_coord.xy, we get unsigned values.. we need * to subtract (integer) 8 and divide by 16 (right- @@ -2255,11 +2258,11 @@ get_frag_coord(struct ir3_context *ctx, nir_intrinsic_instr *intr) * */ struct ir3_instruction_rpt xy = - ir3_COV_rpt(b, 2, xyzw, TYPE_U32, TYPE_F32); + ir3_COV_rpt(&b, 2, xyzw, TYPE_U32, TYPE_F32); xy = - ir3_MUL_F_rpt(b, 2, xy, 0, create_immed_rpt(b, 2, fui(1.0 / 16.0)), 0); + ir3_MUL_F_rpt(&b, 2, xy, 0, create_immed_rpt(&b, 2, fui(1.0 / 16.0)), 0); cp_instrs(xyzw.rpts, xy.rpts, 2); - ctx->frag_coord = ir3_create_collect(b, xyzw.rpts, 4); + ctx->frag_coord = ir3_create_collect(&b, xyzw.rpts, 4); } ctx->so->fragcoord_compmask |= nir_def_components_read(&intr->def); @@ -2273,9 +2276,9 @@ get_frag_coord(struct ir3_context *ctx, nir_intrinsic_instr *intr) * away by ir3_cp. */ static struct ir3_instruction * -create_multidst_mov(struct ir3_block *block, struct ir3_register *dst) +create_multidst_mov(struct ir3_builder *build, struct ir3_register *dst) { - struct ir3_instruction *mov = ir3_instr_create(block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_build_instr(build, OPC_MOV, 1, 1); unsigned dst_flags = dst->flags & IR3_REG_HALF; unsigned src_flags = dst->flags & (IR3_REG_HALF | IR3_REG_SHARED); @@ -2361,7 +2364,7 @@ emit_intrinsic_reduce(struct ir3_context *ctx, nir_intrinsic_instr *intr) * not supported. */ struct ir3_instruction *identity = - create_immed_shared(ctx->block, get_reduce_identity(nir_reduce_op, dst_size), + create_immed_shared(&ctx->build, get_reduce_identity(nir_reduce_op, dst_size), true); /* OPC_SCAN_MACRO has the following destinations: @@ -2373,7 +2376,7 @@ emit_intrinsic_reduce(struct ir3_context *ctx, nir_intrinsic_instr *intr) * choose which destination to return. */ struct ir3_instruction *scan = - ir3_instr_create(ctx->block, OPC_SCAN_MACRO, 3, 2); + ir3_build_instr(&ctx->build, OPC_SCAN_MACRO, 3, 2); scan->cat1.reduce_op = reduce_op; struct ir3_register *exclusive = __ssa_dst(scan); @@ -2405,7 +2408,7 @@ emit_intrinsic_reduce(struct ir3_context *ctx, nir_intrinsic_instr *intr) unreachable("unknown reduce intrinsic"); } - return create_multidst_mov(ctx->block, dst); + return create_multidst_mov(&ctx->build, dst); } static struct ir3_instruction * @@ -2425,7 +2428,7 @@ emit_intrinsic_reduce_clusters(struct ir3_context *ctx, * not supported. */ struct ir3_instruction *identity = - create_immed_shared(ctx->block, get_reduce_identity(nir_reduce_op, dst_size), + create_immed_shared(&ctx->build, get_reduce_identity(nir_reduce_op, dst_size), true); struct ir3_instruction *inclusive_src = ir3_get_src(ctx, &intr->src[0])[0]; @@ -2455,7 +2458,7 @@ emit_intrinsic_reduce_clusters(struct ir3_context *ctx, unsigned ndst = 2 + need_exclusive + need_scratch; unsigned nsrc = 2 + need_exclusive; struct ir3_instruction *scan = - ir3_instr_create(ctx->block, OPC_SCAN_CLUSTERS_MACRO, ndst, nsrc); + ir3_build_instr(&ctx->build, OPC_SCAN_CLUSTERS_MACRO, ndst, nsrc); scan->cat1.reduce_op = reduce_op; unsigned dst_flags = IR3_REG_EARLY_CLOBBER; @@ -2503,7 +2506,7 @@ emit_intrinsic_reduce_clusters(struct ir3_context *ctx, unreachable("unknown reduce intrinsic"); } - return create_multidst_mov(ctx->block, dst); + return create_multidst_mov(&ctx->build, dst); } static struct ir3_instruction * @@ -2511,7 +2514,7 @@ emit_intrinsic_brcst_active(struct ir3_context *ctx, nir_intrinsic_instr *intr) { struct ir3_instruction *default_src = ir3_get_src(ctx, &intr->src[0])[0]; struct ir3_instruction *brcst_val = ir3_get_src(ctx, &intr->src[1])[0]; - return ir3_BRCST_ACTIVE(ctx->block, nir_intrinsic_cluster_size(intr), + return ir3_BRCST_ACTIVE(&ctx->build, nir_intrinsic_cluster_size(intr), brcst_val, default_src); } @@ -2540,7 +2543,7 @@ emit_shfl(struct ir3_context *ctx, nir_intrinsic_instr *intr) struct ir3_instruction *val = ir3_get_src(ctx, &intr->src[0])[0]; struct ir3_instruction *idx = ir3_get_src(ctx, &intr->src[1])[0]; - struct ir3_instruction *shfl = ir3_SHFL(ctx->block, val, 0, idx, 0); + struct ir3_instruction *shfl = ir3_SHFL(&ctx->build, val, 0, idx, 0); shfl->cat6.shfl_mode = shfl_mode(intr); shfl->cat6.type = is_half(val) ? TYPE_U16 : TYPE_U32; @@ -2556,7 +2559,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic]; struct ir3_instruction **dst; struct ir3_instruction *const *src; - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; unsigned dest_components = nir_intrinsic_dest_components(intr); int idx; bool create_rpt = false; @@ -2576,7 +2579,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) /* There's logically nothing to do, but this has a destination in NIR so * plug in something... It will get DCE'd. */ - dst[0] = create_immed(ctx->block, 0); + dst[0] = create_immed(b, 0); break; case nir_intrinsic_load_reg: @@ -3070,7 +3073,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) IR3_BARRIER_ACTIVE_FIBERS_R; kill->srcs[0]->flags |= IR3_REG_PREDICATE; - array_insert(b, b->keeps, kill); + array_insert(ctx->block, ctx->block->keeps, kill); ctx->so->has_kill = true; break; @@ -3081,27 +3084,27 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; struct ir3_instruction *pred = ir3_get_predicate(ctx, src); if (intr->intrinsic == nir_intrinsic_vote_any) - dst[0] = ir3_ANY_MACRO(ctx->block, pred, 0); + dst[0] = ir3_ANY_MACRO(b, pred, 0); else - dst[0] = ir3_ALL_MACRO(ctx->block, pred, 0); + dst[0] = ir3_ALL_MACRO(b, pred, 0); dst[0]->srcs[0]->flags |= IR3_REG_PREDICATE; break; } case nir_intrinsic_elect: - dst[0] = ir3_ELECT_MACRO(ctx->block); + dst[0] = ir3_ELECT_MACRO(b); dst[0]->flags |= IR3_INSTR_NEEDS_HELPERS; break; case nir_intrinsic_elect_any_ir3: - dst[0] = ir3_ELECT_MACRO(ctx->block); + dst[0] = ir3_ELECT_MACRO(b); break; case nir_intrinsic_preamble_start_ir3: - dst[0] = ir3_SHPS_MACRO(ctx->block); + dst[0] = ir3_SHPS_MACRO(b); break; case nir_intrinsic_read_invocation_cond_ir3: { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; struct ir3_instruction *cond = ir3_get_src(ctx, &intr->src[1])[0]; - dst[0] = ir3_READ_COND_MACRO(ctx->block, ir3_get_predicate(ctx, cond), 0, + dst[0] = ir3_READ_COND_MACRO(b, ir3_get_predicate(ctx, cond), 0, src, 0); dst[0]->dsts[0]->flags |= IR3_REG_SHARED; dst[0]->srcs[0]->flags |= IR3_REG_PREDICATE; @@ -3118,7 +3121,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) case nir_intrinsic_read_first_invocation: { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; - dst[0] = ir3_READ_FIRST_MACRO(ctx->block, src, 0); + dst[0] = ir3_READ_FIRST_MACRO(b, src, 0); dst[0]->dsts[0]->flags |= IR3_REG_SHARED; /* See above. */ if (src->dsts[0]->flags & IR3_REG_HALF) { @@ -3131,7 +3134,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) case nir_intrinsic_read_getlast_ir3: { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; - dst[0] = ir3_READ_GETLAST_MACRO(ctx->block, src, 0); + dst[0] = ir3_READ_GETLAST_MACRO(b, src, 0); dst[0]->dsts[0]->flags |= IR3_REG_SHARED; /* See above. */ if (src->dsts[0]->flags & IR3_REG_HALF) { @@ -3147,18 +3150,18 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) unsigned components = intr->def.num_components; if (nir_src_is_const(intr->src[0]) && nir_src_as_bool(intr->src[0])) { /* ballot(true) is just MOVMSK */ - ballot = ir3_MOVMSK(ctx->block, components); + ballot = ir3_MOVMSK(b, components); } else { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; struct ir3_instruction *pred = ir3_get_predicate(ctx, src); - ballot = ir3_BALLOT_MACRO(ctx->block, pred, components); + ballot = ir3_BALLOT_MACRO(b, pred, components); ballot->srcs[0]->flags |= IR3_REG_PREDICATE; } ballot->barrier_class = IR3_BARRIER_ACTIVE_FIBERS_R; ballot->barrier_conflict = IR3_BARRIER_ACTIVE_FIBERS_W; - ir3_split_dest(ctx->block, dst, ballot, 0, components); + ir3_split_dest(b, dst, ballot, 0, components); break; } @@ -3169,30 +3172,30 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) type_t dst_type = type_uint_size(intr->def.bit_size); if (dst_type != TYPE_U32) - idx = ir3_COV(ctx->block, idx, TYPE_U32, dst_type); + idx = ir3_COV(b, idx, TYPE_U32, dst_type); - dst[0] = ir3_QUAD_SHUFFLE_BRCST(ctx->block, src, 0, idx, 0); + dst[0] = ir3_QUAD_SHUFFLE_BRCST(b, src, 0, idx, 0); dst[0]->cat5.type = dst_type; break; } case nir_intrinsic_quad_swap_horizontal: { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; - dst[0] = ir3_QUAD_SHUFFLE_HORIZ(ctx->block, src, 0); + dst[0] = ir3_QUAD_SHUFFLE_HORIZ(b, src, 0); dst[0]->cat5.type = type_uint_size(intr->def.bit_size); break; } case nir_intrinsic_quad_swap_vertical: { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; - dst[0] = ir3_QUAD_SHUFFLE_VERT(ctx->block, src, 0); + dst[0] = ir3_QUAD_SHUFFLE_VERT(b, src, 0); dst[0]->cat5.type = type_uint_size(intr->def.bit_size); break; } case nir_intrinsic_quad_swap_diagonal: { struct ir3_instruction *src = ir3_get_src(ctx, &intr->src[0])[0]; - dst[0] = ir3_QUAD_SHUFFLE_DIAG(ctx->block, src, 0); + dst[0] = ir3_QUAD_SHUFFLE_DIAG(b, src, 0); dst[0]->cat5.type = type_uint_size(intr->def.bit_size); break; } @@ -3254,9 +3257,9 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) break; case nir_intrinsic_preamble_end_ir3: { - struct ir3_instruction *instr = ir3_SHPE(ctx->block); + struct ir3_instruction *instr = ir3_SHPE(b); instr->barrier_class = instr->barrier_conflict = IR3_BARRIER_CONST_W; - array_insert(b, b->keeps, instr); + array_insert(ctx->block, ctx->block->keeps, instr); break; } case nir_intrinsic_store_const_ir3: { @@ -3279,7 +3282,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) } struct ir3_instruction *stc = - ir3_STC(ctx->block, create_immed(b, dst_lo), 0, src, 0); + ir3_STC(b, create_immed(b, dst_lo), 0, src, 0); stc->cat6.iim_val = components; stc->cat6.type = TYPE_U32; stc->barrier_conflict = IR3_BARRIER_CONST_W; @@ -3292,13 +3295,13 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) */ ctx->so->constlen = MAX2(ctx->so->constlen, DIV_ROUND_UP(dst + components, 4)); - array_insert(b, b->keeps, stc); + array_insert(ctx->block, ctx->block->keeps, stc); break; } case nir_intrinsic_copy_push_const_to_uniform_ir3: { struct ir3_instruction *load = - ir3_instr_create(ctx->block, OPC_PUSH_CONSTS_LOAD_MACRO, 0, 0); - array_insert(b, b->keeps, load); + ir3_build_instr(b, OPC_PUSH_CONSTS_LOAD_MACRO, 0, 0); + array_insert(ctx->block, ctx->block->keeps, load); load->push_consts.dst_base = nir_src_as_uint(intr->src[0]); load->push_consts.src_base = nir_intrinsic_base(intr); @@ -3337,7 +3340,7 @@ emit_intrinsic(struct ir3_context *ctx, nir_intrinsic_instr *intr) break; } case nir_intrinsic_prefetch_ubo_ir3: { - struct ir3_instruction *offset = create_immed(ctx->block, 0); + struct ir3_instruction *offset = create_immed(b, 0); struct ir3_instruction *idx = ir3_get_src(ctx, &intr->src[0])[0]; struct ir3_instruction *ldc = ir3_LDC(b, idx, 0, offset, 0); ldc->cat6.iim_val = 1; @@ -3379,18 +3382,18 @@ emit_load_const(struct ir3_context *ctx, nir_load_const_instr *instr) if (bit_size <= 8) { for (int i = 0; i < instr->def.num_components; i++) - dst[i] = create_immed_typed(ctx->block, instr->value[i].u8, TYPE_U8); + dst[i] = create_immed_typed(&ctx->build, instr->value[i].u8, TYPE_U8); } else if (bit_size <= 16) { for (int i = 0; i < instr->def.num_components; i++) - dst[i] = create_immed_typed(ctx->block, instr->value[i].u16, TYPE_U16); + dst[i] = create_immed_typed(&ctx->build, instr->value[i].u16, TYPE_U16); } else if (bit_size <= 32) { for (int i = 0; i < instr->def.num_components; i++) - dst[i] = create_immed_typed(ctx->block, instr->value[i].u32, TYPE_U32); + dst[i] = create_immed_typed(&ctx->build, instr->value[i].u32, TYPE_U32); } else { assert(instr->def.num_components == 1); for (int i = 0; i < instr->def.num_components; i++) { - dst[2 * i] = create_immed_typed(ctx->block, (uint32_t)(instr->value[i].u64), TYPE_U32); - dst[2 * i + 1] = create_immed_typed(ctx->block, (uint32_t)(instr->value[i].u64 >> 32), TYPE_U32); + dst[2 * i] = create_immed_typed(&ctx->build, (uint32_t)(instr->value[i].u64), TYPE_U32); + dst[2 * i + 1] = create_immed_typed(&ctx->build, (uint32_t)(instr->value[i].u64 >> 32), TYPE_U32); } } } @@ -3406,7 +3409,7 @@ emit_undef(struct ir3_context *ctx, nir_undef_instr *undef) * in 0.0.. */ for (int i = 0; i < undef->def.num_components; i++) - dst[i] = create_immed_typed(ctx->block, fui(0.0), type); + dst[i] = create_immed_typed(&ctx->build, fui(0.0), type); } /* @@ -3472,7 +3475,7 @@ tex_info(nir_tex_instr *tex, unsigned *flagsp, unsigned *coordsp) static struct tex_src_info get_tex_samp_tex_src(struct ir3_context *ctx, nir_tex_instr *tex) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct tex_src_info info = {0}; int texture_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_handle); int sampler_idx = nir_tex_instr_src_index(tex, nir_tex_src_sampler_handle); @@ -3492,7 +3495,7 @@ get_tex_samp_tex_src(struct ir3_context *ctx, nir_tex_instr *tex) sampler_idx = nir_tex_instr_src_index(tex, nir_tex_src_sampler_offset); if (texture_idx >= 0) { texture = ir3_get_src(ctx, &tex->src[texture_idx].src)[0]; - texture = ir3_COV(ctx->block, texture, TYPE_U32, TYPE_U16); + texture = ir3_COV(b, texture, TYPE_U32, TYPE_U16); } else { /* TODO what to do for dynamic case? I guess we only need the * max index for astc srgb workaround so maybe not a problem @@ -3501,15 +3504,15 @@ get_tex_samp_tex_src(struct ir3_context *ctx, nir_tex_instr *tex) */ ctx->max_texture_index = MAX2(ctx->max_texture_index, tex->texture_index); - texture = create_immed_typed(ctx->block, tex->texture_index, TYPE_U16); + texture = create_immed_typed(b, tex->texture_index, TYPE_U16); info.tex_idx = tex->texture_index; } if (sampler_idx >= 0) { sampler = ir3_get_src(ctx, &tex->src[sampler_idx].src)[0]; - sampler = ir3_COV(ctx->block, sampler, TYPE_U32, TYPE_U16); + sampler = ir3_COV(b, sampler, TYPE_U32, TYPE_U16); } else { - sampler = create_immed_typed(ctx->block, tex->sampler_index, TYPE_U16); + sampler = create_immed_typed(b, tex->sampler_index, TYPE_U16); info.samp_idx = tex->texture_index; } @@ -3522,7 +3525,7 @@ get_tex_samp_tex_src(struct ir3_context *ctx, nir_tex_instr *tex) static void emit_tex(struct ir3_context *ctx, nir_tex_instr *tex) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction **dst, *sam, *src0[12], *src1[4]; struct ir3_instruction *const *coord, *const *off, *const *ddx, *const *ddy; struct ir3_instruction *lod, *compare, *proj, *sample_index; @@ -3783,12 +3786,12 @@ emit_tex(struct ir3_context *ctx, nir_tex_instr *tex) TYPE_U32); } else { texture = create_immed_typed( - ctx->block, ctx->compiler->options.bindless_fb_read_slot, TYPE_U32); + b, ctx->compiler->options.bindless_fb_read_slot, TYPE_U32); struct ir3_instruction *base = ir3_get_src(ctx, &tex->src[base_index].src)[0]; texture = ir3_ADD_U(b, texture, 0, base, 0); } - sampler = create_immed_typed(ctx->block, 0, TYPE_U32); + sampler = create_immed_typed(b, 0, TYPE_U32); info.samp_tex = ir3_collect(b, texture, sampler); info.flags |= IR3_INSTR_S2EN; if (tex->texture_non_uniform) { @@ -3799,8 +3802,8 @@ emit_tex(struct ir3_context *ctx, nir_tex_instr *tex) * state: */ info.samp_tex = ir3_collect( - b, create_immed_typed(ctx->block, ctx->so->num_samp, TYPE_U16), - create_immed_typed(ctx->block, ctx->so->num_samp, TYPE_U16)); + b, create_immed_typed(b, ctx->so->num_samp, TYPE_U16), + create_immed_typed(b, ctx->so->num_samp, TYPE_U16)); info.flags = IR3_INSTR_S2EN; } @@ -3835,8 +3838,9 @@ emit_tex(struct ir3_context *ctx, nir_tex_instr *tex) if (opc == OPC_META_TEX_PREFETCH) { int idx = nir_tex_instr_src_index(tex, nir_tex_src_coord); - - sam = ir3_SAM(ctx->in_block, opc, type, MASK(ncomp), 0, NULL, + struct ir3_builder build = + ir3_builder_at(ir3_before_terminator(ctx->in_block)); + sam = ir3_SAM(&build, opc, type, MASK(ncomp), 0, NULL, get_barycentric(ctx, IJ_PERSP_PIXEL), 0); sam->prefetch.input_offset = ir3_nir_coord_offset(tex->src[idx].src.ssa); /* make sure not to add irrelevant flags like S2EN */ @@ -3934,7 +3938,7 @@ emit_tex(struct ir3_context *ctx, nir_tex_instr *tex) static void emit_tex_info(struct ir3_context *ctx, nir_tex_instr *tex, unsigned idx) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction **dst, *sam; type_t dst_type = get_tex_dest_type(tex); struct tex_src_info info = get_tex_samp_tex_src(ctx, tex); @@ -3960,7 +3964,7 @@ emit_tex_info(struct ir3_context *ctx, nir_tex_instr *tex, unsigned idx) static void emit_tex_txs(struct ir3_context *ctx, nir_tex_instr *tex) { - struct ir3_block *b = ctx->block; + struct ir3_builder *b = &ctx->build; struct ir3_instruction **dst, *sam; struct ir3_instruction *lod; unsigned flags, coords; @@ -4038,7 +4042,7 @@ emit_phi(struct ir3_context *ctx, nir_phi_instr *nphi) } for (unsigned i = 0; i < num_components; i++) { - phi = ir3_instr_create(ctx->block, OPC_META_PHI, 1, + phi = ir3_build_instr(&ctx->build, OPC_META_PHI, 1, exec_list_length(&nphi->srcs)); __ssa_dst(phi); phi->phi.nphi = nphi; @@ -4061,8 +4065,9 @@ read_phi_src(struct ir3_context *ctx, struct ir3_block *blk, struct ir3_instruction *phi, nir_phi_instr *nphi) { if (!blk->nblock) { + struct ir3_builder build = ir3_builder_at(ir3_before_terminator(blk)); struct ir3_instruction *continue_phi = - ir3_instr_create(blk, OPC_META_PHI, 1, blk->predecessors_count); + ir3_build_instr(&build, OPC_META_PHI, 1, blk->predecessors_count); __ssa_dst(continue_phi)->flags = phi->dsts[0]->flags; for (unsigned i = 0; i < blk->predecessors_count; i++) { @@ -4085,11 +4090,11 @@ read_phi_src(struct ir3_context *ctx, struct ir3_block *blk, } else { /* We need to insert the move at the end of the block */ struct ir3_block *old_block = ctx->block; - ctx->block = blk; + ir3_context_set_block(ctx, blk); struct ir3_instruction *src = ir3_get_src_shared( ctx, &nsrc->src, phi->dsts[0]->flags & IR3_REG_SHARED)[phi->phi.comp]; - ctx->block = old_block; + ir3_context_set_block(ctx, old_block); return src; } } @@ -4237,7 +4242,7 @@ create_continue_block(struct ir3_context *ctx, const nir_block *nblock) static void emit_block(struct ir3_context *ctx, nir_block *nblock) { - ctx->block = get_block(ctx, nblock); + ir3_context_set_block(ctx, get_block(ctx, nblock)); list_addtail(&ctx->block->node, &ctx->ir->block_list); @@ -4272,7 +4277,7 @@ emit_block(struct ir3_context *ctx, nir_block *nblock) */ if (ctx->block->successors[0] && !ctx->block->successors[1]) { if (!ir3_block_get_terminator(ctx->block)) - ir3_JUMP(ctx->block); + ir3_JUMP(&ctx->build); } _mesa_hash_table_clear(ctx->sel_cond_conversions, NULL); @@ -4339,10 +4344,10 @@ fold_conditional_branch(struct ir3_context *ctx, struct nir_src *nir_cond) struct ir3_instruction *branch; if (alu_cond->op == nir_op_iand) { - branch = ir3_BRAA(ctx->block, cond1, IR3_REG_PREDICATE, cond2, + branch = ir3_BRAA(&ctx->build, cond1, IR3_REG_PREDICATE, cond2, IR3_REG_PREDICATE); } else { - branch = ir3_BRAO(ctx->block, cond1, IR3_REG_PREDICATE, cond2, + branch = ir3_BRAO(&ctx->build, cond1, IR3_REG_PREDICATE, cond2, IR3_REG_PREDICATE); } @@ -4471,14 +4476,15 @@ emit_predicated_branch(struct ir3_context *ctx, nir_if *nif) bool inv; struct ir3_instruction *condition = get_branch_condition(ctx, &nif->condition, 0, &inv); + struct ir3_builder then_build = ir3_builder_at(ir3_after_block(then_block)); struct ir3_instruction *pred, *pred_inv; if (!inv) { - pred = ir3_PREDT(ctx->block, condition, IR3_REG_PREDICATE); - pred_inv = ir3_PREDF(then_block, condition, IR3_REG_PREDICATE); + pred = ir3_PREDT(&ctx->build, condition, IR3_REG_PREDICATE); + pred_inv = ir3_PREDF(&then_build, condition, IR3_REG_PREDICATE); } else { - pred = ir3_PREDF(ctx->block, condition, IR3_REG_PREDICATE); - pred_inv = ir3_PREDT(then_block, condition, IR3_REG_PREDICATE); + pred = ir3_PREDF(&ctx->build, condition, IR3_REG_PREDICATE); + pred_inv = ir3_PREDT(&then_build, condition, IR3_REG_PREDICATE); } pred->srcs[0]->num = REG_P0_X; @@ -4502,7 +4508,7 @@ emit_conditional_branch(struct ir3_context *ctx, nir_if *nif) struct ir3_instruction *cond1 = get_branch_condition(ctx, nir_cond, 0, &inv1); struct ir3_instruction *branch = - ir3_BR(ctx->block, cond1, IR3_REG_PREDICATE); + ir3_BR(&ctx->build, cond1, IR3_REG_PREDICATE); branch->cat0.inv1 = inv1; return branch; } @@ -4514,14 +4520,14 @@ emit_if(struct ir3_context *ctx, nir_if *nif) if (condition->opc == OPC_ANY_MACRO && condition->block == ctx->block) { struct ir3_instruction *pred = ssa(condition->srcs[0]); - ir3_BANY(ctx->block, pred, IR3_REG_PREDICATE); + ir3_BANY(&ctx->build, pred, IR3_REG_PREDICATE); } else if (condition->opc == OPC_ALL_MACRO && condition->block == ctx->block) { struct ir3_instruction *pred = ssa(condition->srcs[0]); - ir3_BALL(ctx->block, pred, IR3_REG_PREDICATE); + ir3_BALL(&ctx->build, pred, IR3_REG_PREDICATE); } else if (condition->opc == OPC_ELECT_MACRO && condition->block == ctx->block) { - struct ir3_instruction *branch = ir3_GETONE(ctx->block); + struct ir3_instruction *branch = ir3_GETONE(&ctx->build); branch->flags |= condition->flags & IR3_INSTR_NEEDS_HELPERS; } else if (condition->opc == OPC_SHPS_MACRO && condition->block == ctx->block) { @@ -4529,7 +4535,7 @@ emit_if(struct ir3_context *ctx, nir_if *nif) * shps, but we only use it in very constrained scenarios so this should * be ok. */ - ir3_SHPS(ctx->block); + ir3_SHPS(&ctx->build); } else { emit_conditional_branch(ctx, nif); } @@ -4587,7 +4593,8 @@ emit_loop(struct ir3_context *ctx, nir_loop *nloop) if (continue_blk) { struct ir3_block *start = get_block(ctx, nstart); - ir3_JUMP(continue_blk); + struct ir3_builder build = ir3_builder_at(ir3_after_block(continue_blk)); + ir3_JUMP(&build); continue_blk->successors[0] = start; continue_blk->loop_depth = ctx->loop_depth; list_addtail(&continue_blk->node, &ctx->ir->block_list); @@ -4675,7 +4682,7 @@ emit_stream_out(struct ir3_context *ctx) stream_out_block->successors[0] = new_end_block; /* setup 'if (vtxcnt < maxvtxcnt)' condition: */ - cond = ir3_CMPS_S(ctx->block, vtxcnt, 0, maxvtxcnt, 0); + cond = ir3_CMPS_S(&ctx->build, vtxcnt, 0, maxvtxcnt, 0); cond->dsts[0]->flags |= IR3_REG_PREDICATE; cond->cat2.condition = IR3_COND_LT; @@ -4683,12 +4690,12 @@ emit_stream_out(struct ir3_context *ctx) * since it is used to pick which of the two successor * paths to take: */ - ir3_BR(orig_end_block, cond, IR3_REG_PREDICATE); + ir3_BR(&ctx->build, cond, IR3_REG_PREDICATE); /* switch to stream_out_block to generate the stream-out * instructions: */ - ctx->block = stream_out_block; + ir3_context_set_block(ctx, stream_out_block); /* Calculate base addresses based on vtxcnt. Instructions * generated for bases not used in following loop will be @@ -4699,13 +4706,13 @@ emit_stream_out(struct ir3_context *ctx) unsigned stride = strmout->stride[i]; struct ir3_instruction *base, *off; - base = create_uniform(ctx->block, regid(const_state->offsets.tfbo, i)); + base = create_uniform(&ctx->build, regid(const_state->offsets.tfbo, i)); /* 24-bit should be enough: */ - off = ir3_MUL_U24(ctx->block, vtxcnt, 0, - create_immed(ctx->block, stride * 4), 0); + off = ir3_MUL_U24(&ctx->build, vtxcnt, 0, + create_immed(&ctx->build, stride * 4), 0); - bases[i] = ir3_ADD_S(ctx->block, off, 0, base, 0); + bases[i] = ir3_ADD_S(&ctx->build, off, 0, base, 0); } /* Generate the per-output store instructions: */ @@ -4718,19 +4725,19 @@ emit_stream_out(struct ir3_context *ctx) out = ctx->outputs[regid(strmout->output[i].register_index, c)]; stg = ir3_STG( - ctx->block, base, 0, - create_immed(ctx->block, (strmout->output[i].dst_offset + j) * 4), - 0, out, 0, create_immed(ctx->block, 1), 0); + &ctx->build, base, 0, + create_immed(&ctx->build, (strmout->output[i].dst_offset + j) * 4), + 0, out, 0, create_immed(&ctx->build, 1), 0); stg->cat6.type = TYPE_U32; array_insert(ctx->block, ctx->block->keeps, stg); } } - ir3_JUMP(ctx->block); + ir3_JUMP(&ctx->build); /* and finally switch to the new_end_block: */ - ctx->block = new_end_block; + ir3_context_set_block(ctx, new_end_block); } static void @@ -4788,7 +4795,7 @@ setup_input(struct ir3_context *ctx, nir_intrinsic_instr *intr) struct ir3_instruction *coord = NULL; if (intr->intrinsic == nir_intrinsic_load_interpolated_input) - coord = ir3_create_collect(ctx->block, ir3_get_src(ctx, &intr->src[0]), 2); + coord = ir3_create_collect(&ctx->build, ir3_get_src(ctx, &intr->src[0]), 2); compile_assert(ctx, nir_src_is_const(intr->src[coord ? 1 : 0])); @@ -4877,7 +4884,7 @@ setup_input(struct ir3_context *ctx, nir_intrinsic_instr *intr) continue; } - ir3_split_dest(ctx->block, &ctx->inputs[idx], input, i, 1); + ir3_split_dest(&ctx->build, &ctx->inputs[idx], input, i, 1); } for (int i = 0; i < ncomp; i++) { @@ -5112,7 +5119,7 @@ setup_output(struct ir3_context *ctx, nir_intrinsic_instr *intr) for (int i = 0; i < ncomp; i++) { unsigned idx = (n * 4) + i + frac; compile_assert(ctx, idx < ctx->noutputs); - ctx->outputs[idx] = create_immed(ctx->block, fui(0.0)); + ctx->outputs[idx] = create_immed(&ctx->build, fui(0.0)); } /* if varying packing doesn't happen, we could end up in a situation @@ -5126,7 +5133,7 @@ setup_output(struct ir3_context *ctx, nir_intrinsic_instr *intr) for (int i = 0; i < frac; i++) { unsigned idx = (n * 4) + i; if (!ctx->outputs[idx]) { - ctx->outputs[idx] = create_immed(ctx->block, fui(0.0)); + ctx->outputs[idx] = create_immed(&ctx->build, fui(0.0)); } } @@ -5210,7 +5217,7 @@ emit_instructions(struct ir3_context *ctx) ctx->ir = ir3_create(ctx->compiler, ctx->so); /* Create inputs in first block: */ - ctx->block = get_block(ctx, nir_start_block(fxn)); + ir3_context_set_block(ctx, get_block(ctx, nir_start_block(fxn))); ctx->in_block = ctx->block; /* for fragment shader, the vcoord input register is used as the @@ -5315,8 +5322,9 @@ emit_instructions(struct ir3_context *ctx) * preamble is enabled on a7xx, so we have to put the barrier after. */ struct ir3_block *block = ir3_after_preamble(ctx->ir); + struct ir3_builder build = ir3_builder_at(ir3_after_block(block)); - struct ir3_instruction *barrier = ir3_BAR(block); + struct ir3_instruction *barrier = ir3_BAR(&build); barrier->flags = IR3_INSTR_SS | IR3_INSTR_SY; barrier->barrier_class = IR3_BARRIER_EVERYTHING; array_insert(block, block->keeps, barrier); @@ -5530,7 +5538,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, unsigned n = so->outputs_count++; so->outputs[n].slot = VARYING_SLOT_PRIMITIVE_ID; - struct ir3_instruction *out = ir3_collect(ctx->block, ctx->primitive_id); + struct ir3_instruction *out = ir3_collect(&ctx->build, ctx->primitive_id); outputs[outputs_count] = out; outidxs[outputs_count] = n; if (so->type == MESA_SHADER_VERTEX && ctx->rel_patch_id) @@ -5543,7 +5551,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, if (so->type == MESA_SHADER_VERTEX && ctx->rel_patch_id) { unsigned n = so->outputs_count++; so->outputs[n].slot = VARYING_SLOT_REL_PATCH_ID_IR3; - struct ir3_instruction *out = ir3_collect(ctx->block, ctx->rel_patch_id); + struct ir3_instruction *out = ir3_collect(&ctx->build, ctx->rel_patch_id); outputs[outputs_count] = out; outidxs[outputs_count] = n; regids[outputs_count] = regid(0, 1); @@ -5553,7 +5561,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, if (ctx->gs_header) { unsigned n = so->outputs_count++; so->outputs[n].slot = VARYING_SLOT_GS_HEADER_IR3; - struct ir3_instruction *out = ir3_collect(ctx->block, ctx->gs_header); + struct ir3_instruction *out = ir3_collect(&ctx->build, ctx->gs_header); outputs[outputs_count] = out; outidxs[outputs_count] = n; regids[outputs_count] = regid(0, 0); @@ -5563,7 +5571,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, if (ctx->tcs_header) { unsigned n = so->outputs_count++; so->outputs[n].slot = VARYING_SLOT_TCS_HEADER_IR3; - struct ir3_instruction *out = ir3_collect(ctx->block, ctx->tcs_header); + struct ir3_instruction *out = ir3_collect(&ctx->build, ctx->tcs_header); outputs[outputs_count] = out; outidxs[outputs_count] = n; regids[outputs_count] = regid(0, 0); @@ -5571,7 +5579,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, } struct ir3_instruction *chmask = - ir3_instr_create(ctx->block, OPC_CHMASK, 0, outputs_count); + ir3_build_instr(&ctx->build, OPC_CHMASK, 0, outputs_count); chmask->barrier_class = IR3_BARRIER_EVERYTHING; chmask->barrier_conflict = IR3_BARRIER_EVERYTHING; @@ -5583,7 +5591,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, array_insert(ctx->block, ctx->block->keeps, chmask); - struct ir3_instruction *chsh = ir3_CHSH(ctx->block); + struct ir3_instruction *chsh = ir3_CHSH(&ctx->build); chsh->barrier_class = IR3_BARRIER_EVERYTHING; chsh->barrier_conflict = IR3_BARRIER_EVERYTHING; } else { @@ -5624,8 +5632,9 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, if (!ncomp) continue; + struct ir3_builder build = ir3_builder_at(ir3_before_terminator(b)); struct ir3_instruction *out = - ir3_create_collect(b, &ctx->outputs[i], ncomp); + ir3_create_collect(&build, &ctx->outputs[i], ncomp); int outidx = i / 4; assert(outidx < so->outputs_count); @@ -5662,7 +5671,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, } struct ir3_instruction *end = - ir3_instr_create(ctx->block, OPC_END, 0, outputs_count); + ir3_build_instr(&ctx->build, OPC_END, 0, outputs_count); for (unsigned i = 0; i < outputs_count; i++) { __ssa_src(end, outputs[i], 0); @@ -5872,12 +5881,12 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, if (ctx->compiler->gen >= 7 && so->type == MESA_SHADER_COMPUTE) { struct ir3_instruction *end = find_end(so->ir); struct ir3_instruction *lock = - ir3_instr_create(ctx->block, OPC_LOCK, 0, 0); + ir3_build_instr(&ctx->build, OPC_LOCK, 0, 0); /* TODO: This flags should be set by scheduler only when needed */ lock->flags = IR3_INSTR_SS | IR3_INSTR_SY | IR3_INSTR_JP; ir3_instr_move_before(lock, end); struct ir3_instruction *unlock = - ir3_instr_create(ctx->block, OPC_UNLOCK, 0, 0); + ir3_build_instr(&ctx->build, OPC_UNLOCK, 0, 0); ir3_instr_move_before(unlock, end); } diff --git a/src/freedreno/ir3/ir3_context.c b/src/freedreno/ir3/ir3_context.c index a1b7fad16e8..e820ee7dec9 100644 --- a/src/freedreno/ir3/ir3_context.c +++ b/src/freedreno/ir3/ir3_context.c @@ -244,11 +244,11 @@ ir3_get_src_maybe_shared(struct ir3_context *ctx, nir_src *src) } static struct ir3_instruction * -get_shared(struct ir3_block *block, struct ir3_instruction *src, bool shared) +get_shared(struct ir3_builder *build, struct ir3_instruction *src, bool shared) { if (!!(src->dsts[0]->flags & IR3_REG_SHARED) != shared) { struct ir3_instruction *mov = - ir3_MOV(block, src, (src->dsts[0]->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32); + ir3_MOV(build, src, (src->dsts[0]->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32); mov->dsts[0]->flags &= ~IR3_REG_SHARED; mov->dsts[0]->flags |= COND(shared, IR3_REG_SHARED); return mov; @@ -276,7 +276,7 @@ ir3_get_src_shared(struct ir3_context *ctx, nir_src *src, bool shared) struct ir3_instruction **new_value = ralloc_array(ctx, struct ir3_instruction *, num_components); for (unsigned i = 0; i < num_components; i++) - new_value[i] = get_shared(ctx->block, value[i], shared); + new_value[i] = get_shared(&ctx->build, value[i], shared); return new_value; } @@ -310,7 +310,7 @@ dest_flags(struct ir3_instruction *instr) } struct ir3_instruction * -ir3_create_collect(struct ir3_block *block, struct ir3_instruction *const *arr, +ir3_create_collect(struct ir3_builder *build, struct ir3_instruction *const *arr, unsigned arrsz) { struct ir3_instruction *collect; @@ -323,7 +323,7 @@ ir3_create_collect(struct ir3_block *block, struct ir3_instruction *const *arr, unsigned flags = dest_flags(arr[0]); - collect = ir3_instr_create(block, OPC_META_COLLECT, 1, arrsz); + collect = ir3_build_instr(build, OPC_META_COLLECT, 1, arrsz); __ssa_dst(collect)->flags |= flags; for (unsigned i = 0; i < arrsz; i++) { struct ir3_instruction *elem = arr[i]; @@ -354,7 +354,7 @@ ir3_create_collect(struct ir3_block *block, struct ir3_instruction *const *arr, */ if (elem->dsts[0]->flags & IR3_REG_ARRAY) { type_t type = (flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32; - elem = ir3_MOV(block, elem, type); + elem = ir3_MOV(build, elem, type); } assert(dest_flags(elem) == flags); @@ -370,7 +370,7 @@ ir3_create_collect(struct ir3_block *block, struct ir3_instruction *const *arr, * outputs which need to have a split meta instruction inserted */ void -ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst, +ir3_split_dest(struct ir3_builder *build, struct ir3_instruction **dst, struct ir3_instruction *src, unsigned base, unsigned n) { if ((n == 1) && (src->dsts[0]->wrmask == 0x1) && @@ -394,7 +394,7 @@ ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst, for (int i = 0, j = 0; i < n; i++) { struct ir3_instruction *split = - ir3_instr_create(block, OPC_META_SPLIT, 1, 1); + ir3_build_instr(build, OPC_META_SPLIT, 1, 1); __ssa_dst(split)->flags |= flags; __ssa_src(split, src, flags); split->split.off = i + base; @@ -426,11 +426,11 @@ ir3_context_error(struct ir3_context *ctx, const char *format, ...) } static struct ir3_instruction * -create_addr0(struct ir3_block *block, struct ir3_instruction *src, int align) +create_addr0(struct ir3_builder *build, struct ir3_instruction *src, int align) { struct ir3_instruction *instr, *immed; - instr = ir3_COV(block, src, TYPE_U32, TYPE_S16); + instr = ir3_COV(build, src, TYPE_U32, TYPE_S16); bool shared = (src->dsts[0]->flags & IR3_REG_SHARED); switch (align) { @@ -439,18 +439,18 @@ create_addr0(struct ir3_block *block, struct ir3_instruction *src, int align) break; case 2: /* src *= 2 => src <<= 1: */ - immed = create_immed_typed_shared(block, 1, TYPE_S16, shared); - instr = ir3_SHL_B(block, instr, 0, immed, 0); + immed = create_immed_typed_shared(build, 1, TYPE_S16, shared); + instr = ir3_SHL_B(build, instr, 0, immed, 0); break; case 3: /* src *= 3: */ - immed = create_immed_typed_shared(block, 3, TYPE_S16, shared); - instr = ir3_MULL_U(block, instr, 0, immed, 0); + immed = create_immed_typed_shared(build, 3, TYPE_S16, shared); + instr = ir3_MULL_U(build, instr, 0, immed, 0); break; case 4: /* src *= 4 => src <<= 2: */ - immed = create_immed_typed_shared(block, 2, TYPE_S16, shared); - instr = ir3_SHL_B(block, instr, 0, immed, 0); + immed = create_immed_typed_shared(build, 2, TYPE_S16, shared); + instr = ir3_SHL_B(build, instr, 0, immed, 0); break; default: unreachable("bad align"); @@ -459,7 +459,7 @@ create_addr0(struct ir3_block *block, struct ir3_instruction *src, int align) instr->dsts[0]->flags |= IR3_REG_HALF; - instr = ir3_MOV(block, instr, TYPE_S16); + instr = ir3_MOV(build, instr, TYPE_S16); instr->dsts[0]->num = regid(REG_A0, 0); instr->dsts[0]->flags &= ~IR3_REG_SHARED; @@ -467,11 +467,11 @@ create_addr0(struct ir3_block *block, struct ir3_instruction *src, int align) } static struct ir3_instruction * -create_addr1(struct ir3_block *block, unsigned const_val) +create_addr1(struct ir3_builder *build, unsigned const_val) { struct ir3_instruction *immed = - create_immed_typed(block, const_val, TYPE_U16); - struct ir3_instruction *instr = ir3_MOV(block, immed, TYPE_U16); + create_immed_typed(build, const_val, TYPE_U16); + struct ir3_instruction *instr = ir3_MOV(build, immed, TYPE_U16); instr->dsts[0]->num = regid(REG_A0, 1); return instr; } @@ -497,7 +497,7 @@ ir3_get_addr0(struct ir3_context *ctx, struct ir3_instruction *src, int align) return entry->data; } - addr = create_addr0(ctx->block, src, align); + addr = create_addr0(&ctx->build, src, align); _mesa_hash_table_insert(ctx->addr0_ht[idx], src, addr); return addr; @@ -517,7 +517,7 @@ ir3_get_addr1(struct ir3_context *ctx, unsigned const_val) return addr; } - addr = create_addr1(ctx->block, const_val); + addr = create_addr1(&ctx->build, const_val); _mesa_hash_table_u64_insert(ctx->addr1_ht, const_val, addr); return addr; @@ -533,28 +533,20 @@ ir3_get_predicate(struct ir3_context *ctx, struct ir3_instruction *src) if (src_entry) return src_entry->data; - struct ir3_block *b = src->block; + struct ir3_builder b = ir3_builder_at(ir3_after_instr_and_phis(src)); struct ir3_instruction *cond; /* NOTE: we use cpms.s.ne x, 0 to move x into a predicate register */ struct ir3_instruction *zero = - create_immed_typed_shared(b, 0, is_half(src) ? TYPE_U16 : TYPE_U32, + create_immed_typed_shared(&b, 0, is_half(src) ? TYPE_U16 : TYPE_U32, src->dsts[0]->flags & IR3_REG_SHARED); - cond = ir3_CMPS_S(b, src, 0, zero, 0); + cond = ir3_CMPS_S(&b, src, 0, zero, 0); cond->cat2.condition = IR3_COND_NE; /* condition always goes in predicate register: */ cond->dsts[0]->flags |= IR3_REG_PREDICATE; cond->dsts[0]->flags &= ~IR3_REG_SHARED; - /* phi's should stay first in a block */ - if (src->opc == OPC_META_PHI) - ir3_instr_move_after(zero, ir3_block_get_last_phi(src->block)); - else - ir3_instr_move_after(zero, src); - - ir3_instr_move_after(cond, zero); - _mesa_hash_table_insert(ctx->predicate_conversions, src, cond); return cond; } @@ -605,7 +597,7 @@ ir3_create_array_load(struct ir3_context *ctx, struct ir3_array *arr, int n, struct ir3_register *src; unsigned flags = 0; - mov = ir3_instr_create(block, OPC_MOV, 1, 1); + mov = ir3_build_instr(&ctx->build, OPC_MOV, 1, 1); if (arr->half) { mov->cat1.src_type = TYPE_U16; mov->cat1.dst_type = TYPE_U16; @@ -645,7 +637,7 @@ ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n, struct ir3_register *dst; unsigned flags = 0; - mov = ir3_instr_create(block, OPC_MOV, 1, 1); + mov = ir3_build_instr(&ctx->build, OPC_MOV, 1, 1); if (arr->half) { mov->cat1.src_type = TYPE_U16; mov->cat1.dst_type = TYPE_U16; @@ -700,7 +692,7 @@ ir3_lower_imm_offset(struct ir3_context *ctx, nir_intrinsic_instr *intr, */ uint32_t full_offset = base + nir_const_offset->u32; *offset = - create_immed(ctx->block, ROUND_DOWN_TO(full_offset, imm_offset_bound)); + create_immed(&ctx->build, ROUND_DOWN_TO(full_offset, imm_offset_bound)); *imm_offset = full_offset % imm_offset_bound; } else { *offset = ir3_get_src(ctx, offset_src)[0]; diff --git a/src/freedreno/ir3/ir3_context.h b/src/freedreno/ir3/ir3_context.h index b1d32a85dec..c944bfdeb3c 100644 --- a/src/freedreno/ir3/ir3_context.h +++ b/src/freedreno/ir3/ir3_context.h @@ -48,6 +48,7 @@ struct ir3_context { struct ir3_instruction **outputs; struct ir3_block *block; /* the current block */ + struct ir3_builder build; struct ir3_block *in_block; /* block created for shader inputs */ nir_function_impl *impl; @@ -181,6 +182,13 @@ struct ir3_context *ir3_context_init(struct ir3_compiler *compiler, struct ir3_shader_variant *so); void ir3_context_free(struct ir3_context *ctx); +static inline void +ir3_context_set_block(struct ir3_context *ctx, struct ir3_block *block) +{ + ctx->block = block; + ctx->build = ir3_builder_at(ir3_before_terminator(block)); +} + struct ir3_instruction **ir3_get_dst_ssa(struct ir3_context *ctx, nir_def *dst, unsigned n); struct ir3_instruction **ir3_get_def(struct ir3_context *ctx, nir_def *def, @@ -197,10 +205,10 @@ ir3_get_src(struct ir3_context *ctx, nir_src *src) } void ir3_put_def(struct ir3_context *ctx, nir_def *def); -struct ir3_instruction *ir3_create_collect(struct ir3_block *block, +struct ir3_instruction *ir3_create_collect(struct ir3_builder *build, struct ir3_instruction *const *arr, unsigned arrsz); -void ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst, +void ir3_split_dest(struct ir3_builder *build, struct ir3_instruction **dst, struct ir3_instruction *src, unsigned base, unsigned n); void ir3_handle_bindless_cat6(struct ir3_instruction *instr, nir_src rsrc); void ir3_handle_nonuniform(struct ir3_instruction *instr, @@ -209,10 +217,10 @@ void emit_intrinsic_image_size_tex(struct ir3_context *ctx, nir_intrinsic_instr *intr, struct ir3_instruction **dst); -#define ir3_collect(block, ...) \ +#define ir3_collect(build, ...) \ ({ \ struct ir3_instruction *__arr[] = {__VA_ARGS__}; \ - ir3_create_collect(block, __arr, ARRAY_SIZE(__arr)); \ + ir3_create_collect(build, __arr, ARRAY_SIZE(__arr)); \ }) NORETURN void ir3_context_error(struct ir3_context *ctx, const char *format, diff --git a/src/freedreno/ir3/ir3_image.c b/src/freedreno/ir3/ir3_image.c index 2cb5058ff7d..422a005a181 100644 --- a/src/freedreno/ir3/ir3_image.c +++ b/src/freedreno/ir3/ir3_image.c @@ -49,13 +49,13 @@ ir3_image_to_ibo(struct ir3_context *ctx, nir_src src) if (nir_src_is_const(src)) { int image_idx = nir_src_as_uint(src); - return create_immed(ctx->block, ctx->s->info.num_ssbos + image_idx); + return create_immed(&ctx->build, ctx->s->info.num_ssbos + image_idx); } else { struct ir3_instruction *image_idx = ir3_get_src(ctx, &src)[0]; if (ctx->s->info.num_ssbos) { - return ir3_ADD_U(ctx->block, + return ir3_ADD_U(&ctx->build, image_idx, 0, - create_immed(ctx->block, ctx->s->info.num_ssbos), 0); + create_immed(&ctx->build, ctx->s->info.num_ssbos), 0); } else { return image_idx; } diff --git a/src/freedreno/ir3/ir3_legalize.c b/src/freedreno/ir3/ir3_legalize.c index e13aaeb890f..bda4a8feb75 100644 --- a/src/freedreno/ir3/ir3_legalize.c +++ b/src/freedreno/ir3/ir3_legalize.c @@ -330,6 +330,7 @@ legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) struct ir3_legalize_state *state = &bd->begin_state; bool last_input_needs_ss = false; bool mergedregs = ctx->so->mergedregs; + struct ir3_builder build = ir3_builder_at(ir3_after_block(block)); /* Our input state is the OR of all predecessor blocks' state. * @@ -541,7 +542,7 @@ legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) */ if ((n->flags & IR3_INSTR_SS) && (opc_cat(n->opc) >= 5)) { struct ir3_instruction *nop; - nop = ir3_NOP(block); + nop = ir3_NOP(&build); nop->flags |= IR3_INSTR_SS; n->flags &= ~IR3_INSTR_SS; last_n = nop; @@ -576,7 +577,7 @@ legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) if (delay > 0) { assert(delay <= 6); - ir3_NOP(block)->repeat = delay - 1; + ir3_NOP(&build)->repeat = delay - 1; cycle += delay; } @@ -622,7 +623,7 @@ legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) ctx->has_tex_prefetch = true; } else if (n->opc == OPC_RESINFO && n->dsts_count > 0) { regmask_set(&state->needs_ss, n->dsts[0]); - ir3_NOP(block)->flags |= IR3_INSTR_SS; + ir3_NOP(&build)->flags |= IR3_INSTR_SS; last_input_needs_ss = false; } else if (is_load(n)) { if (is_local_mem_load(n)) @@ -717,7 +718,7 @@ legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) struct ir3_instruction *baryf; /* (ss)bary.f (ei)r63.x, 0, r0.x */ - baryf = ir3_instr_create(block, OPC_BARY_F, 1, 2); + baryf = ir3_build_instr(&build, OPC_BARY_F, 1, 2); ir3_dst_create(baryf, regid(63, 0), 0); ir3_src_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0; ir3_src_create(baryf, regid(0, 0), 0); @@ -746,7 +747,7 @@ legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) struct ir3_instruction *baryf; /* (ss)bary.f (ei)r63.x, 0, r0.x */ - baryf = ir3_instr_create(block, OPC_BARY_F, 1, 2); + baryf = ir3_build_instr(&build, OPC_BARY_F, 1, 2); ir3_dst_create(baryf, regid(63, 0), 0)->flags |= IR3_REG_EI; ir3_src_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0; ir3_src_create(baryf, regid(0, 0), 0); @@ -851,8 +852,7 @@ apply_push_consts_load_macro(struct ir3_legalize_ctx *ctx, { foreach_instr (n, &block->instr_list) { if (n->opc == OPC_PUSH_CONSTS_LOAD_MACRO) { - struct ir3_instruction *stsc = ir3_instr_create(block, OPC_STSC, 0, 2); - ir3_instr_move_after(stsc, n); + struct ir3_instruction *stsc = ir3_instr_create_at(ir3_after_instr(n), OPC_STSC, 0, 2); ir3_src_create(stsc, 0, IR3_REG_IMMED)->iim_val = n->push_consts.dst_base; ir3_src_create(stsc, 0, IR3_REG_IMMED)->iim_val = @@ -861,8 +861,8 @@ apply_push_consts_load_macro(struct ir3_legalize_ctx *ctx, stsc->cat6.type = TYPE_U32; if (ctx->compiler->stsc_duplication_quirk) { - struct ir3_instruction *nop = ir3_NOP(block); - ir3_instr_move_after(nop, stsc); + struct ir3_builder build = ir3_builder_at(ir3_after_instr(stsc)); + struct ir3_instruction *nop = ir3_NOP(&build); nop->flags |= IR3_INSTR_SS; ir3_instr_move_after(ir3_instr_clone(stsc), nop); } @@ -1171,7 +1171,8 @@ block_sched(struct ir3 *ir) br1 = terminator; br1->cat0.target = block->successors[1]; - br2 = ir3_JUMP(block); + struct ir3_builder build = ir3_builder_at(ir3_after_block(block)); + br2 = ir3_JUMP(&build); br2->cat0.target = block->successors[0]; } else if (opc == OPC_BR || opc == OPC_BRAA || opc == OPC_BRAO || opc == OPC_BALL || opc == OPC_BANY) { @@ -1219,13 +1220,15 @@ add_predication_workaround(struct ir3_compiler *compiler, struct ir3_instruction *prede) { if (predtf && compiler->predtf_nop_quirk) { - struct ir3_instruction *nop = ir3_NOP(predtf->block); + struct ir3_builder build = ir3_builder_at(ir3_after_block(predtf->block)); + struct ir3_instruction *nop = ir3_NOP(&build); nop->repeat = 4; ir3_instr_move_after(nop, predtf); } if (compiler->prede_nop_quirk) { - struct ir3_instruction *nop = ir3_NOP(prede->block); + struct ir3_builder build = ir3_builder_at(ir3_after_block(prede->block)); + struct ir3_instruction *nop = ir3_NOP(&build); nop->repeat = 6; ir3_instr_move_after(nop, prede); } @@ -1300,7 +1303,9 @@ prede_sched(struct ir3 *ir) * |----------| */ if (!list_is_empty(&succ1->instr_list)) { - struct ir3_instruction *prede = ir3_PREDE(succ1); + struct ir3_builder build = + ir3_builder_at(ir3_before_terminator(succ1)); + struct ir3_instruction *prede = ir3_PREDE(&build); add_predication_workaround(ir->compiler, succ0_terminator, prede); continue; } @@ -1321,7 +1326,8 @@ prede_sched(struct ir3 *ir) * |----------| */ list_delinit(&succ0_terminator->node); - struct ir3_instruction *prede = ir3_PREDE(succ0); + struct ir3_builder build = ir3_builder_at(ir3_before_terminator(succ0)); + struct ir3_instruction *prede = ir3_PREDE(&build); add_predication_workaround(ir->compiler, NULL, prede); remove_unused_block(succ1); block->successors[1] = succ0->successors[0]; @@ -1372,15 +1378,12 @@ kill_sched(struct ir3 *ir, struct ir3_shader_variant *so) if (instr->opc != OPC_KILL) continue; - struct ir3_instruction *br = ir3_instr_create(block, OPC_BR, 0, 1); + struct ir3_instruction *br = ir3_instr_create_at(ir3_after_instr(instr), OPC_BR, 0, 1); ir3_src_create(br, instr->srcs[0]->num, instr->srcs[0]->flags)->wrmask = 1; br->cat0.target = list_last_entry(&ir->block_list, struct ir3_block, node); - list_del(&br->node); - list_add(&br->node, &instr->node); - added = true; } } @@ -1405,9 +1408,9 @@ dbg_sync_sched(struct ir3 *ir, struct ir3_shader_variant *so) foreach_block (block, &ir->block_list) { foreach_instr_safe (instr, &block->instr_list) { if (is_ss_producer(instr) || is_sy_producer(instr)) { - struct ir3_instruction *nop = ir3_NOP(block); + struct ir3_builder build = ir3_builder_at(ir3_after_instr(instr)); + struct ir3_instruction *nop = ir3_NOP(&build); nop->flags |= IR3_INSTR_SS | IR3_INSTR_SY; - ir3_instr_move_after(nop, instr); } } } @@ -1418,9 +1421,9 @@ dbg_nop_sched(struct ir3 *ir, struct ir3_shader_variant *so) { foreach_block (block, &ir->block_list) { foreach_instr_safe (instr, &block->instr_list) { - struct ir3_instruction *nop = ir3_NOP(block); + struct ir3_builder build = ir3_builder_at(ir3_before_instr(instr)); + struct ir3_instruction *nop = ir3_NOP(&build); nop->repeat = 5; - ir3_instr_move_before(nop, instr); } } } @@ -1669,10 +1672,11 @@ helper_sched(struct ir3_legalize_ctx *ctx, struct ir3 *ir, */ if (!killed && (expensive_instruction_in_block || block->successors[0] != ir3_end_block(ir))) { - struct ir3_instruction *nop = ir3_NOP(block); + struct ir3_cursor cursor = first_instr ? ir3_before_instr(first_instr) + : ir3_before_terminator(block); + struct ir3_builder build = ir3_builder_at(cursor); + struct ir3_instruction *nop = ir3_NOP(&build); nop->flags |= IR3_INSTR_EQ; - if (first_instr) - ir3_instr_move_before(nop, first_instr); } } } diff --git a/src/freedreno/ir3/ir3_lower_parallelcopy.c b/src/freedreno/ir3/ir3_lower_parallelcopy.c index 9e431df08a1..fbed64b6954 100644 --- a/src/freedreno/ir3/ir3_lower_parallelcopy.c +++ b/src/freedreno/ir3/ir3_lower_parallelcopy.c @@ -55,12 +55,10 @@ do_xor(struct ir3_instruction *instr, unsigned dst_num, unsigned src1_num, unsigned src2_num, unsigned flags) { struct ir3_instruction * xor - = ir3_instr_create(instr->block, OPC_XOR_B, 1, 2); + = ir3_instr_create_at(ir3_before_instr(instr), OPC_XOR_B, 1, 2); ir3_dst_create(xor, dst_num, flags); ir3_src_create(xor, src1_num, flags); ir3_src_create(xor, src2_num, flags); - - ir3_instr_move_before(xor, instr); } static void @@ -144,7 +142,7 @@ do_swap(struct ir3_compiler *compiler, struct ir3_instruction *instr, do_xor(instr, src_num, src_num, dst_num, entry->flags); do_xor(instr, dst_num, dst_num, src_num, entry->flags); } else { - struct ir3_instruction *swz = ir3_instr_create(instr->block, OPC_SWZ, 2, 2); + struct ir3_instruction *swz = ir3_instr_create_at(ir3_before_instr(instr), OPC_SWZ, 2, 2); ir3_dst_create(swz, dst_num, entry->flags); ir3_dst_create(swz, src_num, entry->flags); ir3_src_create(swz, src_num, entry->flags); @@ -152,7 +150,6 @@ do_swap(struct ir3_compiler *compiler, struct ir3_instruction *instr, swz->cat1.dst_type = (entry->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32; swz->cat1.src_type = (entry->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32; swz->repeat = 1; - ir3_instr_move_before(swz, instr); } } @@ -206,20 +203,18 @@ do_copy(struct ir3_compiler *compiler, struct ir3_instruction *instr, if (entry->src.reg % 2 == 0) { /* cov.u32u16 dst, src */ struct ir3_instruction *cov = - ir3_instr_create(instr->block, OPC_MOV, 1, 1); + ir3_instr_create_at(ir3_before_instr(instr), OPC_MOV, 1, 1); ir3_dst_create(cov, dst_num, entry->flags); ir3_src_create(cov, src_num, entry->flags & ~IR3_REG_HALF); cov->cat1.dst_type = TYPE_U16; cov->cat1.src_type = TYPE_U32; - ir3_instr_move_before(cov, instr); } else { /* shr.b dst, src, (16) */ struct ir3_instruction *shr = - ir3_instr_create(instr->block, OPC_SHR_B, 1, 2); + ir3_instr_create_at(ir3_before_instr(instr), OPC_SHR_B, 1, 2); ir3_dst_create(shr, dst_num, entry->flags); ir3_src_create(shr, src_num, entry->flags & ~IR3_REG_HALF); ir3_src_create(shr, 0, IR3_REG_IMMED)->uim_val = 16; - ir3_instr_move_before(shr, instr); } return; } @@ -228,7 +223,7 @@ do_copy(struct ir3_compiler *compiler, struct ir3_instruction *instr, unsigned src_num = ra_physreg_to_num(entry->src.reg, entry->flags); unsigned dst_num = ra_physreg_to_num(entry->dst, entry->flags); - struct ir3_instruction *mov = ir3_instr_create(instr->block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_before_instr(instr), OPC_MOV, 1, 1); ir3_dst_create(mov, dst_num, entry->flags); if (entry->src.flags & (IR3_REG_IMMED | IR3_REG_CONST)) ir3_src_create(mov, INVALID_REG, (entry->flags & IR3_REG_HALF) | entry->src.flags); @@ -240,7 +235,6 @@ do_copy(struct ir3_compiler *compiler, struct ir3_instruction *instr, mov->srcs[0]->uim_val = entry->src.imm; else if (entry->src.flags & IR3_REG_CONST) mov->srcs[0]->num = entry->src.const_num; - ir3_instr_move_before(mov, instr); } struct copy_ctx { @@ -570,7 +564,10 @@ ir3_lower_copies(struct ir3_shader_variant *v) src_num++, dst_num++) { if (src_num & 1) { for (unsigned i = 0; i < 2; i++) { - struct ir3_instruction *swz = ir3_instr_create(instr->block, OPC_SWZ, 2, 2); + struct ir3_cursor cursor = i == 0 + ? ir3_before_instr(instr) + : ir3_after_instr(instr); + struct ir3_instruction *swz = ir3_instr_create_at(cursor, OPC_SWZ, 2, 2); ir3_dst_create(swz, src_num - 1, IR3_REG_HALF); ir3_dst_create(swz, src_num, IR3_REG_HALF); ir3_src_create(swz, src_num, IR3_REG_HALF); @@ -578,15 +575,11 @@ ir3_lower_copies(struct ir3_shader_variant *v) swz->cat1.dst_type = TYPE_U16; swz->cat1.src_type = TYPE_U16; swz->repeat = 1; - if (i == 0) - ir3_instr_move_before(swz, instr); - else - ir3_instr_move_after(swz, instr); } } - struct ir3_instruction *mov = - ir3_instr_create(instr->block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at( + ir3_before_instr(instr), OPC_MOV, 1, 1); ir3_dst_create(mov, dst_num, instr->dsts[0]->flags); ir3_src_create(mov, src_num / 2, @@ -600,8 +593,6 @@ ir3_lower_copies(struct ir3_shader_variant *v) instr->cat1.src_type == TYPE_S16); mov->cat1.src_type = TYPE_U32; mov->cat1.dst_type = TYPE_U16; - - ir3_instr_move_before(mov, instr); } list_del(&instr->node); diff --git a/src/freedreno/ir3/ir3_lower_shared_phi.c b/src/freedreno/ir3/ir3_lower_shared_phi.c index e39d8d2f871..019359a533c 100644 --- a/src/freedreno/ir3/ir3_lower_shared_phi.c +++ b/src/freedreno/ir3/ir3_lower_shared_phi.c @@ -41,7 +41,7 @@ lower_phi(void *ctx, struct ir3_instruction *phi) for (unsigned i = 0; i < block->predecessors_count; i++) { struct ir3_block *pred = block->predecessors[i]; if (phi->srcs[i]->def) { - struct ir3_instruction *pred_mov = ir3_instr_create(pred, OPC_MOV, 1, 1); + struct ir3_instruction *pred_mov = ir3_instr_create_at(ir3_before_terminator(pred), OPC_MOV, 1, 1); pred_mov->uses = _mesa_pointer_set_create(ctx); __ssa_dst(pred_mov)->flags |= (phi->srcs[i]->flags & IR3_REG_HALF); unsigned src_flags = IR3_REG_SSA | IR3_REG_SHARED | @@ -60,12 +60,12 @@ lower_phi(void *ctx, struct ir3_instruction *phi) phi->dsts[0]->flags &= ~IR3_REG_SHARED; + struct ir3_builder build = ir3_builder_at(ir3_after_phis(block)); struct ir3_instruction *shared_mov = - ir3_MOV(block, phi, + ir3_MOV(&build, phi, (phi->dsts[0]->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32); shared_mov->uses = _mesa_pointer_set_create(ctx); shared_mov->dsts[0]->flags |= IR3_REG_SHARED; - ir3_instr_move_after_phis(shared_mov, block); foreach_ssa_use (use, phi) { for (unsigned i = 0; i < use->srcs_count; i++) { diff --git a/src/freedreno/ir3/ir3_lower_spill.c b/src/freedreno/ir3/ir3_lower_spill.c index 86f1eb5b07e..69f5d626213 100644 --- a/src/freedreno/ir3/ir3_lower_spill.c +++ b/src/freedreno/ir3/ir3_lower_spill.c @@ -36,12 +36,10 @@ component_bytes(struct ir3_register *src) static void set_base_reg(struct ir3_instruction *mem, unsigned val) { - struct ir3_instruction *mov = ir3_instr_create(mem->block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_before_instr(mem), OPC_MOV, 1, 1); ir3_dst_create(mov, mem->srcs[0]->num, mem->srcs[0]->flags); ir3_src_create(mov, INVALID_REG, IR3_REG_IMMED)->uim_val = val; mov->cat1.dst_type = mov->cat1.src_type = TYPE_U32; - - ir3_instr_move_before(mov, mem); } static void @@ -55,12 +53,10 @@ reset_base_reg(struct ir3_instruction *mem) if (base->flags & IR3_REG_KILL) return; - struct ir3_instruction *mov = ir3_instr_create(mem->block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_after_instr(mem), OPC_MOV, 1, 1); ir3_dst_create(mov, base->num, base->flags); ir3_src_create(mov, INVALID_REG, IR3_REG_IMMED)->uim_val = 0; mov->cat1.dst_type = mov->cat1.src_type = TYPE_U32; - - ir3_instr_move_after(mov, mem); } /* There are 13 bits, but 1 << 12 will be sign-extended into a negative offset diff --git a/src/freedreno/ir3/ir3_lower_subgroups.c b/src/freedreno/ir3/ir3_lower_subgroups.c index 3ca05932079..b522afc3a7b 100644 --- a/src/freedreno/ir3/ir3_lower_subgroups.c +++ b/src/freedreno/ir3/ir3_lower_subgroups.c @@ -44,7 +44,7 @@ replace_physical_pred(struct ir3_block *block, struct ir3_block *old_pred, static void mov_immed(struct ir3_register *dst, struct ir3_block *block, unsigned immed) { - struct ir3_instruction *mov = ir3_instr_create(block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_before_terminator(block), OPC_MOV, 1, 1); struct ir3_register *mov_dst = ir3_dst_create(mov, dst->num, dst->flags); mov_dst->wrmask = dst->wrmask; struct ir3_register *src = ir3_src_create( @@ -59,7 +59,7 @@ static void mov_reg(struct ir3_block *block, struct ir3_register *dst, struct ir3_register *src) { - struct ir3_instruction *mov = ir3_instr_create(block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_before_terminator(block), OPC_MOV, 1, 1); struct ir3_register *mov_dst = ir3_dst_create(mov, dst->num, dst->flags & (IR3_REG_HALF | IR3_REG_SHARED)); @@ -77,7 +77,7 @@ static void binop(struct ir3_block *block, opc_t opc, struct ir3_register *dst, struct ir3_register *src0, struct ir3_register *src1) { - struct ir3_instruction *instr = ir3_instr_create(block, opc, 1, 2); + struct ir3_instruction *instr = ir3_instr_create_at(ir3_before_terminator(block), opc, 1, 2); unsigned flags = dst->flags & IR3_REG_HALF; struct ir3_register *instr_dst = ir3_dst_create(instr, dst->num, flags); @@ -95,7 +95,7 @@ triop(struct ir3_block *block, opc_t opc, struct ir3_register *dst, struct ir3_register *src0, struct ir3_register *src1, struct ir3_register *src2) { - struct ir3_instruction *instr = ir3_instr_create(block, opc, 1, 3); + struct ir3_instruction *instr = ir3_instr_create_at(ir3_before_terminator(block), opc, 1, 3); unsigned flags = dst->flags & IR3_REG_HALF; struct ir3_register *instr_dst = ir3_dst_create(instr, dst->num, flags); @@ -201,7 +201,8 @@ link_blocks(struct ir3_block *pred, struct ir3_block *succ, unsigned index) static void link_blocks_jump(struct ir3_block *pred, struct ir3_block *succ) { - ir3_JUMP(pred); + struct ir3_builder build = ir3_builder_at(ir3_after_block(pred)); + ir3_JUMP(&build); link_blocks(pred, succ, 0); } @@ -211,7 +212,7 @@ link_blocks_branch(struct ir3_block *pred, struct ir3_block *target, struct ir3_instruction *condition) { unsigned nsrc = condition ? 1 : 0; - struct ir3_instruction *branch = ir3_instr_create(pred, opc, 0, nsrc); + struct ir3_instruction *branch = ir3_instr_create_at(ir3_after_block(pred), opc, 0, nsrc); branch->flags |= flags; if (condition) { @@ -464,7 +465,7 @@ lower_instr(struct ir3 *ir, struct ir3_block **block, struct ir3_instruction *in case OPC_BALLOT_MACRO: { unsigned comp_count = util_last_bit(instr->dsts[0]->wrmask); struct ir3_instruction *movmsk = - ir3_instr_create(then_block, OPC_MOVMSK, 1, 0); + ir3_instr_create_at(ir3_before_terminator(then_block), OPC_MOVMSK, 1, 0); ir3_dst_create(movmsk, instr->dsts[0]->num, instr->dsts[0]->flags); movmsk->repeat = comp_count - 1; break; @@ -473,7 +474,7 @@ lower_instr(struct ir3 *ir, struct ir3_block **block, struct ir3_instruction *in case OPC_READ_GETLAST_MACRO: case OPC_READ_COND_MACRO: { struct ir3_instruction *mov = - ir3_instr_create(then_block, OPC_MOV, 1, 1); + ir3_instr_create_at(ir3_before_terminator(then_block), OPC_MOV, 1, 1); ir3_dst_create(mov, instr->dsts[0]->num, instr->dsts[0]->flags); struct ir3_register *new_src = ir3_src_create(mov, 0, 0); unsigned idx = instr->opc == OPC_READ_COND_MACRO ? 1 : 0; diff --git a/src/freedreno/ir3/ir3_merge_regs.c b/src/freedreno/ir3/ir3_merge_regs.c index ec4b553825a..f515f96e050 100644 --- a/src/freedreno/ir3/ir3_merge_regs.c +++ b/src/freedreno/ir3/ir3_merge_regs.c @@ -467,7 +467,7 @@ create_parallel_copy(struct ir3_block *block) assert(j == phi_count); struct ir3_instruction *pcopy = - ir3_instr_create(block, OPC_META_PARALLEL_COPY, phi_count, phi_count); + ir3_instr_create_at(ir3_before_terminator(block), OPC_META_PARALLEL_COPY, phi_count, phi_count); for (j = 0; j < phi_count; j++) { struct ir3_register *reg = __ssa_dst(pcopy); diff --git a/src/freedreno/ir3/ir3_ra.c b/src/freedreno/ir3/ir3_ra.c index f7c412175f7..b0dc5e264ee 100644 --- a/src/freedreno/ir3/ir3_ra.c +++ b/src/freedreno/ir3/ir3_ra.c @@ -1636,7 +1636,7 @@ insert_parallel_copy_instr(struct ra_ctx *ctx, struct ir3_instruction *instr) return; struct ir3_instruction *pcopy = - ir3_instr_create(instr->block, OPC_META_PARALLEL_COPY, + ir3_instr_create_at(ir3_before_instr(instr), OPC_META_PARALLEL_COPY, ctx->parallel_copies_count, ctx->parallel_copies_count); for (unsigned i = 0; i < ctx->parallel_copies_count; i++) { @@ -1661,8 +1661,6 @@ insert_parallel_copy_instr(struct ra_ctx *ctx, struct ir3_instruction *instr) assign_reg(pcopy, reg, ra_physreg_to_num(entry->src, reg->flags)); } - list_del(&pcopy->node); - list_addtail(&pcopy->node, &instr->node); ctx->parallel_copies_count = 0; } @@ -2113,8 +2111,8 @@ insert_liveout_copy(struct ir3_block *block, physreg_t dst, physreg_t src, old_pcopy = last; unsigned old_pcopy_srcs = old_pcopy ? old_pcopy->srcs_count : 0; - struct ir3_instruction *pcopy = ir3_instr_create( - block, OPC_META_PARALLEL_COPY, old_pcopy_srcs + 1, old_pcopy_srcs + 1); + struct ir3_instruction *pcopy = ir3_instr_create_at( + ir3_before_terminator(block), OPC_META_PARALLEL_COPY, old_pcopy_srcs + 1, old_pcopy_srcs + 1); for (unsigned i = 0; i < old_pcopy_srcs; i++) { old_pcopy->dsts[i]->instr = pcopy; diff --git a/src/freedreno/ir3/ir3_shared_folding.c b/src/freedreno/ir3/ir3_shared_folding.c index ee62a4c59e0..51c3596be4a 100644 --- a/src/freedreno/ir3/ir3_shared_folding.c +++ b/src/freedreno/ir3/ir3_shared_folding.c @@ -46,7 +46,7 @@ try_shared_folding(struct ir3_instruction *mov, void *mem_ctx) for (unsigned i = 0; i < block->predecessors_count; i++) { struct ir3_block *pred = block->predecessors[i]; if (src->srcs[i]->def) { - struct ir3_instruction *pred_mov = ir3_instr_create(pred, OPC_MOV, 1, 1); + struct ir3_instruction *pred_mov = ir3_instr_create_at(ir3_before_terminator(pred), OPC_MOV, 1, 1); __ssa_dst(pred_mov)->flags |= (src->srcs[i]->flags & IR3_REG_HALF); unsigned src_flags = IR3_REG_SSA | IR3_REG_SHARED | (src->srcs[i]->flags & IR3_REG_HALF); @@ -95,12 +95,10 @@ try_shared_folding(struct ir3_instruction *mov, void *mem_ctx) continue; if (!shared_mov) { - shared_mov = ir3_MOV(src->block, src, mov->cat1.src_type); + struct ir3_builder build = + ir3_builder_at(ir3_after_instr_and_phis(src)); + shared_mov = ir3_MOV(&build, src, mov->cat1.src_type); shared_mov->dsts[0]->flags |= IR3_REG_SHARED; - if (src->opc == OPC_META_PHI) - ir3_instr_move_after_phis(shared_mov, src->block); - else - ir3_instr_move_after(shared_mov, src); shared_mov->uses = _mesa_pointer_set_create(mem_ctx); } diff --git a/src/freedreno/ir3/ir3_shared_ra.c b/src/freedreno/ir3/ir3_shared_ra.c index 5812afa0609..ab0ec3f9077 100644 --- a/src/freedreno/ir3/ir3_shared_ra.c +++ b/src/freedreno/ir3/ir3_shared_ra.c @@ -391,14 +391,13 @@ split(struct ir3_register *def, unsigned offset, struct ir3_instruction *before) } struct ir3_instruction *split = - ir3_instr_create(before->block, OPC_META_SPLIT, 1, 1); + ir3_instr_create_at(ir3_after_instr(before), OPC_META_SPLIT, 1, 1); split->split.off = offset; struct ir3_register *dst = __ssa_dst(split); struct ir3_register *src = ir3_src_create(split, INVALID_REG, def->flags & (IR3_REG_HALF | IR3_REG_SSA)); src->wrmask = def->wrmask; src->def = def; - ir3_instr_move_after(split, before); return dst; } @@ -413,13 +412,11 @@ extract(struct ir3_register *parent_def, unsigned offset, unsigned elems, return split(parent_def, offset, before); struct ir3_instruction *collect = - ir3_instr_create(before->block, OPC_META_COLLECT, 1, elems); + ir3_instr_create_at(ir3_after_instr(before), OPC_META_COLLECT, 1, elems); struct ir3_register *dst = __ssa_dst(collect); dst->flags |= parent_def->flags & IR3_REG_HALF; dst->wrmask = MASK(elems); - ir3_instr_move_after(collect, before); - for (unsigned i = 0; i < elems; i++) { ir3_src_create(collect, INVALID_REG, parent_def->flags & (IR3_REG_HALF | IR3_REG_SSA))->def = @@ -469,7 +466,7 @@ spill_interval(struct ra_ctx *ctx, struct ra_interval *interval) before = last_phi_input; } - struct ir3_instruction *mov = ir3_instr_create(before->block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_after_instr(before), OPC_MOV, 1, 1); mov->flags |= IR3_INSTR_SHARED_SPILL; struct ir3_register *dst = __ssa_dst(mov); dst->flags |= (interval->interval.reg->flags & IR3_REG_HALF); @@ -482,7 +479,6 @@ spill_interval(struct ra_ctx *ctx, struct ra_interval *interval) mov->cat1.src_type = mov->cat1.dst_type = (interval->interval.reg->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32; - ir3_instr_move_after(mov, before); interval->spill_def = dst; } @@ -697,11 +693,11 @@ reload_src(struct ra_ctx *ctx, struct ir3_instruction *instr, } static void -reload_interval(struct ra_ctx *ctx, struct ir3_instruction *instr, - struct ir3_block *block, struct ra_interval *interval) +reload_interval(struct ra_ctx *ctx, struct ir3_cursor cursor, + struct ra_interval *interval) { struct ir3_register *def = interval->interval.reg; - struct ir3_instruction *mov = ir3_instr_create(block, OPC_MOV, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(cursor, OPC_MOV, 1, 1); mov->flags |= IR3_INSTR_SHARED_SPILL; unsigned flags = IR3_REG_SHARED | (def->flags & IR3_REG_HALF); ir3_dst_create(mov, ra_physreg_to_num(interval->physreg_start, flags), @@ -715,9 +711,6 @@ reload_interval(struct ra_ctx *ctx, struct ir3_instruction *instr, mov_src->wrmask = def->wrmask; mov->cat1.src_type = mov->cat1.dst_type = (def->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32; - - if (instr) - ir3_instr_move_before(mov, instr); } static void @@ -730,7 +723,7 @@ reload_src_finalize(struct ra_ctx *ctx, struct ir3_instruction *instr, if (!interval->needs_reload) return; - reload_interval(ctx, instr, instr->block, interval); + reload_interval(ctx, ir3_before_instr(instr), interval); interval->needs_reload = false; } @@ -903,13 +896,12 @@ handle_dst(struct ra_ctx *ctx, struct ir3_instruction *instr, d("insert dst %u physreg %u", dst->name, physreg); if (dst->tied) { - struct ir3_instruction *mov = ir3_instr_create(instr->block, OPC_META_PARALLEL_COPY, 1, 1); + struct ir3_instruction *mov = ir3_instr_create_at(ir3_before_instr(instr), OPC_META_PARALLEL_COPY, 1, 1); unsigned flags = IR3_REG_SHARED | (dst->flags & IR3_REG_HALF); ir3_dst_create(mov, dst->num, flags)->wrmask = dst->wrmask; ir3_src_create(mov, dst->tied->num, flags)->wrmask = dst->wrmask; mov->cat1.src_type = mov->cat1.dst_type = (dst->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32;; - ir3_instr_move_before(mov, instr); dst->tied->num = dst->num; } } @@ -975,14 +967,13 @@ handle_split(struct ra_ctx *ctx, struct ir3_instruction *split) if (src_interval->spill_def) { struct ir3_instruction *spill_split = - ir3_instr_create(split->block, OPC_META_SPLIT, 1, 1); + ir3_instr_create_at(ir3_after_instr(split), OPC_META_SPLIT, 1, 1); struct ir3_register *dst = __ssa_dst(spill_split); struct ir3_register *src = ir3_src_create(spill_split, INVALID_REG, IR3_REG_SSA); src->def = src_interval->spill_def; src->wrmask = src_interval->spill_def->wrmask; spill_split->split.off = split->split.off; - ir3_instr_move_after(spill_split, split); dst_interval->spill_def = dst; list_del(&split->node); return; @@ -1145,7 +1136,7 @@ reload_live_outs(struct ra_ctx *ctx, struct ir3_block *block) struct ra_interval *interval = &ctx->intervals[name]; if (!interval->interval.inserted) { d("reloading %d at end of backedge", reg->name); - reload_interval(ctx, NULL, block, interval); + reload_interval(ctx, ir3_before_terminator(block), interval); } } } @@ -1273,7 +1264,7 @@ lower_pcopy(struct ir3 *ir, struct ra_ctx *ctx) * non-shared->non-shared copy). */ struct ir3_instruction *mov = - ir3_instr_create(block, OPC_MOV, 1, 1); + ir3_instr_create_at(ir3_before_instr(instr), OPC_MOV, 1, 1); mov->flags |= IR3_INSTR_SHARED_SPILL; struct ir3_register *dst = ir3_dst_create(mov, INVALID_REG, instr->dsts[i]->flags); @@ -1289,7 +1280,6 @@ lower_pcopy(struct ir3 *ir, struct ra_ctx *ctx) (mov->dsts[0]->flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32; instr->srcs[i]->flags = mov->dsts[0]->flags; instr->srcs[i]->def = mov->dsts[0]; - ir3_instr_move_before(mov, instr); } } @@ -1300,7 +1290,7 @@ lower_pcopy(struct ir3 *ir, struct ra_ctx *ctx) /* non-shared->shared. Create a reload move. */ struct ir3_instruction *mov = - ir3_instr_create(block, OPC_MOV, 1, 1); + ir3_instr_create_at(ir3_after_instr(instr), OPC_MOV, 1, 1); mov->flags |= IR3_INSTR_SHARED_SPILL; struct ir3_register *dst = ir3_dst_create(mov, instr->dsts[i]->num, @@ -1345,7 +1335,6 @@ lower_pcopy(struct ir3 *ir, struct ra_ctx *ctx) instr->dsts[i] = instr->dsts[instr->dsts_count - 1]; instr->srcs_count--; instr->dsts_count--; - ir3_instr_move_after(mov, instr); continue; } @@ -1365,7 +1354,7 @@ lower_pcopy(struct ir3 *ir, struct ra_ctx *ctx) if (non_shared_copies != 0) { struct ir3_instruction *pcopy = - ir3_instr_create(block, OPC_META_PARALLEL_COPY, + ir3_instr_create_at(ir3_before_terminator(block), OPC_META_PARALLEL_COPY, non_shared_copies, non_shared_copies); unsigned j = 0; diff --git a/src/freedreno/ir3/ir3_spill.c b/src/freedreno/ir3/ir3_spill.c index 3988bd070f2..857ff971072 100644 --- a/src/freedreno/ir3/ir3_spill.c +++ b/src/freedreno/ir3/ir3_spill.c @@ -127,6 +127,7 @@ static void add_base_reg(struct ra_spill_ctx *ctx, struct ir3 *ir) { struct ir3_block *start = ir3_start_block(ir); + struct ir3_builder build = ir3_builder_at(ir3_after_block(start)); /* We need to stick it after any meta instructions which need to be first. */ struct ir3_instruction *after = NULL; @@ -138,7 +139,7 @@ add_base_reg(struct ra_spill_ctx *ctx, struct ir3 *ir) } } - struct ir3_instruction *mov = create_immed(start, 0); + struct ir3_instruction *mov = create_immed(&build, 0); if (after) ir3_instr_move_before(mov, after);