diff --git a/src/panfrost/compiler/bifrost/bifrost_compile.c b/src/panfrost/compiler/bifrost/bifrost_compile.c index 2f08cddc49e..989a36b7046 100644 --- a/src/panfrost/compiler/bifrost/bifrost_compile.c +++ b/src/panfrost/compiler/bifrost/bifrost_compile.c @@ -703,8 +703,10 @@ bi_emit_load_var_buf(bi_builder *b, nir_intrinsic_instr *intr) assert(intr->intrinsic == nir_intrinsic_load_var_buf_pan || intr->intrinsic == nir_intrinsic_load_var_buf_flat_pan); + const unsigned arch = b->shader->arch; + /* These are only available on Valhall+ */ - assert(b->shader->arch >= 9); + assert(arch >= 9); const bool flat = intr->intrinsic == nir_intrinsic_load_var_buf_flat_pan; const nir_alu_type src_type = nir_intrinsic_src_type(intr); @@ -757,19 +759,36 @@ bi_emit_load_var_buf(bi_builder *b, nir_intrinsic_instr *intr) bool use_imm_form = false; if (nir_src_is_const(intr->src[0])) { imm_offset = nir_src_as_uint(intr->src[0]); - assert(imm_offset < pan_ld_var_buf_off_size(b->shader->arch)); + assert(imm_offset < pan_ld_var_buf_off_size(arch)); use_imm_form = true; } + /* On v14+, flat source formats are removed from LD_VAR_BUF/LD_VAR_BUF_IMM, + * so flat buffer varyings must use the dedicated LD_VAR_BUF_FLAT*. + */ if (use_imm_form) { - bi_ld_var_buf_imm_to(b, sz, dest, src0, regfmt, sample, source_format, + if (arch >= 14 && flat) { + bi_ld_var_buf_flat_imm_to(b, dest, regfmt, vecsize, imm_offset); + } else { + bi_ld_var_buf_imm_to(b, sz, dest, src0, regfmt, sample, source_format, BI_UPDATE_STORE, vecsize, imm_offset); + } } else { bi_index offset = bi_src_index(&intr->src[0]); - bi_ld_var_buf_to(b, sz, dest, src0, offset, regfmt, sample, - source_format, BI_UPDATE_STORE, vecsize); + if (arch >= 14 && flat) { + bi_ld_var_buf_flat_to(b, dest, offset, regfmt, vecsize); + } else { + bi_ld_var_buf_to(b, sz, dest, src0, offset, regfmt, sample, + source_format, BI_UPDATE_STORE, vecsize); + } } + + /* LD_VAR_BUF_FLAT* only support register formats F16 and F32. */ + assert( + arch < 14 || !flat || + (regfmt == BI_REGISTER_FORMAT_F16 || regfmt == BI_REGISTER_FORMAT_F32)); + bi_split_def(b, &intr->def); } diff --git a/src/panfrost/compiler/bifrost/valhall/ISA.xml b/src/panfrost/compiler/bifrost/valhall/ISA.xml index 43b292f2c57..92d7ecbdc11 100644 --- a/src/panfrost/compiler/bifrost/valhall/ISA.xml +++ b/src/panfrost/compiler/bifrost/valhall/ISA.xml @@ -940,6 +940,32 @@ + + + + + Fetches a given flat varying from hardware buffer + + + + + + + + + + + + + Fetches a given flat varying from hardware buffer + + + + + + + + Interpolates a given varying from hardware buffer diff --git a/src/panfrost/compiler/bifrost/valhall/test/test-packing.cpp b/src/panfrost/compiler/bifrost/valhall/test/test-packing.cpp index 96e11ee180a..0ac71cc2f4f 100644 --- a/src/panfrost/compiler/bifrost/valhall/test/test-packing.cpp +++ b/src/panfrost/compiler/bifrost/valhall/test/test-packing.cpp @@ -294,6 +294,30 @@ TEST_F(ValhallPacking, LdVarBufImmF16) 11, 0x005d80443300083d); } +TEST_F(ValhallPacking, LdVarBufFlatImmFormat) +{ + CASE_ARCH(bi_ld_var_buf_flat_imm_to(b, bi_register(0), + BI_REGISTER_FORMAT_F32, + BI_VECSIZE_V4, 0x12), + 14, 0x0040800832001200); + + CASE_ARCH(bi_ld_var_buf_flat_imm_to(b, bi_register(0), + BI_REGISTER_FORMAT_F16, + BI_VECSIZE_V4, 0x12), + 14, 0x0040800433001200); +} + +TEST_F(ValhallPacking, LdVarBufFlat) +{ + CASE_ARCH(bi_ld_var_buf_flat_to(b, bi_register(0), bi_register(61), + BI_REGISTER_FORMAT_F32, BI_VECSIZE_V4), + 14, 0x005f80083200003d); + + CASE_ARCH(bi_ld_var_buf_flat_to(b, bi_register(0), bi_register(61), + BI_REGISTER_FORMAT_F16, BI_VECSIZE_V4), + 14, 0x005f80043300003d); +} + TEST_F(ValhallPacking, LeaBufImm) { CASE(bi_lea_buf_imm_to(b, bi_register(4), bi_discard(bi_register(59))), diff --git a/src/panfrost/compiler/bifrost/valhall/va_gather_hsr_info.c b/src/panfrost/compiler/bifrost/valhall/va_gather_hsr_info.c index 6fc81ebbb12..2d5ca159bd3 100644 --- a/src/panfrost/compiler/bifrost/valhall/va_gather_hsr_info.c +++ b/src/panfrost/compiler/bifrost/valhall/va_gather_hsr_info.c @@ -77,6 +77,8 @@ walk_bir_shader(bi_context *ctx, struct pan_shader_info *info) if (instr->sample == BI_SAMPLE_CENTROID) info->fs.hsr.centroid_interpolation = true; FALLTHROUGH; + case BI_OPCODE_LD_VAR_BUF_FLAT: + case BI_OPCODE_LD_VAR_BUF_FLAT_IMM: case BI_OPCODE_LD_VAR_FLAT: case BI_OPCODE_LD_VAR_FLAT_IMM: if (!found_atest) diff --git a/src/panfrost/compiler/bifrost/valhall/va_pack.c b/src/panfrost/compiler/bifrost/valhall/va_pack.c index 9665cc1cfd5..fc7ac40ccb8 100644 --- a/src/panfrost/compiler/bifrost/valhall/va_pack.c +++ b/src/panfrost/compiler/bifrost/valhall/va_pack.c @@ -566,6 +566,10 @@ va_pack_alu(const bi_instr *I, unsigned arch) hex |= ((uint64_t)I->sample) << 38; break; + case BI_OPCODE_LD_VAR_BUF_FLAT_IMM: + hex |= ((uint64_t)I->index) << 8; + break; + case BI_OPCODE_LD_ATTR_IMM: hex |= ((uint64_t)I->table) << 16; hex |= ((uint64_t)I->attribute_index) << 20;