diff --git a/src/intel/compiler/brw_fs.h b/src/intel/compiler/brw_fs.h index 0a49a6feb7d..6105bab2a86 100644 --- a/src/intel/compiler/brw_fs.h +++ b/src/intel/compiler/brw_fs.h @@ -320,18 +320,6 @@ public: void set_tcs_invocation_id(); void emit_nir_code(); - fs_reg get_nir_image_intrinsic_image(const brw::fs_builder &bld, - nir_intrinsic_instr *instr); - fs_reg get_nir_buffer_intrinsic_index(const brw::fs_builder &bld, - nir_intrinsic_instr *instr); - bool get_nir_src_bindless(const nir_src &src); - unsigned get_nir_src_block(const nir_src &src); - fs_reg get_nir_src(const nir_src &src); - fs_reg get_nir_src_imm(const nir_src &src); - fs_reg get_nir_def(const nir_def &def); - nir_component_mask_t get_nir_write_mask(const nir_def &def); - fs_reg get_resource_nir_src(const nir_src &src); - fs_reg get_indirect_offset(nir_intrinsic_instr *instr); void emit_alpha_test(); fs_inst *emit_single_fb_write(const brw::fs_builder &bld, diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index ef5684dae9a..ce7d4a919a5 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -34,6 +34,10 @@ using namespace brw; +static fs_reg get_nir_src(fs_visitor *s, const nir_src &src); +static fs_reg get_nir_def(fs_visitor *s, const nir_def &def); +static nir_component_mask_t get_nir_write_mask(const nir_def &def); + static void fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr); static fs_reg emit_samplepos_setup(fs_visitor *s); static fs_reg emit_sampleid_setup(fs_visitor *s); @@ -377,11 +381,11 @@ fs_nir_emit_if(fs_visitor *s, nir_if *if_stmt) nir_alu_instr *cond = nir_src_as_alu_instr(if_stmt->condition); if (cond != NULL && cond->op == nir_op_inot) { invert = true; - cond_reg = s->get_nir_src(cond->src[0].src); + cond_reg = get_nir_src(s, cond->src[0].src); cond_reg = offset(cond_reg, bld, cond->src[0].swizzle[0]); } else { invert = false; - cond_reg = s->get_nir_src(if_stmt->condition); + cond_reg = get_nir_src(s, if_stmt->condition); } /* first, put the condition into f0 */ @@ -461,7 +465,7 @@ optimize_extract_to_float(fs_visitor *s, nir_alu_instr *instr, src0->op == nir_op_extract_u16 || src0->op == nir_op_extract_i16 ? 2 : 1, src0->op == nir_op_extract_i16 || src0->op == nir_op_extract_i8); - fs_reg op0 = s->get_nir_src(src0->src[0].src); + fs_reg op0 = get_nir_src(s, src0->src[0].src); op0.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[src0->op].input_types[0] | nir_src_bit_size(src0->src[0].src))); @@ -591,14 +595,14 @@ prepare_alu_destination_and_sources(const fs_builder &bld, const intel_device_info *devinfo = s->devinfo; fs_reg result = - need_dest ? s->get_nir_def(instr->def) : bld.null_reg_ud(); + need_dest ? get_nir_def(s, instr->def) : bld.null_reg_ud(); result.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[instr->op].output_type | instr->def.bit_size)); for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { - op[i] = s->get_nir_src(instr->src[i].src); + op[i] = get_nir_src(s, instr->src[i].src); op[i].type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[instr->op].input_types[i] | nir_src_bit_size(instr->src[i].src))); @@ -630,7 +634,7 @@ prepare_alu_destination_and_sources(const fs_builder &bld, /* Since NIR is doing the scalarizing for us, we should only ever see * vectorized operations with a single channel. */ - nir_component_mask_t write_mask = s->get_nir_write_mask(instr->def); + nir_component_mask_t write_mask = get_nir_write_mask(instr->def); assert(util_bitcount(write_mask) == 1); channel = ffs(write_mask) - 1; @@ -746,7 +750,7 @@ emit_fsign(const fs_builder &bld, const nir_alu_instr *instr, if (fsign_src != 0) op[1] = op[0]; - op[0] = s->get_nir_src(fsign_instr->src[0].src); + op[0] = get_nir_src(s, fsign_instr->src[0].src); const nir_alu_type t = (nir_alu_type)(nir_op_infos[instr->op].input_types[0] | @@ -759,7 +763,7 @@ emit_fsign(const fs_builder &bld, const nir_alu_instr *instr, /* Since NIR is doing the scalarizing for us, we should only ever see * vectorized operations with a single channel. */ - nir_component_mask_t write_mask = s->get_nir_write_mask(instr->def); + nir_component_mask_t write_mask = get_nir_write_mask(instr->def); assert(util_bitcount(write_mask) == 1); channel = ffs(write_mask) - 1; } @@ -966,7 +970,7 @@ fs_nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr, } } - nir_component_mask_t write_mask = s->get_nir_write_mask(instr->def); + nir_component_mask_t write_mask = get_nir_write_mask(instr->def); unsigned last_bit = util_last_bit(write_mask); for (unsigned i = 0; i < last_bit; i++) { @@ -1941,18 +1945,10 @@ fs_nir_emit_load_const(const fs_builder &bld, s->nir_ssa_values[instr->def.index] = reg; } -bool -fs_visitor::get_nir_src_bindless(const nir_src &src) +static bool +get_nir_src_bindless(fs_visitor *s, const nir_src &src) { - return nir_ssa_bind_infos[src.ssa->index].bindless; -} - -unsigned -fs_visitor::get_nir_src_block(const nir_src &src) -{ - return nir_ssa_bind_infos[src.ssa->index].valid ? - nir_ssa_bind_infos[src.ssa->index].block : - UINT32_MAX; + return s->nir_ssa_bind_infos[src.ssa->index].bindless; } static bool @@ -1962,17 +1958,19 @@ is_resource_src(nir_src src) nir_instr_as_intrinsic(src.ssa->parent_instr)->intrinsic == nir_intrinsic_resource_intel; } -fs_reg -fs_visitor::get_resource_nir_src(const nir_src &src) +static fs_reg +get_resource_nir_src(fs_visitor *s, const nir_src &src) { if (!is_resource_src(src)) return fs_reg(); - return nir_resource_values[src.ssa->index]; + return s->nir_resource_values[src.ssa->index]; } -fs_reg -fs_visitor::get_nir_src(const nir_src &src) +static fs_reg +get_nir_src(fs_visitor *s, const nir_src &src) { + const intel_device_info *devinfo = s->devinfo; + nir_intrinsic_instr *load_reg = nir_load_reg_for_def(src.ssa); fs_reg reg; @@ -1981,16 +1979,16 @@ fs_visitor::get_nir_src(const nir_src &src) const brw_reg_type reg_type = brw_reg_type_from_bit_size(src.ssa->bit_size, BRW_REGISTER_TYPE_D); - reg = bld.vgrf(reg_type, src.ssa->num_components); + reg = s->bld.vgrf(reg_type, src.ssa->num_components); } else { - reg = nir_ssa_values[src.ssa->index]; + reg = s->nir_ssa_values[src.ssa->index]; } } else { nir_intrinsic_instr *decl_reg = nir_reg_get_decl(load_reg->src[0].ssa); /* We don't handle indirects on locals */ assert(nir_intrinsic_base(load_reg) == 0); assert(load_reg->intrinsic != nir_intrinsic_load_reg_indirect); - reg = nir_ssa_values[decl_reg->def.index]; + reg = s->nir_ssa_values[decl_reg->def.index]; } if (nir_src_bit_size(src) == 64 && devinfo->ver == 7) { @@ -2017,17 +2015,19 @@ fs_visitor::get_nir_src(const nir_src &src) * enough restrictions in 64-bit immediates that you can't take the return * value and treat it the same as the result of get_nir_src(). */ -fs_reg -fs_visitor::get_nir_src_imm(const nir_src &src) +static fs_reg +get_nir_src_imm(fs_visitor *s, const nir_src &src) { assert(nir_src_bit_size(src) == 32); return nir_src_is_const(src) ? - fs_reg(brw_imm_d(nir_src_as_int(src))) : get_nir_src(src); + fs_reg(brw_imm_d(nir_src_as_int(src))) : get_nir_src(s, src); } -fs_reg -fs_visitor::get_nir_def(const nir_def &def) +static fs_reg +get_nir_def(fs_visitor *s, const nir_def &def) { + const fs_builder &bld = s->bld; + nir_intrinsic_instr *store_reg = nir_store_reg_for_def(&def); if (!store_reg) { const brw_reg_type reg_type = @@ -2035,22 +2035,22 @@ fs_visitor::get_nir_def(const nir_def &def) def.bit_size == 8 ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_F); - nir_ssa_values[def.index] = + s->nir_ssa_values[def.index] = bld.vgrf(reg_type, def.num_components); - bld.UNDEF(nir_ssa_values[def.index]); - return nir_ssa_values[def.index]; + bld.UNDEF(s->nir_ssa_values[def.index]); + return s->nir_ssa_values[def.index]; } else { nir_intrinsic_instr *decl_reg = nir_reg_get_decl(store_reg->src[1].ssa); /* We don't handle indirects on locals */ assert(nir_intrinsic_base(store_reg) == 0); assert(store_reg->intrinsic != nir_intrinsic_store_reg_indirect); - return nir_ssa_values[decl_reg->def.index]; + return s->nir_ssa_values[decl_reg->def.index]; } } -nir_component_mask_t -fs_visitor::get_nir_write_mask(const nir_def &def) +static nir_component_mask_t +get_nir_write_mask(const nir_def &def) { nir_intrinsic_instr *store_reg = nir_store_reg_for_def(&def); if (!store_reg) { @@ -2132,7 +2132,7 @@ emit_gs_end_primitive(fs_visitor *s, const nir_src &vertex_count_nir_src) /* Cut bits use one bit per vertex. */ assert(s->gs_compile->control_data_bits_per_vertex == 1); - fs_reg vertex_count = s->get_nir_src(vertex_count_nir_src); + fs_reg vertex_count = get_nir_src(s, vertex_count_nir_src); vertex_count.type = BRW_REGISTER_TYPE_UD; /* Cut bit n should be set to 1 if EndPrimitive() was called after emitting @@ -2323,7 +2323,7 @@ emit_gs_vertex(fs_visitor *s, const nir_src &vertex_count_nir_src, struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(s->prog_data); - fs_reg vertex_count = s->get_nir_src(vertex_count_nir_src); + fs_reg vertex_count = get_nir_src(s, vertex_count_nir_src); vertex_count.type = BRW_REGISTER_TYPE_UD; /* Haswell and later hardware ignores the "Render Stream Select" bits @@ -2471,7 +2471,7 @@ emit_gs_input_load(fs_visitor *s, const fs_reg &dst, bld.SHL(channel_offsets, sequence, brw_imm_ud(2u)); /* Convert vertex_index to bytes (multiply by 32) */ bld.SHL(vertex_offset_bytes, - retype(s->get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), + retype(get_nir_src(s, vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(5u)); bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets); @@ -2499,7 +2499,7 @@ emit_gs_input_load(fs_visitor *s, const fs_reg &dst, /* Convert vertex_index to bytes (multiply by 4) */ bld.SHL(icp_offset_bytes, - retype(s->get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), + retype(get_nir_src(s, vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(2u)); /* Use first_icp_handle as the base offset. There is one DWord @@ -2514,7 +2514,7 @@ emit_gs_input_load(fs_visitor *s, const fs_reg &dst, } fs_inst *inst; - fs_reg indirect_offset = s->get_nir_src(offset_src); + fs_reg indirect_offset = get_nir_src(s, offset_src); if (nir_src_is_const(offset_src)) { fs_reg srcs[URB_LOGICAL_NUM_SRCS]; @@ -2567,8 +2567,8 @@ emit_gs_input_load(fs_visitor *s, const fs_reg &dst, } } -fs_reg -fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr) +static fs_reg +get_indirect_offset(fs_visitor *s, nir_intrinsic_instr *instr) { nir_src *offset_src = nir_get_io_offset_src(instr); @@ -2581,7 +2581,7 @@ fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr) return fs_reg(); } - return get_nir_src(*offset_src); + return get_nir_src(s, *offset_src); } static void @@ -2593,7 +2593,7 @@ fs_nir_emit_vs_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_vertex_id: @@ -2660,7 +2660,7 @@ get_tcs_single_patch_icp_handle(const fs_builder &bld, /* Each ICP handle is a single DWord (4 bytes) */ fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.SHL(vertex_offset_bytes, - retype(s->get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), + retype(get_nir_src(s, vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(2u)); /* We might read up to 4 registers. */ @@ -2710,7 +2710,7 @@ get_tcs_multi_patch_icp_handle(const fs_builder &bld, /* Convert vertex_index to bytes (multiply by 32) */ assert(util_is_power_of_two_nonzero(grf_size_bytes)); /* for ffs() */ bld.SHL(vertex_offset_bytes, - retype(s->get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), + retype(get_nir_src(s, vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(ffs(grf_size_bytes) - 1)); bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets); @@ -2840,7 +2840,7 @@ fs_nir_emit_tcs_intrinsic(const fs_builder &bld, fs_reg dst; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dst = s->get_nir_def(instr->def); + dst = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: @@ -2865,7 +2865,7 @@ fs_nir_emit_tcs_intrinsic(const fs_builder &bld, case nir_intrinsic_load_per_vertex_input: { assert(instr->def.bit_size == 32); - fs_reg indirect_offset = s->get_indirect_offset(instr); + fs_reg indirect_offset = get_indirect_offset(s, instr); unsigned imm_offset = nir_intrinsic_base(instr); fs_inst *inst; @@ -2940,7 +2940,7 @@ fs_nir_emit_tcs_intrinsic(const fs_builder &bld, case nir_intrinsic_load_output: case nir_intrinsic_load_per_vertex_output: { assert(instr->def.bit_size == 32); - fs_reg indirect_offset = s->get_indirect_offset(instr); + fs_reg indirect_offset = get_indirect_offset(s, instr); unsigned imm_offset = nir_intrinsic_base(instr); unsigned first_component = nir_intrinsic_component(instr); @@ -3004,8 +3004,8 @@ fs_nir_emit_tcs_intrinsic(const fs_builder &bld, case nir_intrinsic_store_output: case nir_intrinsic_store_per_vertex_output: { assert(nir_src_bit_size(instr->src[0]) == 32); - fs_reg value = s->get_nir_src(instr->src[0]); - fs_reg indirect_offset = s->get_indirect_offset(instr); + fs_reg value = get_nir_src(s, instr->src[0]); + fs_reg indirect_offset = get_indirect_offset(s, instr); unsigned imm_offset = nir_intrinsic_base(instr); unsigned mask = nir_intrinsic_write_mask(instr); @@ -3070,7 +3070,7 @@ fs_nir_emit_tes_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: @@ -3085,7 +3085,7 @@ fs_nir_emit_tes_intrinsic(const fs_builder &bld, case nir_intrinsic_load_input: case nir_intrinsic_load_per_vertex_input: { assert(instr->def.bit_size == 32); - fs_reg indirect_offset = s->get_indirect_offset(instr); + fs_reg indirect_offset = get_indirect_offset(s, instr); unsigned imm_offset = nir_intrinsic_base(instr); unsigned first_component = nir_intrinsic_component(instr); @@ -3178,7 +3178,7 @@ fs_nir_emit_gs_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: @@ -3205,7 +3205,7 @@ fs_nir_emit_gs_intrinsic(const fs_builder &bld, break; case nir_intrinsic_set_vertex_and_primitive_count: - bld.MOV(s->final_gs_vertex_count, s->get_nir_src(instr->src[0])); + bld.MOV(s->final_gs_vertex_count, get_nir_src(s, instr->src[0])); break; case nir_intrinsic_load_invocation_id: { @@ -3821,7 +3821,7 @@ fs_nir_emit_fs_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_front_face: @@ -3861,7 +3861,7 @@ fs_nir_emit_fs_intrinsic(const fs_builder &bld, } case nir_intrinsic_store_output: { - const fs_reg src = s->get_nir_src(instr->src[0]); + const fs_reg src = get_nir_src(s, instr->src[0]); const unsigned store_offset = nir_src_as_uint(instr->src[1]); const unsigned location = nir_intrinsic_base(instr) + SET_FIELD(store_offset, BRW_NIR_FRAG_OUTPUT_LOCATION); @@ -3954,7 +3954,7 @@ fs_nir_emit_fs_intrinsic(const fs_builder &bld, } if (cmp == NULL) { - cmp = bld.CMP(bld.null_reg_f(), s->get_nir_src(instr->src[0]), + cmp = bld.CMP(bld.null_reg_f(), get_nir_src(s, instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_Z); } } else { @@ -4088,7 +4088,7 @@ fs_nir_emit_fs_intrinsic(const fs_builder &bld, if (nir_src_is_const(instr->src[0])) { msg_data = brw_imm_ud(nir_src_as_uint(instr->src[0]) << 4); } else { - const fs_reg sample_src = retype(s->get_nir_src(instr->src[0]), + const fs_reg sample_src = retype(get_nir_src(s, instr->src[0]), BRW_REGISTER_TYPE_UD); const fs_reg sample_id = bld.emit_uniformize(sample_src); msg_data = component(bld.group(8, 0).vgrf(BRW_REGISTER_TYPE_UD), 0); @@ -4135,7 +4135,7 @@ fs_nir_emit_fs_intrinsic(const fs_builder &bld, fs_reg(), /* flag_reg */ interpolation); } else { - fs_reg src = retype(s->get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_D); + fs_reg src = retype(get_nir_src(s, instr->src[0]), BRW_REGISTER_TYPE_D); const enum opcode opcode = FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET; emit_pixel_interpolater_send(bld, opcode, @@ -4165,7 +4165,7 @@ fs_nir_emit_fs_intrinsic(const fs_builder &bld, if (bary_intrin == nir_intrinsic_load_barycentric_at_offset || bary_intrin == nir_intrinsic_load_barycentric_at_sample) { /* Use the result of the PI message. */ - dst_xy = retype(s->get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F); + dst_xy = retype(get_nir_src(s, instr->src[0]), BRW_REGISTER_TYPE_F); } else { /* Use the delta_xy values computed from the payload */ enum brw_barycentric_mode bary = brw_barycentric_mode(bary_intrinsic); @@ -4208,7 +4208,7 @@ fs_nir_emit_cs_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_barrier: @@ -4284,7 +4284,7 @@ fs_nir_emit_cs_intrinsic(const fs_builder &bld, fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX7_BTI_SLM); - fs_reg addr = s->get_nir_src(instr->src[0]); + fs_reg addr = get_nir_src(s, instr->src[0]); int base = nir_intrinsic_base(instr); if (base) { fs_reg addr_off = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); @@ -4330,7 +4330,7 @@ fs_nir_emit_cs_intrinsic(const fs_builder &bld, fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX7_BTI_SLM); - fs_reg addr = s->get_nir_src(instr->src[1]); + fs_reg addr = get_nir_src(s, instr->src[1]); int base = nir_intrinsic_base(instr); if (base) { fs_reg addr_off = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); @@ -4346,7 +4346,7 @@ fs_nir_emit_cs_intrinsic(const fs_builder &bld, */ srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); - fs_reg data = s->get_nir_src(instr->src[0]); + fs_reg data = get_nir_src(s, instr->src[0]); data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); assert(bit_size <= 32); @@ -4423,7 +4423,7 @@ fs_nir_emit_bs_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_btd_global_arg_addr_intel: @@ -4695,26 +4695,30 @@ try_rebuild_resource(const brw::fs_builder &bld, nir_def *resource_def) return component(s->nir_resource_insts[resource_def->index]->dst, 0); } -fs_reg -fs_visitor::get_nir_image_intrinsic_image(const brw::fs_builder &bld, - nir_intrinsic_instr *instr) +static fs_reg +get_nir_image_intrinsic_image(const brw::fs_builder &bld, + nir_intrinsic_instr *instr) { + fs_visitor *s = (fs_visitor *)bld.shader; + if (is_resource_src(instr->src[0])) { - fs_reg surf_index = get_resource_nir_src(instr->src[0]); + fs_reg surf_index = get_resource_nir_src(s, instr->src[0]); if (surf_index.file != BAD_FILE) return surf_index; } - fs_reg image = retype(get_nir_src_imm(instr->src[0]), BRW_REGISTER_TYPE_UD); + fs_reg image = retype(get_nir_src_imm(s, instr->src[0]), BRW_REGISTER_TYPE_UD); fs_reg surf_index = image; return bld.emit_uniformize(surf_index); } -fs_reg -fs_visitor::get_nir_buffer_intrinsic_index(const brw::fs_builder &bld, - nir_intrinsic_instr *instr) +static fs_reg +get_nir_buffer_intrinsic_index(const brw::fs_builder &bld, + nir_intrinsic_instr *instr) { + fs_visitor *s = (fs_visitor *)bld.shader; + /* SSBO stores are weird in that their index is in src[1] */ const bool is_store = instr->intrinsic == nir_intrinsic_store_ssbo || @@ -4724,11 +4728,11 @@ fs_visitor::get_nir_buffer_intrinsic_index(const brw::fs_builder &bld, if (nir_src_is_const(src)) { return brw_imm_ud(nir_src_as_uint(src)); } else if (is_resource_src(src)) { - fs_reg surf_index = get_resource_nir_src(src); + fs_reg surf_index = get_resource_nir_src(s, src); if (surf_index.file != BAD_FILE) return surf_index; } - return bld.emit_uniformize(get_nir_src(src)); + return bld.emit_uniformize(get_nir_src(s, src)); } /** @@ -5435,7 +5439,7 @@ emit_task_mesh_store(const fs_builder &bld, nir_intrinsic_instr *instr, { fs_visitor *s = (fs_visitor *)bld.shader; - fs_reg src = s->get_nir_src(instr->src[0]); + fs_reg src = get_nir_src(s, instr->src[0]); nir_src *offset_nir_src = nir_get_io_offset_src(instr); if (nir_src_is_const(*offset_nir_src)) { @@ -5445,7 +5449,7 @@ emit_task_mesh_store(const fs_builder &bld, nir_intrinsic_instr *instr, emit_urb_direct_writes(bld, instr, src, urb_handle); } else { if (bld.shader->devinfo->ver >= 20) { - emit_urb_indirect_writes_xe2(bld, instr, src, s->get_nir_src(*offset_nir_src), urb_handle); + emit_urb_indirect_writes_xe2(bld, instr, src, get_nir_src(s, *offset_nir_src), urb_handle); return; } bool use_mod = false; @@ -5461,9 +5465,9 @@ emit_task_mesh_store(const fs_builder &bld, nir_intrinsic_instr *instr, } if (use_mod) { - emit_urb_indirect_writes_mod(bld, instr, src, s->get_nir_src(*offset_nir_src), urb_handle, mod); + emit_urb_indirect_writes_mod(bld, instr, src, get_nir_src(s, *offset_nir_src), urb_handle, mod); } else { - emit_urb_indirect_writes(bld, instr, src, s->get_nir_src(*offset_nir_src), urb_handle); + emit_urb_indirect_writes(bld, instr, src, get_nir_src(s, *offset_nir_src), urb_handle); } } } @@ -5474,7 +5478,7 @@ emit_task_mesh_load(const fs_builder &bld, nir_intrinsic_instr *instr, { fs_visitor *s = (fs_visitor *)bld.shader; - fs_reg dest = s->get_nir_def(instr->def); + fs_reg dest = get_nir_def(s, instr->def); nir_src *offset_nir_src = nir_get_io_offset_src(instr); /* TODO(mesh): for per_vertex and per_primitive, if we could keep around @@ -5489,9 +5493,9 @@ emit_task_mesh_load(const fs_builder &bld, nir_intrinsic_instr *instr, emit_urb_direct_reads(bld, instr, dest, urb_handle); } else { if (bld.shader->devinfo->ver >= 20) - emit_urb_indirect_reads_xe2(bld, instr, dest, s->get_nir_src(*offset_nir_src), urb_handle); + emit_urb_indirect_reads_xe2(bld, instr, dest, get_nir_src(s, *offset_nir_src), urb_handle); else - emit_urb_indirect_reads(bld, instr, dest, s->get_nir_src(*offset_nir_src), urb_handle); + emit_urb_indirect_reads(bld, instr, dest, get_nir_src(s, *offset_nir_src), urb_handle); } } @@ -5506,7 +5510,7 @@ fs_nir_emit_task_mesh_intrinsic(const fs_builder &bld, fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_load_mesh_inline_data_intel: { @@ -5629,7 +5633,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) - dest = s->get_nir_def(instr->def); + dest = get_nir_def(s, instr->def); switch (instr->intrinsic) { case nir_intrinsic_resource_intel: @@ -5679,17 +5683,17 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_image_atomic: case nir_intrinsic_image_atomic_swap: srcs[SURFACE_LOGICAL_SRC_SURFACE] = - s->get_nir_image_intrinsic_image(bld, instr); + get_nir_image_intrinsic_image(bld, instr); break; default: /* Bindless */ srcs[SURFACE_LOGICAL_SRC_SURFACE_HANDLE] = - s->get_nir_image_intrinsic_image(bld, instr); + get_nir_image_intrinsic_image(bld, instr); break; } - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->get_nir_src(instr->src[1]); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(s, instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(nir_image_intrinsic_coord_components(instr)); @@ -5705,7 +5709,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } else if (instr->intrinsic == nir_intrinsic_image_store || instr->intrinsic == nir_intrinsic_bindless_image_store) { srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); - srcs[SURFACE_LOGICAL_SRC_DATA] = s->get_nir_src(instr->src[3]); + srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(s, instr->src[3]); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); bld.emit(SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); @@ -5721,10 +5725,10 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) fs_reg data; if (num_srcs >= 4) - data = s->get_nir_src(instr->src[3]); + data = get_nir_src(s, instr->src[3]); if (num_srcs >= 5) { fs_reg tmp = bld.vgrf(data.type, 2); - fs_reg sources[2] = { data, s->get_nir_src(instr->src[4]) }; + fs_reg sources[2] = { data, get_nir_src(s, instr->src[4]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } @@ -5747,7 +5751,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) * Incidentally, this means that we can handle bindless with exactly the * same code. */ - fs_reg image = retype(s->get_nir_src_imm(instr->src[0]), + fs_reg image = retype(get_nir_src_imm(s, instr->src[0]), BRW_REGISTER_TYPE_UD); image = bld.emit_uniformize(image); @@ -5783,8 +5787,8 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_image_load_raw_intel: { fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = - s->get_nir_image_intrinsic_image(bld, instr); - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->get_nir_src(instr->src[1]); + get_nir_image_intrinsic_image(bld, instr); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(s, instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); @@ -5799,9 +5803,9 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_image_store_raw_intel: { fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = - s->get_nir_image_intrinsic_image(bld, instr); - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->get_nir_src(instr->src[1]); - srcs[SURFACE_LOGICAL_SRC_DATA] = s->get_nir_src(instr->src[2]); + get_nir_image_intrinsic_image(bld, instr); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(s, instr->src[1]); + srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(s, instr->src[2]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); @@ -6117,7 +6121,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) bld.MOV(offset(dest, bld, j), offset(src, bld, j)); } } else { - fs_reg indirect = retype(s->get_nir_src(instr->src[0]), + fs_reg indirect = retype(get_nir_src(s, instr->src[0]), BRW_REGISTER_TYPE_UD); /* We need to pass a size to the MOV_INDIRECT but we don't want it to @@ -6164,15 +6168,15 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_load_ubo_uniform_block_intel: { fs_reg surface, surface_handle; - if (s->get_nir_src_bindless(instr->src[0])) - surface_handle = s->get_nir_buffer_intrinsic_index(bld, instr); + if (get_nir_src_bindless(s, instr->src[0])) + surface_handle = get_nir_buffer_intrinsic_index(bld, instr); else - surface = s->get_nir_buffer_intrinsic_index(bld, instr); + surface = get_nir_buffer_intrinsic_index(bld, instr); if (!nir_src_is_const(instr->src[1])) { if (instr->intrinsic == nir_intrinsic_load_ubo) { /* load_ubo with non-uniform offset */ - fs_reg base_offset = retype(s->get_nir_src(instr->src[1]), + fs_reg base_offset = retype(get_nir_src(s, instr->src[1]), BRW_REGISTER_TYPE_UD); for (int i = 0; i < instr->num_components; i++) @@ -6200,7 +6204,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) srcs[SURFACE_LOGICAL_SRC_ADDRESS] = component(addr, 0); } else { srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - bld.emit_uniformize(s->get_nir_src(load_offset)); + bld.emit_uniformize(get_nir_src(s, load_offset)); } const unsigned total_dwords = ALIGN(instr->num_components, REG_SIZE / 4); @@ -6316,7 +6320,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) assert(instr->def.bit_size <= 32); assert(nir_intrinsic_align(instr) > 0); fs_reg srcs[A64_LOGICAL_NUM_SRCS]; - srcs[A64_LOGICAL_ADDRESS] = s->get_nir_src(instr->src[0]); + srcs[A64_LOGICAL_ADDRESS] = get_nir_src(s, instr->src[0]); srcs[A64_LOGICAL_SRC] = fs_reg(); /* No source data */ srcs[A64_LOGICAL_ENABLE_HELPERS] = brw_imm_ud(nir_intrinsic_access(instr) & ACCESS_INCLUDE_HELPERS); @@ -6355,7 +6359,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) assert(nir_intrinsic_align(instr) > 0); fs_reg srcs[A64_LOGICAL_NUM_SRCS]; - srcs[A64_LOGICAL_ADDRESS] = s->get_nir_src(instr->src[1]); + srcs[A64_LOGICAL_ADDRESS] = get_nir_src(s, instr->src[1]); srcs[A64_LOGICAL_ENABLE_HELPERS] = brw_imm_ud(nir_intrinsic_access(instr) & ACCESS_INCLUDE_HELPERS); @@ -6363,7 +6367,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) nir_intrinsic_align(instr) >= 4) { assert(nir_src_num_components(instr->src[0]) <= 4); - srcs[A64_LOGICAL_SRC] = s->get_nir_src(instr->src[0]); /* Data */ + srcs[A64_LOGICAL_SRC] = get_nir_src(s, instr->src[0]); /* Data */ srcs[A64_LOGICAL_ARG] = brw_imm_ud(instr->num_components); bld.emit(SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL, fs_reg(), @@ -6374,7 +6378,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) brw_reg_type data_type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD); - bld.MOV(tmp, retype(s->get_nir_src(instr->src[0]), data_type)); + bld.MOV(tmp, retype(get_nir_src(s, instr->src[0]), data_type)); srcs[A64_LOGICAL_SRC] = tmp; srcs[A64_LOGICAL_ARG] = brw_imm_ud(bit_size); @@ -6405,7 +6409,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) load_val = brw_imm_ud(0); } else { /* The uniform process may stomp the flag so do this first */ - fs_reg addr = bld.emit_uniformize(s->get_nir_src(instr->src[0])); + fs_reg addr = bld.emit_uniformize(get_nir_src(s, instr->src[0])); load_val = ubld.vgrf(BRW_REGISTER_TYPE_UD); @@ -6414,7 +6418,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) */ if (!is_pred_const) { /* Load the predicate */ - fs_reg pred = bld.emit_uniformize(s->get_nir_src(instr->src[1])); + fs_reg pred = bld.emit_uniformize(get_nir_src(s, instr->src[1])); fs_inst *mov = ubld.MOV(bld.null_reg_d(), pred); mov->conditional_mod = BRW_CONDITIONAL_NZ; @@ -6463,7 +6467,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) const fs_reg packed_consts = ubld1.vgrf(BRW_REGISTER_TYPE_UD, total_dwords); - fs_reg address = bld.emit_uniformize(s->get_nir_src(instr->src[0])); + fs_reg address = bld.emit_uniformize(get_nir_src(s, instr->src[0])); while (loaded_dwords < total_dwords) { const unsigned block = @@ -6498,11 +6502,11 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) const unsigned bit_size = instr->def.bit_size; fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; - srcs[s->get_nir_src_bindless(instr->src[0]) ? + srcs[get_nir_src_bindless(s, instr->src[0]) ? SURFACE_LOGICAL_SRC_SURFACE_HANDLE : SURFACE_LOGICAL_SRC_SURFACE] = - s->get_nir_buffer_intrinsic_index(bld, instr); - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->get_nir_src(instr->src[1]); + get_nir_buffer_intrinsic_index(bld, instr); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(s, instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); @@ -6537,15 +6541,15 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) const unsigned bit_size = nir_src_bit_size(instr->src[0]); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; - srcs[s->get_nir_src_bindless(instr->src[1]) ? + srcs[get_nir_src_bindless(s, instr->src[1]) ? SURFACE_LOGICAL_SRC_SURFACE_HANDLE : SURFACE_LOGICAL_SRC_SURFACE] = - s->get_nir_buffer_intrinsic_index(bld, instr); - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->get_nir_src(instr->src[2]); + get_nir_buffer_intrinsic_index(bld, instr); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(s, instr->src[2]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); - fs_reg data = s->get_nir_src(instr->src[0]); + fs_reg data = get_nir_src(s, instr->src[0]); data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); assert(bit_size <= 32); @@ -6579,10 +6583,10 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) const bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo_uniform_block_intel; if (is_ssbo) { - srcs[s->get_nir_src_bindless(instr->src[0]) ? + srcs[get_nir_src_bindless(s, instr->src[0]) ? SURFACE_LOGICAL_SRC_SURFACE_HANDLE : SURFACE_LOGICAL_SRC_SURFACE] = - s->get_nir_buffer_intrinsic_index(bld, instr); + get_nir_buffer_intrinsic_index(bld, instr); } else { srcs[SURFACE_LOGICAL_SRC_SURFACE] = fs_reg(brw_imm_ud(GFX7_BTI_SLM)); } @@ -6605,7 +6609,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) srcs[SURFACE_LOGICAL_SRC_ADDRESS] = component(addr, 0); } else { srcs[SURFACE_LOGICAL_SRC_ADDRESS] = - bld.emit_uniformize(s->get_nir_src(load_offset)); + bld.emit_uniformize(get_nir_src(s, load_offset)); } while (loaded_dwords < total_dwords) { @@ -6638,7 +6642,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_store_output: { assert(nir_src_bit_size(instr->src[0]) == 32); - fs_reg src = s->get_nir_src(instr->src[0]); + fs_reg src = get_nir_src(s, instr->src[0]); unsigned store_offset = nir_src_as_uint(instr->src[1]); unsigned num_components = instr->num_components; @@ -6656,8 +6660,8 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_ssbo_atomic: case nir_intrinsic_ssbo_atomic_swap: fs_nir_emit_surface_atomic(bld, instr, - s->get_nir_buffer_intrinsic_index(bld, instr), - s->get_nir_src_bindless(instr->src[0])); + get_nir_buffer_intrinsic_index(bld, instr), + get_nir_src_bindless(s, instr->src[0])); break; case nir_intrinsic_get_ssbo_size: { @@ -6679,10 +6683,10 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) ubld.MOV(src_payload, brw_imm_d(0)); fs_reg srcs[GET_BUFFER_SIZE_SRCS]; - srcs[s->get_nir_src_bindless(instr->src[0]) ? + srcs[get_nir_src_bindless(s, instr->src[0]) ? GET_BUFFER_SIZE_SRC_SURFACE_HANDLE : GET_BUFFER_SIZE_SRC_SURFACE] = - s->get_nir_buffer_intrinsic_index(bld, instr); + get_nir_buffer_intrinsic_index(bld, instr); srcs[GET_BUFFER_SIZE_SRC_LOD] = src_payload; fs_inst *inst = ubld.emit(SHADER_OPCODE_GET_BUFFER_SIZE, ret_payload, srcs, GET_BUFFER_SIZE_SRCS); @@ -6746,7 +6750,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); - const fs_reg nir_addr = s->get_nir_src(instr->src[0]); + const fs_reg nir_addr = get_nir_src(s, instr->src[0]); /* Make dest unsigned because that's what the temporary will be */ dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); @@ -6820,9 +6824,9 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) * they should not have different behaviour in the helper invocations. */ srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); - const fs_reg nir_addr = s->get_nir_src(instr->src[1]); + const fs_reg nir_addr = get_nir_src(s, instr->src[1]); - fs_reg data = s->get_nir_src(instr->src[0]); + fs_reg data = get_nir_src(s, instr->src[0]); data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); assert(nir_src_num_components(instr->src[0]) == 1); @@ -6898,7 +6902,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } else { ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0)); } - bld.CMP(bld.null_reg_d(), s->get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ); + bld.CMP(bld.null_reg_d(), get_nir_src(s, instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ); /* For some reason, the any/all predicates don't work properly with * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H @@ -6930,7 +6934,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } else { ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0xffff)); } - bld.CMP(bld.null_reg_d(), s->get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ); + bld.CMP(bld.null_reg_d(), get_nir_src(s, instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ); /* For some reason, the any/all predicates don't work properly with * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H @@ -6950,7 +6954,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_vote_feq: case nir_intrinsic_vote_ieq: { - fs_reg value = s->get_nir_src(instr->src[0]); + fs_reg value = get_nir_src(s, instr->src[0]); if (instr->intrinsic == nir_intrinsic_vote_feq) { const unsigned bit_size = nir_src_bit_size(instr->src[0]); value.type = bit_size == 8 ? BRW_REGISTER_TYPE_B : @@ -6991,7 +6995,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_ballot: { - const fs_reg value = retype(s->get_nir_src(instr->src[0]), + const fs_reg value = retype(get_nir_src(s, instr->src[0]), BRW_REGISTER_TYPE_UD); struct brw_reg flag = brw_flag_reg(0, 0); /* FIXME: For SIMD32 programs, this causes us to stomp on f0.1 as well @@ -7016,8 +7020,8 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_read_invocation: { - const fs_reg value = s->get_nir_src(instr->src[0]); - const fs_reg invocation = s->get_nir_src(instr->src[1]); + const fs_reg value = get_nir_src(s, instr->src[0]); + const fs_reg invocation = get_nir_src(s, instr->src[1]); fs_reg tmp = bld.vgrf(value.type); @@ -7041,14 +7045,14 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_read_first_invocation: { - const fs_reg value = s->get_nir_src(instr->src[0]); + const fs_reg value = get_nir_src(s, instr->src[0]); bld.MOV(retype(dest, value.type), bld.emit_uniformize(value)); break; } case nir_intrinsic_shuffle: { - const fs_reg value = s->get_nir_src(instr->src[0]); - const fs_reg index = s->get_nir_src(instr->src[1]); + const fs_reg value = get_nir_src(s, instr->src[0]); + const fs_reg index = get_nir_src(s, instr->src[1]); bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, index); break; @@ -7071,7 +7075,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_quad_broadcast: { - const fs_reg value = s->get_nir_src(instr->src[0]); + const fs_reg value = get_nir_src(s, instr->src[0]); const unsigned index = nir_src_as_uint(instr->src[1]); bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, retype(dest, value.type), @@ -7080,7 +7084,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_quad_swap_horizontal: { - const fs_reg value = s->get_nir_src(instr->src[0]); + const fs_reg value = get_nir_src(s, instr->src[0]); const fs_reg tmp = bld.vgrf(value.type); if (devinfo->ver <= 7) { /* The hardware doesn't seem to support these crazy regions with @@ -7110,7 +7114,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_quad_swap_vertical: { - const fs_reg value = s->get_nir_src(instr->src[0]); + const fs_reg value = get_nir_src(s, instr->src[0]); if (nir_src_bit_size(instr->src[0]) == 32) { /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */ const fs_reg tmp = bld.vgrf(value.type); @@ -7131,7 +7135,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_quad_swap_diagonal: { - const fs_reg value = s->get_nir_src(instr->src[0]); + const fs_reg value = get_nir_src(s, instr->src[0]); if (nir_src_bit_size(instr->src[0]) == 32) { /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */ const fs_reg tmp = bld.vgrf(value.type); @@ -7152,7 +7156,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) } case nir_intrinsic_reduce: { - fs_reg src = s->get_nir_src(instr->src[0]); + fs_reg src = get_nir_src(s, instr->src[0]); nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr); unsigned cluster_size = nir_intrinsic_cluster_size(instr); if (cluster_size == 0 || cluster_size > s->dispatch_width) @@ -7201,7 +7205,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_inclusive_scan: case nir_intrinsic_exclusive_scan: { - fs_reg src = s->get_nir_src(instr->src[0]); + fs_reg src = get_nir_src(s, instr->src[0]); nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr); /* Figure out the source type */ @@ -7243,7 +7247,7 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_load_global_block_intel: { assert(instr->def.bit_size == 32); - fs_reg address = bld.emit_uniformize(s->get_nir_src(instr->src[0])); + fs_reg address = bld.emit_uniformize(get_nir_src(s, instr->src[0])); const fs_builder ubld1 = bld.exec_all().group(1, 0); const fs_builder ubld8 = bld.exec_all().group(8, 0); @@ -7279,8 +7283,8 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) case nir_intrinsic_store_global_block_intel: { assert(nir_src_bit_size(instr->src[0]) == 32); - fs_reg address = bld.emit_uniformize(s->get_nir_src(instr->src[1])); - fs_reg src = s->get_nir_src(instr->src[0]); + fs_reg address = bld.emit_uniformize(get_nir_src(s, instr->src[1])); + fs_reg src = get_nir_src(s, instr->src[0]); const fs_builder ubld1 = bld.exec_all().group(1, 0); const fs_builder ubld8 = bld.exec_all().group(8, 0); @@ -7319,11 +7323,11 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) const bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo_block_intel; - fs_reg address = bld.emit_uniformize(s->get_nir_src(instr->src[is_ssbo ? 1 : 0])); + fs_reg address = bld.emit_uniformize(get_nir_src(s, instr->src[is_ssbo ? 1 : 0])); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = is_ssbo ? - s->get_nir_buffer_intrinsic_index(bld, instr) : + get_nir_buffer_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM)); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = address; @@ -7361,12 +7365,12 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) const bool is_ssbo = instr->intrinsic == nir_intrinsic_store_ssbo_block_intel; - fs_reg address = bld.emit_uniformize(s->get_nir_src(instr->src[is_ssbo ? 2 : 1])); - fs_reg src = s->get_nir_src(instr->src[0]); + fs_reg address = bld.emit_uniformize(get_nir_src(s, instr->src[is_ssbo ? 2 : 1])); + fs_reg src = get_nir_src(s, instr->src[0]); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = is_ssbo ? - s->get_nir_buffer_intrinsic_index(bld, instr) : + get_nir_buffer_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM)); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = address; @@ -7492,8 +7496,8 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) emit_rt_lsc_fence(bld, LSC_FENCE_LOCAL, LSC_FLUSH_TYPE_NONE); bld.emit(SHADER_OPCODE_BTD_SPAWN_LOGICAL, bld.null_reg_ud(), - bld.emit_uniformize(s->get_nir_src(instr->src[0])), - s->get_nir_src(instr->src[1])); + bld.emit_uniformize(get_nir_src(s, instr->src[0])), + get_nir_src(s, instr->src[1])); break; case nir_intrinsic_btd_retire_intel: @@ -7521,10 +7525,10 @@ fs_nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) fs_reg srcs[RT_LOGICAL_NUM_SRCS]; - fs_reg globals = s->get_nir_src(instr->src[0]); + fs_reg globals = get_nir_src(s, instr->src[0]); srcs[RT_LOGICAL_SRC_GLOBALS] = bld.emit_uniformize(globals); - srcs[RT_LOGICAL_SRC_BVH_LEVEL] = s->get_nir_src(instr->src[1]); - srcs[RT_LOGICAL_SRC_TRACE_RAY_CONTROL] = s->get_nir_src(instr->src[2]); + srcs[RT_LOGICAL_SRC_BVH_LEVEL] = get_nir_src(s, instr->src[1]); + srcs[RT_LOGICAL_SRC_TRACE_RAY_CONTROL] = get_nir_src(s, instr->src[2]); srcs[RT_LOGICAL_SRC_SYNCHRONOUS] = brw_imm_ud(synchronous); bld.emit(RT_OPCODE_TRACE_RAY_LOGICAL, bld.null_reg_ud(), srcs, RT_LOGICAL_NUM_SRCS); @@ -7589,7 +7593,7 @@ fs_nir_emit_surface_atomic(const fs_builder &bld, (instr->def.bit_size == 16 && (devinfo->has_lsc || lsc_opcode_is_atomic_float(op)))); - fs_reg dest = s->get_nir_def(instr->def); + fs_reg dest = get_nir_def(s, instr->def); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[bindless ? @@ -7608,23 +7612,23 @@ fs_nir_emit_surface_atomic(const fs_builder &bld, } else { srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->vgrf(glsl_type::uint_type); bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS], - retype(s->get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD), + retype(get_nir_src(s, instr->src[0]), BRW_REGISTER_TYPE_UD), brw_imm_ud(nir_intrinsic_base(instr))); } } else { /* SSBOs */ - srcs[SURFACE_LOGICAL_SRC_ADDRESS] = s->get_nir_src(instr->src[1]); + srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(s, instr->src[1]); } fs_reg data; if (num_data >= 1) - data = expand_to_32bit(bld, s->get_nir_src(instr->src[shared ? 1 : 2])); + data = expand_to_32bit(bld, get_nir_src(s, instr->src[shared ? 1 : 2])); if (num_data >= 2) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, - expand_to_32bit(bld, s->get_nir_src(instr->src[shared ? 2 : 3])) + expand_to_32bit(bld, get_nir_src(s, instr->src[shared ? 2 : 3])) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; @@ -7663,19 +7667,19 @@ fs_nir_emit_global_atomic(const fs_builder &bld, enum lsc_opcode op = lsc_aop_for_nir_intrinsic(instr); int num_data = lsc_op_num_data_values(op); - fs_reg dest = s->get_nir_def(instr->def); + fs_reg dest = get_nir_def(s, instr->def); - fs_reg addr = s->get_nir_src(instr->src[0]); + fs_reg addr = get_nir_src(s, instr->src[0]); fs_reg data; if (num_data >= 1) - data = expand_to_32bit(bld, s->get_nir_src(instr->src[1])); + data = expand_to_32bit(bld, get_nir_src(s, instr->src[1])); if (num_data >= 2) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, - expand_to_32bit(bld, s->get_nir_src(instr->src[2])) + expand_to_32bit(bld, get_nir_src(s, instr->src[2])) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; @@ -7736,11 +7740,11 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) uint32_t header_bits = 0; for (unsigned i = 0; i < instr->num_srcs; i++) { nir_src nir_src = instr->src[i].src; - fs_reg src = s->get_nir_src(nir_src); + fs_reg src = get_nir_src(s, nir_src); switch (instr->src[i].src_type) { case nir_tex_src_bias: srcs[TEX_LOGICAL_SRC_LOD] = - retype(s->get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F); + retype(get_nir_src_imm(s, instr->src[i].src), BRW_REGISTER_TYPE_F); break; case nir_tex_src_comparator: srcs[TEX_LOGICAL_SRC_SHADOW_C] = retype(src, BRW_REGISTER_TYPE_F); @@ -7769,21 +7773,21 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) switch (instr->op) { case nir_texop_txs: srcs[TEX_LOGICAL_SRC_LOD] = - retype(s->get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_UD); + retype(get_nir_src_imm(s, instr->src[i].src), BRW_REGISTER_TYPE_UD); break; case nir_texop_txf: srcs[TEX_LOGICAL_SRC_LOD] = - retype(s->get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_D); + retype(get_nir_src_imm(s, instr->src[i].src), BRW_REGISTER_TYPE_D); break; default: srcs[TEX_LOGICAL_SRC_LOD] = - retype(s->get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F); + retype(get_nir_src_imm(s, instr->src[i].src), BRW_REGISTER_TYPE_F); break; } break; case nir_tex_src_min_lod: srcs[TEX_LOGICAL_SRC_MIN_LOD] = - retype(s->get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F); + retype(get_nir_src_imm(s, instr->src[i].src), BRW_REGISTER_TYPE_F); break; case nir_tex_src_ms_index: srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = retype(src, BRW_REGISTER_TYPE_UD); @@ -7812,7 +7816,7 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) assert(srcs[TEX_LOGICAL_SRC_SURFACE].file == BAD_FILE); /* Emit code to evaluate the actual indexing expression */ if (instr->texture_index == 0 && is_resource_src(nir_src)) - srcs[TEX_LOGICAL_SRC_SURFACE] = s->get_resource_nir_src(nir_src); + srcs[TEX_LOGICAL_SRC_SURFACE] = get_resource_nir_src(s, nir_src); if (srcs[TEX_LOGICAL_SRC_SURFACE].file == BAD_FILE) { fs_reg tmp = s->vgrf(glsl_type::uint_type); bld.ADD(tmp, src, brw_imm_ud(instr->texture_index)); @@ -7825,7 +7829,7 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) case nir_tex_src_sampler_offset: { /* Emit code to evaluate the actual indexing expression */ if (instr->sampler_index == 0 && is_resource_src(nir_src)) - srcs[TEX_LOGICAL_SRC_SAMPLER] = s->get_resource_nir_src(nir_src); + srcs[TEX_LOGICAL_SRC_SAMPLER] = get_resource_nir_src(s, nir_src); if (srcs[TEX_LOGICAL_SRC_SAMPLER].file == BAD_FILE) { fs_reg tmp = s->vgrf(glsl_type::uint_type); bld.ADD(tmp, src, brw_imm_ud(instr->sampler_index)); @@ -7838,7 +7842,7 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) assert(nir_tex_instr_src_index(instr, nir_tex_src_texture_offset) == -1); srcs[TEX_LOGICAL_SRC_SURFACE] = fs_reg(); if (is_resource_src(nir_src)) - srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = s->get_resource_nir_src(nir_src); + srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = get_resource_nir_src(s, nir_src); if (srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE].file == BAD_FILE) srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = bld.emit_uniformize(src); break; @@ -7847,7 +7851,7 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) assert(nir_tex_instr_src_index(instr, nir_tex_src_sampler_offset) == -1); srcs[TEX_LOGICAL_SRC_SAMPLER] = fs_reg(); if (is_resource_src(nir_src)) - srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = s->get_resource_nir_src(nir_src); + srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = get_resource_nir_src(s, nir_src); if (srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE].file == BAD_FILE) srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = bld.emit_uniformize(src); break; @@ -7939,7 +7943,7 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) opcode = SHADER_OPCODE_SAMPLEINFO_LOGICAL; break; case nir_texop_samples_identical: { - fs_reg dst = retype(s->get_nir_def(instr->def), BRW_REGISTER_TYPE_D); + fs_reg dst = retype(get_nir_def(s, instr->def), BRW_REGISTER_TYPE_D); /* If mcs is an immediate value, it means there is no MCS. In that case * just return false. @@ -8047,7 +8051,7 @@ fs_nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) if (instr->is_sparse) nir_dest[dest_size - 1] = component(offset(dst, bld, dest_size - 1), 0); - bld.LOAD_PAYLOAD(s->get_nir_def(instr->def), nir_dest, dest_size, 0); + bld.LOAD_PAYLOAD(get_nir_def(s, instr->def), nir_dest, dest_size, 0); } static void