diff --git a/src/amd/common/nir/ac_nir_lower_image_opcodes_cdna.c b/src/amd/common/nir/ac_nir_lower_image_opcodes_cdna.c index 55897035726..364ed7280bd 100644 --- a/src/amd/common/nir/ac_nir_lower_image_opcodes_cdna.c +++ b/src/amd/common/nir/ac_nir_lower_image_opcodes_cdna.c @@ -428,6 +428,7 @@ static bool lower_image_opcodes(nir_builder *b, nir_instr *instr, void *data) new_tex->is_array = tex->is_array; new_tex->texture_index = tex->texture_index; new_tex->sampler_index = tex->sampler_index; + new_tex->can_speculate = tex->can_speculate; new_tex->dest_type = nir_type_int32; new_tex->src[0].src = nir_src_for_ssa(tex->src[i].src.ssa); new_tex->src[0].src_type = tex->src[i].src_type; @@ -447,6 +448,7 @@ static bool lower_image_opcodes(nir_builder *b, nir_instr *instr, void *data) new_tex->is_array = tex->is_array; new_tex->texture_index = tex->texture_index; new_tex->sampler_index = tex->sampler_index; + new_tex->can_speculate = tex->can_speculate; new_tex->dest_type = nir_type_int32; new_tex->src[0].src = nir_src_for_ssa(tex->src[i].src.ssa); new_tex->src[0].src_type = tex->src[i].src_type; diff --git a/src/amd/common/nir/ac_nir_lower_resinfo.c b/src/amd/common/nir/ac_nir_lower_resinfo.c index 34e5d5a333d..7173514b7aa 100644 --- a/src/amd/common/nir/ac_nir_lower_resinfo.c +++ b/src/amd/common/nir/ac_nir_lower_resinfo.c @@ -322,6 +322,7 @@ static bool lower_resinfo(nir_builder *b, nir_instr *instr, void *data) new_tex->is_array = tex->is_array; new_tex->texture_index = tex->texture_index; new_tex->sampler_index = tex->sampler_index; + new_tex->can_speculate = tex->can_speculate; new_tex->dest_type = nir_type_int32; new_tex->src[0].src = nir_src_for_ssa(tex->src[i].src.ssa); new_tex->src[0].src_type = tex->src[i].src_type; diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c index a0e4a4be62c..b3082728807 100644 --- a/src/asahi/compiler/agx_compile.c +++ b/src/asahi/compiler/agx_compile.c @@ -2931,14 +2931,22 @@ agx_mem_vectorize_cb(unsigned align_mul, unsigned align_offset, } static bool -set_speculate(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *_) +set_speculate(nir_builder *b, nir_instr *instr, UNUSED void *_) { - if (!nir_intrinsic_has_access(intr)) - return false; + if (instr->type == nir_instr_type_intrinsic && + nir_intrinsic_has_access(nir_instr_as_intrinsic(instr))) { + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + nir_intrinsic_set_access(intr, + ACCESS_CAN_SPECULATE | nir_intrinsic_access(intr)); + return true; + } - nir_intrinsic_set_access(intr, - ACCESS_CAN_SPECULATE | nir_intrinsic_access(intr)); - return true; + if (instr->type == nir_instr_type_tex) { + nir_instr_as_tex(instr)->can_speculate = true; + return true; + } + + return false; } static bool @@ -3067,7 +3075,7 @@ agx_optimize_nir(nir_shader *nir, bool soft_fault, uint16_t *preamble_size, * peephole select and form preambles more aggressively. */ if (soft_fault) { - NIR_PASS(_, nir, nir_shader_intrinsics_pass, set_speculate, + NIR_PASS(_, nir, nir_shader_instructions_pass, set_speculate, nir_metadata_control_flow, NULL); } diff --git a/src/asahi/compiler/agx_nir_lower_texture.c b/src/asahi/compiler/agx_nir_lower_texture.c index 6d86b68ffb3..e5807bae838 100644 --- a/src/asahi/compiler/agx_nir_lower_texture.c +++ b/src/asahi/compiler/agx_nir_lower_texture.c @@ -404,7 +404,8 @@ txs_for_image(nir_builder *b, nir_intrinsic_instr *intr, nir_def *res = nir_build_tex(b, op, .texture_handle = intr->src[0].ssa, .lod = lod, - .dim = dim, .is_array = nir_intrinsic_image_array(intr)); + .dim = dim, .is_array = nir_intrinsic_image_array(intr), + .can_speculate = nir_instr_can_speculate(&intr->instr)); /* Cube images are implemented as 2D arrays, so we need to divide here. */ if (dim == GLSL_SAMPLER_DIM_CUBE && res->num_components > 2 && diff --git a/src/compiler/glsl/gl_nir_linker.c b/src/compiler/glsl/gl_nir_linker.c index d89cf367c94..66f30855d1a 100644 --- a/src/compiler/glsl/gl_nir_linker.c +++ b/src/compiler/glsl/gl_nir_linker.c @@ -206,6 +206,7 @@ gl_nir_inline_functions(nir_shader *shader) &deref->def, instr); replace_tex_src(&intr->src[1], nir_tex_src_sampler_deref, &deref->def, instr); + intr->can_speculate = true; } nir_instr_remove(&intrin->instr); } diff --git a/src/compiler/nir/nir.c b/src/compiler/nir/nir.c index 4d580decd44..9d6970e79b5 100644 --- a/src/compiler/nir/nir.c +++ b/src/compiler/nir/nir.c @@ -816,6 +816,7 @@ nir_tex_instr_create(nir_shader *shader, unsigned num_srcs) instr->texture_index = 0; instr->sampler_index = 0; memcpy(instr->tg4_offsets, default_tg4_offsets, sizeof(instr->tg4_offsets)); + instr->can_speculate = false; return instr; } @@ -2745,7 +2746,7 @@ nir_instr_can_speculate(nir_instr *instr) return true; case nir_instr_type_tex: - return false; + return nir_instr_as_tex(instr)->can_speculate; case nir_instr_type_intrinsic: intr = nir_instr_as_intrinsic(instr); diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index d5c31995844..6fd2df8c63b 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -2474,6 +2474,11 @@ typedef struct nir_tex_instr { /** True if the offset is not dynamically uniform */ bool offset_non_uniform; + /** True whether this returns the same result anywhere in the shader and + * doesn't cause page faults. + */ + bool can_speculate; + /** The texture index * * If this texture instruction has a nir_tex_src_texture_offset source, diff --git a/src/compiler/nir/nir_builder.c b/src/compiler/nir/nir_builder.c index 7c02dc60e85..920d51fa9a4 100644 --- a/src/compiler/nir/nir_builder.c +++ b/src/compiler/nir/nir_builder.c @@ -284,6 +284,7 @@ nir_build_tex_struct(nir_builder *build, nir_texop op, struct nir_tex_builder f) tex->backend_flags = f.backend_flags; tex->texture_index = f.texture_index; tex->sampler_index = f.sampler_index; + tex->can_speculate = f.can_speculate; switch (op) { case nir_texop_txs: diff --git a/src/compiler/nir/nir_builder.h b/src/compiler/nir/nir_builder.h index 4c5bc10cff0..9dbb03f0bff 100644 --- a/src/compiler/nir/nir_builder.h +++ b/src/compiler/nir/nir_builder.h @@ -2189,6 +2189,7 @@ struct nir_tex_builder { enum glsl_sampler_dim dim; nir_alu_type dest_type; bool is_array; + bool can_speculate; uint32_t backend_flags; }; diff --git a/src/compiler/nir/nir_builtin_builder.c b/src/compiler/nir/nir_builtin_builder.c index bee66806583..ee902044bf2 100644 --- a/src/compiler/nir/nir_builtin_builder.c +++ b/src/compiler/nir/nir_builtin_builder.c @@ -319,6 +319,7 @@ nir_build_texture_query(nir_builder *b, nir_tex_instr *tex, nir_texop texop, query->is_new_style_shadow = tex->is_new_style_shadow; query->texture_index = tex->texture_index; query->sampler_index = tex->sampler_index; + query->can_speculate = tex->can_speculate; query->dest_type = dest_type; if (include_coord) { diff --git a/src/compiler/nir/nir_clone.c b/src/compiler/nir/nir_clone.c index 1668eeaddbf..857ab6c6176 100644 --- a/src/compiler/nir/nir_clone.c +++ b/src/compiler/nir/nir_clone.c @@ -414,6 +414,7 @@ clone_tex(clone_state *state, const nir_tex_instr *tex) ntex->texture_index = tex->texture_index; ntex->sampler_index = tex->sampler_index; + ntex->can_speculate = tex->can_speculate; ntex->texture_non_uniform = tex->texture_non_uniform; ntex->sampler_non_uniform = tex->sampler_non_uniform; diff --git a/src/compiler/nir/nir_lower_bitmap.c b/src/compiler/nir/nir_lower_bitmap.c index d87cb7839ef..f4cbc9d579d 100644 --- a/src/compiler/nir/nir_lower_bitmap.c +++ b/src/compiler/nir/nir_lower_bitmap.c @@ -78,7 +78,8 @@ nir_lower_bitmap(nir_shader *shader, const nir_lower_bitmap_options *options) nir_deref_instr *tex_deref = nir_build_deref_var(b, tex_var); nir_def *tex = nir_tex(b, texcoord, .texture_deref = tex_deref, - .sampler_deref = tex_deref); + .sampler_deref = tex_deref, + .can_speculate = true); /* kill if tex != 0.0.. take .x or .w channel according to format: */ nir_def *channel = nir_channel(b, tex, options->swizzle_xxxx ? 0 : 3); diff --git a/src/compiler/nir/nir_lower_drawpixels.c b/src/compiler/nir/nir_lower_drawpixels.c index a780896bc0c..a6247527706 100644 --- a/src/compiler/nir/nir_lower_drawpixels.c +++ b/src/compiler/nir/nir_lower_drawpixels.c @@ -102,7 +102,7 @@ lower_color(nir_builder *b, lower_drawpixels_state *state, nir_intrinsic_instr * * TEX def, texcoord, drawpix_sampler, 2D */ def = nir_tex(b, nir_trim_vector(b, texcoord, 2), .texture_deref = tex_deref, - .sampler_deref = tex_deref); + .sampler_deref = tex_deref, .can_speculate = true); /* Apply the scale and bias. */ if (state->options->scale_and_bias) { @@ -132,6 +132,7 @@ lower_color(nir_builder *b, lower_drawpixels_state *state, nir_intrinsic_instr * tex->coord_components = 2; tex->sampler_index = state->options->pixelmap_sampler; tex->texture_index = state->options->pixelmap_sampler; + tex->can_speculate = true; tex->dest_type = nir_type_float32; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, &pixelmap_deref->def); @@ -150,6 +151,7 @@ lower_color(nir_builder *b, lower_drawpixels_state *state, nir_intrinsic_instr * tex->sampler_dim = GLSL_SAMPLER_DIM_2D; tex->coord_components = 2; tex->sampler_index = state->options->pixelmap_sampler; + tex->can_speculate = true; tex->dest_type = nir_type_float32; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, nir_channels(b, def, 0xc)); diff --git a/src/compiler/nir/nir_lower_fb_read.c b/src/compiler/nir/nir_lower_fb_read.c index a243930aa48..f7efe1b6388 100644 --- a/src/compiler/nir/nir_lower_fb_read.c +++ b/src/compiler/nir/nir_lower_fb_read.c @@ -66,6 +66,7 @@ nir_lower_fb_read_instr(nir_builder *b, nir_intrinsic_instr *intr, tex->coord_components = 3; tex->dest_type = nir_type_float32; tex->is_array = true; + tex->can_speculate = true; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, nir_vec3(b, nir_channel(b, fragcoord, 0), nir_channel(b, fragcoord, 1), layer)); tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_ms_index, sampid); diff --git a/src/compiler/nir/nir_lower_input_attachments.c b/src/compiler/nir/nir_lower_input_attachments.c index dd10a6839b8..323b1887b54 100644 --- a/src/compiler/nir/nir_lower_input_attachments.c +++ b/src/compiler/nir/nir_lower_input_attachments.c @@ -140,6 +140,7 @@ try_lower_input_load(nir_builder *b, nir_intrinsic_instr *load, tex->texture_index = 0; tex->sampler_index = 0; + tex->can_speculate = true; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, &deref->def); diff --git a/src/compiler/nir/nir_lower_readonly_images_to_tex.c b/src/compiler/nir/nir_lower_readonly_images_to_tex.c index 24ce785720c..e57d9005a43 100644 --- a/src/compiler/nir/nir_lower_readonly_images_to_tex.c +++ b/src/compiler/nir/nir_lower_readonly_images_to_tex.c @@ -136,6 +136,7 @@ lower_readonly_image_instr_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin tex->sampler_dim = glsl_get_sampler_dim(deref->type); tex->is_array = glsl_sampler_type_is_array(deref->type); tex->is_shadow = false; + tex->can_speculate = nir_instr_can_speculate(&intrin->instr); unsigned coord_components = glsl_get_sampler_dim_coordinate_components(tex->sampler_dim); diff --git a/src/compiler/nir/nir_lower_tex.c b/src/compiler/nir/nir_lower_tex.c index cce82c3b754..41018e9877a 100644 --- a/src/compiler/nir/nir_lower_tex.c +++ b/src/compiler/nir/nir_lower_tex.c @@ -377,6 +377,7 @@ sample_plane(nir_builder *b, nir_tex_instr *tex, int plane, plane_tex->texture_index = tex->texture_index; plane_tex->sampler_index = tex->sampler_index; + plane_tex->can_speculate = tex->can_speculate; nir_def_init(&plane_tex->instr, &plane_tex->def, 4, tex->def.bit_size); @@ -940,6 +941,7 @@ lower_tex_to_txd(nir_builder *b, nir_tex_instr *tex) txd->is_array = tex->is_array; txd->is_shadow = tex->is_shadow; txd->is_new_style_shadow = tex->is_new_style_shadow; + txd->can_speculate = tex->can_speculate; /* reuse existing srcs */ for (unsigned i = 0; i < tex->num_srcs; i++) { @@ -981,6 +983,7 @@ lower_txb_to_txl(nir_builder *b, nir_tex_instr *tex) txl->is_array = tex->is_array; txl->is_shadow = tex->is_shadow; txl->is_new_style_shadow = tex->is_new_style_shadow; + txl->can_speculate = tex->can_speculate; /* reuse all but bias src */ for (int i = 0; i < tex->num_srcs; i++) { @@ -1275,6 +1278,7 @@ lower_tg4_offsets(nir_builder *b, nir_tex_instr *tex) tex_copy->texture_index = tex->texture_index; tex_copy->sampler_index = tex->sampler_index; tex_copy->backend_flags = tex->backend_flags; + tex_copy->can_speculate = tex->can_speculate; for (unsigned j = 0; j < tex->num_srcs; ++j) { tex_copy->src[j].src = nir_src_for_ssa(tex->src[j].src.ssa); @@ -1407,6 +1411,7 @@ nir_lower_ms_txf_to_fragment_fetch(nir_builder *b, nir_tex_instr *tex) fmask_fetch->texture_non_uniform = tex->texture_non_uniform; fmask_fetch->offset_non_uniform = tex->offset_non_uniform; fmask_fetch->dest_type = nir_type_uint32; + fmask_fetch->can_speculate = tex->can_speculate; nir_def_init(&fmask_fetch->instr, &fmask_fetch->def, 1, 32); fmask_fetch->num_srcs = 0; diff --git a/src/compiler/nir/nir_serialize.c b/src/compiler/nir/nir_serialize.c index d5fbe4c7267..197fdab4e05 100644 --- a/src/compiler/nir/nir_serialize.c +++ b/src/compiler/nir/nir_serialize.c @@ -1366,7 +1366,8 @@ union packed_tex_data { unsigned offset_non_uniform : 1; unsigned array_is_lowered_cube : 1; unsigned is_gather_implicit_lod : 1; - unsigned unused : 4; /* Mark unused for valgrind. */ + unsigned can_speculate : 1; + unsigned unused : 3; /* Mark unused for valgrind. */ } u; }; @@ -1406,6 +1407,7 @@ write_tex(write_ctx *ctx, const nir_tex_instr *tex) .u.offset_non_uniform = tex->offset_non_uniform, .u.array_is_lowered_cube = tex->array_is_lowered_cube, .u.is_gather_implicit_lod = tex->is_gather_implicit_lod, + .u.can_speculate = tex->can_speculate, }; blob_write_uint32(ctx->blob, packed.u32); @@ -1446,6 +1448,7 @@ read_tex(read_ctx *ctx, union packed_instr header) tex->offset_non_uniform = packed.u.offset_non_uniform; tex->array_is_lowered_cube = packed.u.array_is_lowered_cube; tex->is_gather_implicit_lod = packed.u.is_gather_implicit_lod; + tex->can_speculate = packed.u.can_speculate; for (unsigned i = 0; i < tex->num_srcs; i++) { union packed_src src = read_src(ctx, &tex->src[i].src); diff --git a/src/compiler/spirv/spirv_to_nir.c b/src/compiler/spirv/spirv_to_nir.c index f86b43a98d3..6dc34228744 100644 --- a/src/compiler/spirv/spirv_to_nir.c +++ b/src/compiler/spirv/spirv_to_nir.c @@ -3749,6 +3749,8 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode, instr->is_new_style_shadow = is_shadow && glsl_get_components(ret_type->type) == 1; instr->component = gather_component; + /* spirv_to_nir doesn't support OpenGL bindless textures. */ + instr->can_speculate = b->options->environment == NIR_SPIRV_OPENGL; /* If SpvCapabilityImageGatherBiasLodAMD is enabled, texture gather without an explicit LOD * has an implicit one (instead of using level 0). diff --git a/src/freedreno/ir3/ir3_nir_opt_preamble.c b/src/freedreno/ir3/ir3_nir_opt_preamble.c index 64ae33846cf..97980470912 100644 --- a/src/freedreno/ir3/ir3_nir_opt_preamble.c +++ b/src/freedreno/ir3/ir3_nir_opt_preamble.c @@ -269,8 +269,18 @@ avoid_instr(const nir_instr *instr, const void *data) } static bool -set_speculate(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *_) +set_speculate(nir_builder *b, nir_instr *instr, UNUSED void *_) { + if (instr->type == nir_instr_type_tex) { + nir_instr_as_tex(instr)->can_speculate = true; + return true; + } + + if (instr->type != nir_instr_type_intrinsic) + return false; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + switch (intr->intrinsic) { /* These instructions go through bounds-checked hardware descriptors so * should be safe to speculate. @@ -313,8 +323,8 @@ ir3_nir_opt_preamble(nir_shader *nir, struct ir3_shader_variant *v) if (max_size == 0) return false; - bool progress = nir_shader_intrinsics_pass(nir, set_speculate, - nir_metadata_control_flow, NULL); + bool progress = nir_shader_instructions_pass(nir, set_speculate, + nir_metadata_control_flow, NULL); nir_opt_preamble_options options = { .drawid_uniform = true, diff --git a/src/gallium/auxiliary/nir/nir_draw_helpers.c b/src/gallium/auxiliary/nir/nir_draw_helpers.c index 220203ed7c8..3e93011ef33 100644 --- a/src/gallium/auxiliary/nir/nir_draw_helpers.c +++ b/src/gallium/auxiliary/nir/nir_draw_helpers.c @@ -77,6 +77,7 @@ nir_lower_pstipple_block(nir_block *block, tex->dest_type = nir_type_float32; tex->texture_index = state->stip_tex->data.binding; tex->sampler_index = state->stip_tex->data.binding; + tex->can_speculate = true; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, texcoord); nir_def_init(&tex->instr, &tex->def, 4, 32); diff --git a/src/gallium/auxiliary/nir/tgsi_to_nir.c b/src/gallium/auxiliary/nir/tgsi_to_nir.c index 644b6426dd2..a8e2c9b4c78 100644 --- a/src/gallium/auxiliary/nir/tgsi_to_nir.c +++ b/src/gallium/auxiliary/nir/tgsi_to_nir.c @@ -1259,6 +1259,7 @@ ttn_tex(struct ttn_compile *c, nir_def **src) instr = nir_tex_instr_create(b->shader, num_srcs); instr->op = op; + instr->can_speculate = true; /* No shaders come from SPIR-V or GLSL. */ get_texture_info(tgsi_inst->Texture.Texture, &instr->sampler_dim, &instr->is_shadow, &instr->is_array); @@ -1441,12 +1442,14 @@ ttn_txq(struct ttn_compile *c, nir_def **src) txs = nir_tex_instr_create(b->shader, 2); txs->op = nir_texop_txs; txs->dest_type = nir_type_uint32; + txs->can_speculate = true; get_texture_info(tgsi_inst->Texture.Texture, &txs->sampler_dim, &txs->is_shadow, &txs->is_array); qlv = nir_tex_instr_create(b->shader, 1); qlv->op = nir_texop_query_levels; qlv->dest_type = nir_type_uint32; + qlv->can_speculate = true; get_texture_info(tgsi_inst->Texture.Texture, &qlv->sampler_dim, &qlv->is_shadow, &qlv->is_array); diff --git a/src/gallium/drivers/zink/zink_lower_cubemap_to_array.c b/src/gallium/drivers/zink/zink_lower_cubemap_to_array.c index 8557c114d5b..4ce1ceb358e 100644 --- a/src/gallium/drivers/zink/zink_lower_cubemap_to_array.c +++ b/src/gallium/drivers/zink/zink_lower_cubemap_to_array.c @@ -158,6 +158,7 @@ create_array_tex_from_cube_tex(nir_builder *b, nir_tex_instr *tex, nir_def *coor array_tex->sampler_index = tex->sampler_index; array_tex->dest_type = tex->dest_type; array_tex->coord_components = 3; + array_tex->can_speculate = tex->can_speculate; nir_src coord_src = nir_src_for_ssa(coord); unsigned s = 0; @@ -415,6 +416,7 @@ lower_tex_to_txl(nir_builder *b, nir_tex_instr *tex) txl->is_shadow = tex->is_shadow; txl->is_sparse = tex->is_sparse; txl->is_new_style_shadow = tex->is_new_style_shadow; + txl->can_speculate = tex->can_speculate; unsigned s = 0; for (int i = 0; i < tex->num_srcs; i++) { diff --git a/src/mesa/main/ff_fragment_shader.c b/src/mesa/main/ff_fragment_shader.c index 9bb9d96029c..99237053f4d 100644 --- a/src/mesa/main/ff_fragment_shader.c +++ b/src/mesa/main/ff_fragment_shader.c @@ -771,6 +771,7 @@ load_texture(struct texenv_fragment_program *p, GLuint unit) tex->dest_type = nir_type_float32; tex->texture_index = unit; tex->sampler_index = unit; + tex->can_speculate = true; tex->sampler_dim = _mesa_texture_index_to_sampler_dim(texTarget, diff --git a/src/mesa/program/prog_to_nir.c b/src/mesa/program/prog_to_nir.c index 7d4bed6e482..264090d5d3c 100644 --- a/src/mesa/program/prog_to_nir.c +++ b/src/mesa/program/prog_to_nir.c @@ -422,6 +422,7 @@ ptn_tex(struct ptn_compile *c, nir_def **src, instr->op = op; instr->dest_type = nir_type_float32; instr->is_shadow = prog_inst->TexShadow; + instr->can_speculate = true; bool is_array; instr->sampler_dim = _mesa_texture_index_to_sampler_dim(prog_inst->TexSrcTarget, &is_array); diff --git a/src/mesa/state_tracker/st_atifs_to_nir.c b/src/mesa/state_tracker/st_atifs_to_nir.c index 88ffa6d4fc3..b08de70e6ac 100644 --- a/src/mesa/state_tracker/st_atifs_to_nir.c +++ b/src/mesa/state_tracker/st_atifs_to_nir.c @@ -355,6 +355,7 @@ compile_setupinst(struct st_translate *t, tex->dest_type = nir_type_float32; tex->coord_components = glsl_get_sampler_dim_coordinate_components(tex->sampler_dim); + tex->can_speculate = true; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, &tex_deref->def); diff --git a/src/mesa/state_tracker/st_cb_drawpixels.c b/src/mesa/state_tracker/st_cb_drawpixels.c index 6f70eed7205..41fda80dd3a 100644 --- a/src/mesa/state_tracker/st_cb_drawpixels.c +++ b/src/mesa/state_tracker/st_cb_drawpixels.c @@ -129,6 +129,7 @@ sample_via_nir(nir_builder *b, const char *name, int sampler, tex->op = nir_texop_tex; tex->sampler_dim = GLSL_SAMPLER_DIM_2D; tex->coord_components = 2; + tex->can_speculate = true; tex->dest_type = alu_type; tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, &deref->def); diff --git a/src/mesa/state_tracker/st_pbo.c b/src/mesa/state_tracker/st_pbo.c index 44253479564..9aa86849e05 100644 --- a/src/mesa/state_tracker/st_pbo.c +++ b/src/mesa/state_tracker/st_pbo.c @@ -513,6 +513,7 @@ create_fs(struct st_context *st, bool download, tex->coord_components = glsl_get_sampler_coordinate_components(tex_var->type); tex->is_array = target >= PIPE_TEXTURE_1D_ARRAY; + tex->can_speculate = true; tex->dest_type = nir_get_nir_type_for_glsl_base_type(glsl_get_sampler_result_type(tex_var->type)); tex->src[0].src_type = nir_tex_src_texture_deref; diff --git a/src/mesa/state_tracker/st_pbo_compute.c b/src/mesa/state_tracker/st_pbo_compute.c index 5dc30db3588..f4f4b269964 100644 --- a/src/mesa/state_tracker/st_pbo_compute.c +++ b/src/mesa/state_tracker/st_pbo_compute.c @@ -683,6 +683,7 @@ create_conversion_shader(struct st_context *st, enum pipe_texture_target target, txf->coord_components = coord_components; txf->texture_index = 0; txf->sampler_index = 0; + txf->can_speculate = true; txf->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, coord); txf->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(&b, 0)); txf->src[2].src_type = nir_tex_src_texture_deref; diff --git a/src/microsoft/compiler/dxil_nir_lower_int_cubemaps.c b/src/microsoft/compiler/dxil_nir_lower_int_cubemaps.c index bad408a33e3..3e0cbc09ced 100644 --- a/src/microsoft/compiler/dxil_nir_lower_int_cubemaps.c +++ b/src/microsoft/compiler/dxil_nir_lower_int_cubemaps.c @@ -178,6 +178,7 @@ create_array_tex_from_cube_tex(nir_builder *b, nir_tex_instr *tex, nir_def *coor array_tex->sampler_index = tex->sampler_index; array_tex->dest_type = tex->dest_type; array_tex->coord_components = 3; + array_tex->can_speculate = tex->can_speculate; nir_src coord_src = nir_src_for_ssa(coord); for (unsigned i = 0; i < tex->num_srcs; i++) { diff --git a/src/microsoft/compiler/dxil_nir_lower_int_samplers.c b/src/microsoft/compiler/dxil_nir_lower_int_samplers.c index 0fde461543f..c5b6b40231b 100644 --- a/src/microsoft/compiler/dxil_nir_lower_int_samplers.c +++ b/src/microsoft/compiler/dxil_nir_lower_int_samplers.c @@ -78,6 +78,7 @@ dx_get_texture_lod(nir_builder *b, nir_tex_instr *tex) tql->is_new_style_shadow = tex->is_new_style_shadow; tql->texture_index = tex->texture_index; tql->sampler_index = tex->sampler_index; + tql->can_speculate = tex->can_speculate; tql->dest_type = nir_type_float32; /* The coordinate needs special handling because we might have @@ -282,6 +283,7 @@ create_txf_from_tex(nir_builder *b, nir_tex_instr *tex) txf->is_new_style_shadow = tex->is_new_style_shadow; txf->texture_index = tex->texture_index; txf->sampler_index = tex->sampler_index; + txf->can_speculate = tex->can_speculate; txf->dest_type = tex->dest_type; unsigned idx = 0; diff --git a/src/nouveau/compiler/nak_nir_lower_tex.c b/src/nouveau/compiler/nak_nir_lower_tex.c index 7c11031eb76..083353c47b9 100644 --- a/src/nouveau/compiler/nak_nir_lower_tex.c +++ b/src/nouveau/compiler/nak_nir_lower_tex.c @@ -43,7 +43,7 @@ tex_handle_as_cbuf(nir_def *tex_h, uint32_t *cbuf_out) } static nir_def * -build_txq(nir_builder *b, nir_texop op, nir_def *img_h, +build_txq(nir_builder *b, nir_texop op, bool can_speculate, nir_def *img_h, nir_def *lod_idx, const struct nak_compiler *nak) { uint32_t texture_index = 0; @@ -63,6 +63,7 @@ build_txq(nir_builder *b, nir_texop op, nir_def *img_h, txq->sampler_dim = GLSL_SAMPLER_DIM_2D, txq->is_array = false, txq->dest_type = nir_type_int32; + txq->can_speculate = can_speculate; nir_def *src[2] = { NULL, }; unsigned src_comps = 0; @@ -94,23 +95,23 @@ build_txq(nir_builder *b, nir_texop op, nir_def *img_h, } static nir_def * -build_txq_levels(nir_builder *b, nir_def *img_h, +build_txq_levels(nir_builder *b, bool can_speculate, nir_def *img_h, const struct nak_compiler *nak) { - nir_def *res = build_txq(b, nir_texop_hdr_dim_nv, img_h, + nir_def *res = build_txq(b, nir_texop_hdr_dim_nv, can_speculate, img_h, nir_imm_int(b, 0), nak); return nir_channel(b, res, 3); } static nir_def * -build_img_is_null(nir_builder *b, nir_def *img_h, +build_img_is_null(nir_builder *b, bool can_speculate, nir_def *img_h, const struct nak_compiler *nak) { /* Prior to Volta, we don't have real NULL descriptors but we can figure * out if it's null based on the number of levels returned by * txq.dimension. */ - return nir_ieq_imm(b, build_txq_levels(b, img_h, nak), 0); + return nir_ieq_imm(b, build_txq_levels(b, can_speculate, img_h, nak), 0); } static enum glsl_sampler_dim @@ -413,7 +414,7 @@ lower_tex(nir_builder *b, nir_tex_instr *tex, const struct nak_compiler *nak) nir_def *res = nir_vec2(b, abs, rel); if (!has_null_descriptors(nak)) { - res = nir_bcsel(b, build_img_is_null(b, tex_h, nak), + res = nir_bcsel(b, build_img_is_null(b, tex->can_speculate, tex_h, nak), nir_imm_int(b, 0), res); } @@ -424,21 +425,22 @@ lower_tex(nir_builder *b, nir_tex_instr *tex, const struct nak_compiler *nak) } static nir_def * -build_txq_samples_raw(nir_builder *b, nir_def *img_h, +build_txq_samples_raw(nir_builder *b, bool can_speculate, nir_def *img_h, const struct nak_compiler *nak) { - nir_def *res = build_txq(b, nir_texop_tex_type_nv, img_h, NULL, nak); + nir_def *res = build_txq(b, nir_texop_tex_type_nv, can_speculate, img_h, + NULL, nak); return nir_channel(b, res, 2); } static nir_def * -build_txq_samples(nir_builder *b, nir_def *img_h, +build_txq_samples(nir_builder *b, bool can_speculate, nir_def *img_h, const struct nak_compiler *nak) { - nir_def *res = build_txq_samples_raw(b, img_h, nak); + nir_def *res = build_txq_samples_raw(b, can_speculate, img_h, nak); if (!has_null_descriptors(nak)) { - res = nir_bcsel(b, build_img_is_null(b, img_h, nak), + res = nir_bcsel(b, build_img_is_null(b, can_speculate, img_h, nak), nir_imm_int(b, 0), res); } @@ -446,18 +448,19 @@ build_txq_samples(nir_builder *b, nir_def *img_h, } static nir_def * -build_txq_size(nir_builder *b, unsigned num_components, +build_txq_size(nir_builder *b, unsigned num_components, bool can_speculate, nir_def *img_h, nir_def *lod, const struct nak_compiler *nak) { if (lod == NULL) lod = nir_imm_int(b, 0); - nir_def *res = build_txq(b, nir_texop_hdr_dim_nv, img_h, lod, nak); + nir_def *res = build_txq(b, nir_texop_hdr_dim_nv, can_speculate, img_h, + lod, nak); res = nir_trim_vector(b, res, num_components); if (!has_null_descriptors(nak)) { - res = nir_bcsel(b, build_img_is_null(b, img_h, nak), + res = nir_bcsel(b, build_img_is_null(b, can_speculate, img_h, nak), nir_imm_int(b, 0), res); } @@ -485,13 +488,14 @@ lower_txq(nir_builder *b, nir_tex_instr *tex, const struct nak_compiler *nak) nir_def *res; switch (tex->op) { case nir_texop_txs: - res = build_txq_size(b, tex->def.num_components, tex_h, lod, nak); + res = build_txq_size(b, tex->can_speculate, tex->def.num_components, + tex_h, lod, nak); break; case nir_texop_query_levels: - res = build_txq_levels(b, tex_h, nak); + res = build_txq_levels(b, tex->can_speculate, tex_h, nak); break; case nir_texop_texture_samples: - res = build_txq_samples(b, tex_h, nak); + res = build_txq_samples(b, tex->can_speculate, tex_h, nak); break; default: unreachable("Invalid texture query op"); @@ -696,6 +700,7 @@ lower_msaa_image_access(nir_builder *b, nir_intrinsic_instr *intrin, b->cursor = nir_before_instr(&intrin->instr); + bool can_speculate = nir_instr_can_speculate(&intrin->instr); nir_def *img_h = intrin->src[0].ssa; nir_def *x = nir_channel(b, intrin->src[1].ssa, 0); nir_def *y = nir_channel(b, intrin->src[1].ssa, 1); @@ -703,7 +708,7 @@ lower_msaa_image_access(nir_builder *b, nir_intrinsic_instr *intrin, nir_def *w = nir_channel(b, intrin->src[1].ssa, 3); nir_def *s = intrin->src[2].ssa; - nir_def *samples = build_txq_samples_raw(b, img_h, nak); + nir_def *samples = build_txq_samples_raw(b, can_speculate, img_h, nak); nir_def *px_size_sa_log2 = build_px_size_sa_log2(b, samples); nir_def *px_w_log2 = nir_channel(b, px_size_sa_log2, 0); @@ -714,7 +719,8 @@ lower_msaa_image_access(nir_builder *b, nir_intrinsic_instr *intrin, * txq.sampler_pos gives us the sample coordinates as a signed 4.12 fixed * point with x in the bottom 16 bits and y in the top 16 bits. */ - nir_def *spos_sf = build_txq(b, nir_texop_sample_pos_nv, img_h, s, nak); + nir_def *spos_sf = build_txq(b, nir_texop_sample_pos_nv, can_speculate, + img_h, s, nak); spos_sf = nir_trim_vector(b, spos_sf, 2); /* Fortunately, the samples are laid out in the supersampled image the same @@ -755,10 +761,12 @@ lower_image_txq(nir_builder *b, nir_intrinsic_instr *intrin, nir_def *img_h = intrin->src[0].ssa; + bool can_speculate = nir_instr_can_speculate(&intrin->instr); nir_def *res; + switch (intrin->intrinsic) { case nir_intrinsic_bindless_image_size: - res = build_txq_size(b, intrin->def.num_components, img_h, + res = build_txq_size(b, intrin->def.num_components, can_speculate, img_h, intrin->src[1].ssa /* lod */, nak); if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS) { @@ -768,13 +776,13 @@ lower_image_txq(nir_builder *b, nir_intrinsic_instr *intrin, * they're given. This means we need to divide back out the pixel * size in order to get the size in pixels. */ - nir_def *samples = build_txq_samples_raw(b, img_h, nak); + nir_def *samples = build_txq_samples_raw(b, can_speculate, img_h, nak); nir_def *px_size_sa_log2 = build_px_size_sa_log2(b, samples); res = nir_ushr(b, res, px_size_sa_log2); } break; case nir_intrinsic_bindless_image_samples: - res = build_txq_samples(b, img_h, nak); + res = build_txq_samples(b, can_speculate, img_h, nak); break; default: unreachable("Invalid image query op");