diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c b/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c index 34470ec7f7c..6d2beb27468 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c @@ -34,6 +34,7 @@ #include "gallivm/lp_bld_sample.h" #include "gallivm/lp_bld_jit_types.h" #include "gallivm/lp_bld_jit_sample.h" +#include "gallivm/lp_bld_flow.h" struct lp_bld_sampler_dynamic_state { @@ -436,6 +437,121 @@ lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base, struct gallivm_state *gallivm, const struct lp_img_params *params) { + LLVMBuilderRef builder = gallivm->builder; + + if (params->resource) { + const struct util_format_description *desc = util_format_description(params->format); + LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, lp_build_texel_type(params->type, desc)); + + LLVMValueRef out_data[4]; + for (uint32_t i = 0; i < 4; i++) { + out_data[i] = lp_build_alloca(gallivm, out_data_type, ""); + LLVMBuildStore(builder, lp_build_const_vec(gallivm, lp_build_texel_type(params->type, desc), 0), out_data[i]); + } + + struct lp_type uint_type = lp_uint_type(params->type); + LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0); + + LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec"); + + LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length); + bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask"); + + LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active"); + + LLVMValueRef binding_index = LLVMBuildExtractValue(builder, params->resource, 1, ""); + LLVMValueRef inbounds = LLVMBuildICmp(builder, LLVMIntSGE, binding_index, lp_build_const_int32(gallivm, 0), "inbounds"); + + struct lp_build_if_state if_state; + lp_build_if(&if_state, gallivm, LLVMBuildAnd(builder, any_active, inbounds, "")); + + LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr); + + LLVMValueRef image_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS); + + LLVMValueRef image_base_ptr = load_texture_functions_ptr( + gallivm, image_descriptor, offsetof(union lp_descriptor, image_functions), + offsetof(struct lp_texture_functions, image_functions)); + + LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, params->ms_index); + LLVMTypeRef image_function_ptr_type = LLVMPointerType(image_function_type, 0); + LLVMTypeRef image_functions_type = LLVMPointerType(image_function_ptr_type, 0); + LLVMTypeRef image_base_type = LLVMPointerType(image_functions_type, 0); + + image_base_ptr = LLVMBuildIntToPtr(builder, image_base_ptr, image_base_type, ""); + LLVMValueRef image_functions = LLVMBuildLoad2(builder, image_functions_type, image_base_ptr, ""); + + uint32_t op = params->img_op; + if (op == LP_IMG_ATOMIC_CAS) + op--; + else if (op == LP_IMG_ATOMIC) + op = params->op + (LP_IMG_OP_COUNT - 1); + + if (params->ms_index) + op += LP_TOTAL_IMAGE_OP_COUNT / 2; + + LLVMValueRef function_index = lp_build_const_int32(gallivm, op); + + LLVMValueRef image_function_ptr = LLVMBuildGEP2(builder, image_function_ptr_type, image_functions, &function_index, 1, ""); + LLVMValueRef image_function = LLVMBuildLoad2(builder, image_function_ptr_type, image_function_ptr, ""); + + LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS] = { 0 }; + uint32_t num_args = 0; + + args[num_args++] = image_descriptor; + + if (params->img_op != LP_IMG_LOAD) + args[num_args++] = params->exec_mask; + + for (uint32_t i = 0; i < 3; i++) + args[num_args++] = params->coords[i]; + + if (params->ms_index) + args[num_args++] = params->ms_index; + + if (params->img_op != LP_IMG_LOAD) + for (uint32_t i = 0; i < 4; i++) + args[num_args++] = params->indata[i]; + + if (params->img_op == LP_IMG_ATOMIC_CAS) + for (uint32_t i = 0; i < 4; i++) + args[num_args++] = params->indata2[i]; + + assert(num_args == LLVMCountParamTypes(image_function_type)); + + LLVMTypeRef param_types[LP_MAX_TEX_FUNC_ARGS]; + LLVMGetParamTypes(image_function_type, param_types); + for (uint32_t i = 0; i < num_args; i++) + if (!args[i]) + args[i] = LLVMGetUndef(param_types[i]); + + if (params->type.length != lp_native_vector_width / 32) + for (uint32_t i = 0; i < num_args; i++) + args[i] = widen_to_simd_width(gallivm, args[i]); + + LLVMValueRef result = LLVMBuildCall2(builder, image_function_type, image_function, args, num_args, ""); + + if (params->img_op != LP_IMG_STORE) { + for (unsigned i = 0; i < 4; i++) { + LLVMValueRef channel = LLVMBuildExtractValue(gallivm->builder, result, i, ""); + if (params->type.length != lp_native_vector_width / 32) + channel = truncate_to_type_width(gallivm, channel, params->type); + + LLVMBuildStore(builder, channel, out_data[i]); + } + } + + lp_build_endif(&if_state); + + if (params->img_op != LP_IMG_STORE) { + for (unsigned i = 0; i < 4; i++) { + params->outdata[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], ""); + } + } + + return; + } + struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base; const unsigned image_index = params->image_index; assert(image_index < PIPE_MAX_SHADER_IMAGES); @@ -475,6 +591,29 @@ lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa *base, { struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base; + if (params->resource) { + LLVMValueRef old_texture = gallivm->texture_descriptor; + + LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr); + gallivm->texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS); + + enum pipe_format format = params->format; + if (format == PIPE_FORMAT_NONE) + format = PIPE_FORMAT_R8G8B8A8_UNORM; + + struct lp_static_texture_state state = { + .format = format, + .res_format = format, + .target = params->target, + }; + + lp_build_size_query_soa(gallivm, &state, &image->dynamic_state.base, params); + + gallivm->texture_descriptor = old_texture; + + return; + } + assert(params->texture_unit < PIPE_MAX_SHADER_IMAGES); lp_build_size_query_soa(gallivm, diff --git a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c index a71c8b80b42..9fcda3a46c6 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_jit_types.c @@ -566,26 +566,44 @@ lp_build_llvm_image_member(struct gallivm_state *gallivm, bool emit_load) { LLVMBuilderRef builder = gallivm->builder; - LLVMValueRef indices[4]; - assert(image_unit < PIPE_MAX_SHADER_IMAGES); + LLVMValueRef ptr; + if (gallivm->texture_descriptor) { + LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(union lp_descriptor, image)); + LLVMValueRef image_ptr = LLVMBuildAdd(builder, gallivm->texture_descriptor, image_offset, ""); - /* resources[0] */ - indices[0] = lp_build_const_int32(gallivm, 0); - /* resources[0].images */ - indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES); - /* resources[0].images[unit] */ - indices[2] = lp_build_const_int32(gallivm, image_unit); - if (image_unit_offset) { - indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, ""); - LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), ""); - indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), ""); + LLVMTypeRef image_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES); + LLVMTypeRef image_type = LLVMGetElementType(image_ptr_type); + image_ptr_type = LLVMPointerType(image_type, 0); + + image_ptr = LLVMBuildIntToPtr(builder, image_ptr, image_ptr_type, ""); + + LLVMValueRef indices[2] = { + lp_build_const_int32(gallivm, 0), + lp_build_const_int32(gallivm, member_index), + }; + ptr = LLVMBuildGEP2(builder, image_type, image_ptr, indices, ARRAY_SIZE(indices), ""); + } else { + LLVMValueRef indices[4]; + + assert(image_unit < PIPE_MAX_SHADER_IMAGES); + + /* resources[0] */ + indices[0] = lp_build_const_int32(gallivm, 0); + /* resources[0].images */ + indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES); + /* resources[0].images[unit] */ + indices[2] = lp_build_const_int32(gallivm, image_unit); + if (image_unit_offset) { + indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, ""); + LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), ""); + indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), ""); + } + /* resources[0].images[unit].member */ + indices[3] = lp_build_const_int32(gallivm, member_index); + + ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), ""); } - /* resources[0].images[unit].member */ - indices[3] = lp_build_const_int32(gallivm, member_index); - - LLVMValueRef ptr = - LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), ""); LLVMValueRef res; if (emit_load) { diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.c b/src/gallium/auxiliary/gallivm/lp_bld_nir.c index f66bdaead43..67f247dd41c 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.c @@ -1623,6 +1623,35 @@ visit_ssbo_atomic(struct lp_build_nir_context *bld_base, offset, val, val2, &result[0]); } +static void +img_params_init_resource(struct lp_build_nir_context *bld_base, struct lp_img_params *params, nir_src src) +{ + if (nir_src_num_components(src) == 1) { + if (nir_src_is_const(src)) + params->image_index = nir_src_as_int(src); + else + params->image_index_offset = get_src(bld_base, src); + + return; + } + + params->resource = get_src(bld_base, src); +} + +static void +sampler_size_params_init_resource(struct lp_build_nir_context *bld_base, struct lp_sampler_size_query_params *params, nir_src src) +{ + if (nir_src_num_components(src) == 1) { + if (nir_src_is_const(src)) + params->texture_unit = nir_src_as_int(src); + else + params->texture_unit_offset = get_src(bld_base, src); + + return; + } + + params->resource = get_src(bld_base, src); +} static void visit_load_image(struct lp_build_nir_context *bld_base, @@ -1649,10 +1678,10 @@ visit_load_image(struct lp_build_nir_context *bld_base, nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_SUBPASS_MS) params.ms_index = cast_type(bld_base, get_src(bld_base, instr->src[2]), nir_type_uint, 32); - if (nir_src_is_const(instr->src[0])) - params.image_index = nir_src_as_int(instr->src[0]); - else - params.image_index_offset = get_src(bld_base, instr->src[0]); + + img_params_init_resource(bld_base, ¶ms, instr->src[0]); + params.format = nir_intrinsic_format(instr); + bld_base->image_op(bld_base, ¶ms); } @@ -1675,17 +1704,24 @@ visit_store_image(struct lp_build_nir_context *bld_base, coords[2] = coords[1]; params.coords = coords; + params.format = nir_intrinsic_format(instr); + + const struct util_format_description *desc = util_format_description(params.format); + bool integer = desc->channel[util_format_get_first_non_void_channel(params.format)].pure_integer; + for (unsigned i = 0; i < 4; i++) { params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, ""); - params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, ""); + + if (integer) + params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->int_bld.vec_type, ""); + else + params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, ""); } if (nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_MS) params.ms_index = get_src(bld_base, instr->src[2]); params.img_op = LP_IMG_STORE; - if (nir_src_is_const(instr->src[0])) - params.image_index = nir_src_as_int(instr->src[0]); - else - params.image_index_offset = get_src(bld_base, instr->src[0]); + + img_params_init_resource(bld_base, ¶ms, instr->src[0]); if (params.target == PIPE_TEXTURE_1D_ARRAY) coords[2] = coords[1]; @@ -1768,27 +1804,38 @@ visit_atomic_image(struct lp_build_nir_context *bld_base, params.coords = coords; + params.format = nir_intrinsic_format(instr); + + const struct util_format_description *desc = util_format_description(params.format); + bool integer = desc->channel[util_format_get_first_non_void_channel(params.format)].pure_integer; + if (nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_MS) params.ms_index = get_src(bld_base, instr->src[2]); - if (instr->intrinsic == nir_intrinsic_image_atomic_swap) { + + if (instr->intrinsic == nir_intrinsic_image_atomic_swap || + instr->intrinsic == nir_intrinsic_bindless_image_atomic_swap) { LLVMValueRef cas_val = get_src(bld_base, instr->src[4]); params.indata[0] = in_val; params.indata2[0] = cas_val; + + if (integer) + params.indata2[0] = LLVMBuildBitCast(builder, params.indata2[0], bld_base->int_bld.vec_type, ""); + else + params.indata2[0] = LLVMBuildBitCast(builder, params.indata2[0], bld_base->base.vec_type, ""); } else { params.indata[0] = in_val; } + if (integer) + params.indata[0] = LLVMBuildBitCast(builder, params.indata[0], bld_base->int_bld.vec_type, ""); + else + params.indata[0] = LLVMBuildBitCast(builder, params.indata[0], bld_base->base.vec_type, ""); + params.outdata = result; lp_img_op_from_intrinsic(¶ms, instr); - params.img_op = - (instr->intrinsic == nir_intrinsic_image_atomic_swap) - ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC; - if (nir_src_is_const(instr->src[0])) - params.image_index = nir_src_as_int(instr->src[0]); - else - params.image_index_offset = get_src(bld_base, instr->src[0]); + img_params_init_resource(bld_base, ¶ms, instr->src[0]); bld_base->image_op(bld_base, ¶ms); } @@ -1801,14 +1848,14 @@ visit_image_size(struct lp_build_nir_context *bld_base, { struct lp_sampler_size_query_params params = { 0 }; - if (nir_src_is_const(instr->src[0])) - params.texture_unit = nir_src_as_int(instr->src[0]); - else - params.texture_unit_offset = get_src(bld_base, instr->src[0]); + sampler_size_params_init_resource(bld_base, ¶ms, instr->src[0]); + params.target = glsl_sampler_to_pipe(nir_intrinsic_image_dim(instr), nir_intrinsic_image_array(instr)); params.sizes_out = result; + params.format = nir_intrinsic_format(instr); + bld_base->image_size(bld_base, ¶ms); } @@ -1820,15 +1867,15 @@ visit_image_samples(struct lp_build_nir_context *bld_base, { struct lp_sampler_size_query_params params = { 0 }; - if (nir_src_is_const(instr->src[0])) - params.texture_unit = nir_src_as_int(instr->src[0]); - else - params.texture_unit_offset = get_src(bld_base, instr->src[0]); + sampler_size_params_init_resource(bld_base, ¶ms, instr->src[0]); + params.target = glsl_sampler_to_pipe(nir_intrinsic_image_dim(instr), nir_intrinsic_image_array(instr)); params.sizes_out = result; params.samples_only = true; + params.format = nir_intrinsic_format(instr); + bld_base->image_size(bld_base, ¶ms); } @@ -2164,19 +2211,25 @@ visit_intrinsic(struct lp_build_nir_context *bld_base, visit_ssbo_atomic(bld_base, instr, result); break; case nir_intrinsic_image_load: + case nir_intrinsic_bindless_image_load: visit_load_image(bld_base, instr, result); break; case nir_intrinsic_image_store: + case nir_intrinsic_bindless_image_store: visit_store_image(bld_base, instr); break; case nir_intrinsic_image_atomic: case nir_intrinsic_image_atomic_swap: + case nir_intrinsic_bindless_image_atomic: + case nir_intrinsic_bindless_image_atomic_swap: visit_atomic_image(bld_base, instr, result); break; case nir_intrinsic_image_size: + case nir_intrinsic_bindless_image_size: visit_image_size(bld_base, instr, result); break; case nir_intrinsic_image_samples: + case nir_intrinsic_bindless_image_samples: visit_image_samples(bld_base, instr, result); break; case nir_intrinsic_load_shared: diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c index 04ed37b73da..b7d7a7e4414 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c @@ -1713,6 +1713,9 @@ static void emit_image_op(struct lp_build_nir_context *bld_base, params->image_index_offset = LLVMBuildExtractElement(gallivm->builder, params->image_index_offset, first_active_invocation(bld_base), ""); + if (params->resource) + params->resource = build_resource_to_scalar(bld_base, params->resource); + bld->image->emit_op(bld->image, bld->bld_base.base.gallivm, params); @@ -1841,6 +1844,12 @@ static void emit_tex(struct lp_build_nir_context *bld_base, first_active_invocation(bld_base), ""); } + if (params->texture_resource) + params->texture_resource = build_resource_to_scalar(bld_base, params->texture_resource); + + if (params->sampler_resource) + params->sampler_resource = build_resource_to_scalar(bld_base, params->sampler_resource); + params->type = bld_base->base.type; bld->sampler->emit_tex_sample(bld->sampler, bld->bld_base.base.gallivm,