gallivm: Implement vulkan images

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22828>

src/gallium/auxiliary/gallivm/lp_bld_jit_sample.c

@@ -34,6 +34,7 @@
 #include "gallivm/lp_bld_sample.h"
 #include "gallivm/lp_bld_jit_types.h"
 #include "gallivm/lp_bld_jit_sample.h"
+#include "gallivm/lp_bld_flow.h"
 
 struct lp_bld_sampler_dynamic_state
 {
@@ -436,6 +437,121 @@ lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base,
                               struct gallivm_state *gallivm,
                               const struct lp_img_params *params)
 {
+   LLVMBuilderRef builder = gallivm->builder;
+
+   if (params->resource) {
+      const struct util_format_description *desc = util_format_description(params->format);
+      LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, lp_build_texel_type(params->type, desc));
+
+      LLVMValueRef out_data[4];
+      for (uint32_t i = 0; i < 4; i++) {
+         out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
+         LLVMBuildStore(builder, lp_build_const_vec(gallivm, lp_build_texel_type(params->type, desc), 0), out_data[i]);
+      }
+
+      struct lp_type uint_type = lp_uint_type(params->type);
+      LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
+
+      LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
+
+      LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
+      bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
+
+      LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
+
+      LLVMValueRef binding_index = LLVMBuildExtractValue(builder, params->resource, 1, "");
+      LLVMValueRef inbounds = LLVMBuildICmp(builder, LLVMIntSGE, binding_index, lp_build_const_int32(gallivm, 0), "inbounds");
+
+      struct lp_build_if_state if_state;
+      lp_build_if(&if_state, gallivm, LLVMBuildAnd(builder, any_active, inbounds, ""));
+
+      LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
+
+      LLVMValueRef image_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
+
+      LLVMValueRef image_base_ptr = load_texture_functions_ptr(
+         gallivm, image_descriptor, offsetof(union lp_descriptor, image_functions),
+         offsetof(struct lp_texture_functions, image_functions));
+
+      LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, params->ms_index);
+      LLVMTypeRef image_function_ptr_type = LLVMPointerType(image_function_type, 0);
+      LLVMTypeRef image_functions_type = LLVMPointerType(image_function_ptr_type, 0);
+      LLVMTypeRef image_base_type = LLVMPointerType(image_functions_type, 0);
+
+      image_base_ptr = LLVMBuildIntToPtr(builder, image_base_ptr, image_base_type, "");
+      LLVMValueRef image_functions = LLVMBuildLoad2(builder, image_functions_type, image_base_ptr, "");
+
+      uint32_t op = params->img_op;
+      if (op == LP_IMG_ATOMIC_CAS)
+         op--;
+      else if (op == LP_IMG_ATOMIC)
+         op = params->op + (LP_IMG_OP_COUNT - 1);
+
+      if (params->ms_index)
+         op += LP_TOTAL_IMAGE_OP_COUNT / 2;
+
+      LLVMValueRef function_index = lp_build_const_int32(gallivm, op);
+
+      LLVMValueRef image_function_ptr = LLVMBuildGEP2(builder, image_function_ptr_type, image_functions, &function_index, 1, "");
+      LLVMValueRef image_function = LLVMBuildLoad2(builder, image_function_ptr_type, image_function_ptr, "");
+
+      LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS] = { 0 };
+      uint32_t num_args = 0;
+
+      args[num_args++] = image_descriptor;
+
+      if (params->img_op != LP_IMG_LOAD)
+         args[num_args++] = params->exec_mask;
+
+      for (uint32_t i = 0; i < 3; i++)
+         args[num_args++] = params->coords[i];
+
+      if (params->ms_index)
+         args[num_args++] = params->ms_index;
+
+      if (params->img_op != LP_IMG_LOAD)
+         for (uint32_t i = 0; i < 4; i++)
+            args[num_args++] = params->indata[i];
+
+      if (params->img_op == LP_IMG_ATOMIC_CAS)
+         for (uint32_t i = 0; i < 4; i++)
+            args[num_args++] = params->indata2[i];
+
+      assert(num_args == LLVMCountParamTypes(image_function_type));
+
+      LLVMTypeRef param_types[LP_MAX_TEX_FUNC_ARGS];
+      LLVMGetParamTypes(image_function_type, param_types);
+      for (uint32_t i = 0; i < num_args; i++)
+         if (!args[i])
+            args[i] = LLVMGetUndef(param_types[i]);
+
+      if (params->type.length != lp_native_vector_width / 32)
+         for (uint32_t i = 0; i < num_args; i++)
+            args[i] = widen_to_simd_width(gallivm, args[i]);
+
+      LLVMValueRef result = LLVMBuildCall2(builder, image_function_type, image_function, args, num_args, "");
+
+      if (params->img_op != LP_IMG_STORE) {
+         for (unsigned i = 0; i < 4; i++) {
+            LLVMValueRef channel = LLVMBuildExtractValue(gallivm->builder, result, i, "");
+            if (params->type.length != lp_native_vector_width / 32)
+               channel = truncate_to_type_width(gallivm, channel, params->type);
+
+            LLVMBuildStore(builder, channel, out_data[i]);
+         }
+      }
+
+      lp_build_endif(&if_state);
+
+      if (params->img_op != LP_IMG_STORE) {
+         for (unsigned i = 0; i < 4; i++) {
+            params->outdata[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], "");
+         }
+      }
+
+      return;
+   }
+
    struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
    const unsigned image_index = params->image_index;
    assert(image_index < PIPE_MAX_SHADER_IMAGES);
@@ -475,6 +591,29 @@ lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa *base,
 {
    struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
 
+   if (params->resource) {
+      LLVMValueRef old_texture = gallivm->texture_descriptor;
+
+      LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
+      gallivm->texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
+
+      enum pipe_format format = params->format;
+      if (format == PIPE_FORMAT_NONE)
+         format = PIPE_FORMAT_R8G8B8A8_UNORM;
+
+      struct lp_static_texture_state state = {
+         .format = format,
+         .res_format = format,
+         .target = params->target,
+      };
+      
+      lp_build_size_query_soa(gallivm, &state, &image->dynamic_state.base, params);
+
+      gallivm->texture_descriptor = old_texture;
+
+      return;
+   }
+
    assert(params->texture_unit < PIPE_MAX_SHADER_IMAGES);
 
    lp_build_size_query_soa(gallivm,

src/gallium/auxiliary/gallivm/lp_bld_jit_types.c

@@ -566,26 +566,44 @@ lp_build_llvm_image_member(struct gallivm_state *gallivm,
                            bool emit_load)
 {
    LLVMBuilderRef builder = gallivm->builder;
-   LLVMValueRef indices[4];
 
-   assert(image_unit < PIPE_MAX_SHADER_IMAGES);
+   LLVMValueRef ptr;
+   if (gallivm->texture_descriptor) {
+      LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(union lp_descriptor, image));
+      LLVMValueRef image_ptr = LLVMBuildAdd(builder, gallivm->texture_descriptor, image_offset, "");
 
-   /* resources[0] */
-   indices[0] = lp_build_const_int32(gallivm, 0);
-   /* resources[0].images */
-   indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES);
-   /* resources[0].images[unit] */
-   indices[2] = lp_build_const_int32(gallivm, image_unit);
-   if (image_unit_offset) {
-      indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, "");
-      LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), "");
-      indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), "");
+      LLVMTypeRef image_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
+      LLVMTypeRef image_type = LLVMGetElementType(image_ptr_type);
+      image_ptr_type = LLVMPointerType(image_type, 0);
+
+      image_ptr = LLVMBuildIntToPtr(builder, image_ptr, image_ptr_type, "");
+
+      LLVMValueRef indices[2] = {
+         lp_build_const_int32(gallivm, 0),
+         lp_build_const_int32(gallivm, member_index),
+      };
+      ptr = LLVMBuildGEP2(builder, image_type, image_ptr, indices, ARRAY_SIZE(indices), "");
+   } else {
+      LLVMValueRef indices[4];
+
+      assert(image_unit < PIPE_MAX_SHADER_IMAGES);
+
+      /* resources[0] */
+      indices[0] = lp_build_const_int32(gallivm, 0);
+      /* resources[0].images */
+      indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES);
+      /* resources[0].images[unit] */
+      indices[2] = lp_build_const_int32(gallivm, image_unit);
+      if (image_unit_offset) {
+         indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, "");
+         LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), "");
+         indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), "");
+      }
+      /* resources[0].images[unit].member */
+      indices[3] = lp_build_const_int32(gallivm, member_index);
+
+      ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
    }
-   /* resources[0].images[unit].member */
-   indices[3] = lp_build_const_int32(gallivm, member_index);
-
-   LLVMValueRef ptr =
-      LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
 
    LLVMValueRef res;
    if (emit_load) {

src/gallium/auxiliary/gallivm/lp_bld_nir.c

@@ -1623,6 +1623,35 @@ visit_ssbo_atomic(struct lp_build_nir_context *bld_base,
                         offset, val, val2, &result[0]);
 }
 
+static void
+img_params_init_resource(struct lp_build_nir_context *bld_base, struct lp_img_params *params, nir_src src)
+{
+   if (nir_src_num_components(src) == 1) {
+      if (nir_src_is_const(src))
+         params->image_index = nir_src_as_int(src);
+      else
+         params->image_index_offset = get_src(bld_base, src);
+   
+      return;
+   }
+
+   params->resource = get_src(bld_base, src);
+}
+
+static void
+sampler_size_params_init_resource(struct lp_build_nir_context *bld_base, struct lp_sampler_size_query_params *params, nir_src src)
+{
+   if (nir_src_num_components(src) == 1) {
+      if (nir_src_is_const(src))
+         params->texture_unit = nir_src_as_int(src);
+      else
+         params->texture_unit_offset = get_src(bld_base, src);
+   
+      return;
+   }
+
+   params->resource = get_src(bld_base, src);
+}
 
 static void
 visit_load_image(struct lp_build_nir_context *bld_base,
@@ -1649,10 +1678,10 @@ visit_load_image(struct lp_build_nir_context *bld_base,
        nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_SUBPASS_MS)
       params.ms_index = cast_type(bld_base, get_src(bld_base, instr->src[2]),
                                   nir_type_uint, 32);
-   if (nir_src_is_const(instr->src[0]))
-      params.image_index = nir_src_as_int(instr->src[0]);
-   else
-      params.image_index_offset = get_src(bld_base, instr->src[0]);
+
+   img_params_init_resource(bld_base, &params, instr->src[0]);
+   params.format = nir_intrinsic_format(instr);
+
    bld_base->image_op(bld_base, &params);
 }
 
@@ -1675,17 +1704,24 @@ visit_store_image(struct lp_build_nir_context *bld_base,
       coords[2] = coords[1];
    params.coords = coords;
 
+   params.format = nir_intrinsic_format(instr);
+
+   const struct util_format_description *desc = util_format_description(params.format);
+   bool integer = desc->channel[util_format_get_first_non_void_channel(params.format)].pure_integer;
+
    for (unsigned i = 0; i < 4; i++) {
       params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, "");
-      params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
+
+      if (integer)
+         params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->int_bld.vec_type, "");
+      else
+         params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
    }
    if (nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_MS)
       params.ms_index = get_src(bld_base, instr->src[2]);
    params.img_op = LP_IMG_STORE;
-   if (nir_src_is_const(instr->src[0]))
-      params.image_index = nir_src_as_int(instr->src[0]);
-   else
-      params.image_index_offset = get_src(bld_base, instr->src[0]);
+
+   img_params_init_resource(bld_base, &params, instr->src[0]);
 
    if (params.target == PIPE_TEXTURE_1D_ARRAY)
       coords[2] = coords[1];
@@ -1768,27 +1804,38 @@ visit_atomic_image(struct lp_build_nir_context *bld_base,
 
    params.coords = coords;
 
+   params.format = nir_intrinsic_format(instr);
+
+   const struct util_format_description *desc = util_format_description(params.format);
+   bool integer = desc->channel[util_format_get_first_non_void_channel(params.format)].pure_integer;
+
    if (nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_MS)
       params.ms_index = get_src(bld_base, instr->src[2]);
-   if (instr->intrinsic == nir_intrinsic_image_atomic_swap) {
+
+   if (instr->intrinsic == nir_intrinsic_image_atomic_swap ||
+       instr->intrinsic == nir_intrinsic_bindless_image_atomic_swap) {
       LLVMValueRef cas_val = get_src(bld_base, instr->src[4]);
       params.indata[0] = in_val;
       params.indata2[0] = cas_val;
+
+      if (integer)
+         params.indata2[0] = LLVMBuildBitCast(builder, params.indata2[0], bld_base->int_bld.vec_type, "");
+      else
+         params.indata2[0] = LLVMBuildBitCast(builder, params.indata2[0], bld_base->base.vec_type, "");
    } else {
       params.indata[0] = in_val;
    }
 
+   if (integer)
+      params.indata[0] = LLVMBuildBitCast(builder, params.indata[0], bld_base->int_bld.vec_type, "");
+   else
+      params.indata[0] = LLVMBuildBitCast(builder, params.indata[0], bld_base->base.vec_type, "");
+
    params.outdata = result;
 
    lp_img_op_from_intrinsic(&params, instr);
 
-   params.img_op =
-      (instr->intrinsic == nir_intrinsic_image_atomic_swap)
-      ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC;
-   if (nir_src_is_const(instr->src[0]))
-      params.image_index = nir_src_as_int(instr->src[0]);
-   else
-      params.image_index_offset = get_src(bld_base, instr->src[0]);
+   img_params_init_resource(bld_base, &params, instr->src[0]);
 
    bld_base->image_op(bld_base, &params);
 }
@@ -1801,14 +1848,14 @@ visit_image_size(struct lp_build_nir_context *bld_base,
 {
    struct lp_sampler_size_query_params params = { 0 };
 
-   if (nir_src_is_const(instr->src[0]))
-      params.texture_unit = nir_src_as_int(instr->src[0]);
-   else
-      params.texture_unit_offset = get_src(bld_base, instr->src[0]);
+   sampler_size_params_init_resource(bld_base, &params, instr->src[0]);
+
    params.target = glsl_sampler_to_pipe(nir_intrinsic_image_dim(instr),
                                         nir_intrinsic_image_array(instr));
    params.sizes_out = result;
 
+   params.format = nir_intrinsic_format(instr);
+
    bld_base->image_size(bld_base, &params);
 }
 
@@ -1820,15 +1867,15 @@ visit_image_samples(struct lp_build_nir_context *bld_base,
 {
    struct lp_sampler_size_query_params params = { 0 };
 
-   if (nir_src_is_const(instr->src[0]))
-      params.texture_unit = nir_src_as_int(instr->src[0]);
-   else
-      params.texture_unit_offset = get_src(bld_base, instr->src[0]);
+   sampler_size_params_init_resource(bld_base, &params, instr->src[0]);
+
    params.target = glsl_sampler_to_pipe(nir_intrinsic_image_dim(instr),
                                         nir_intrinsic_image_array(instr));
    params.sizes_out = result;
    params.samples_only = true;
 
+   params.format = nir_intrinsic_format(instr);
+
    bld_base->image_size(bld_base, &params);
 }
 
@@ -2164,19 +2211,25 @@ visit_intrinsic(struct lp_build_nir_context *bld_base,
       visit_ssbo_atomic(bld_base, instr, result);
       break;
    case nir_intrinsic_image_load:
+   case nir_intrinsic_bindless_image_load:
       visit_load_image(bld_base, instr, result);
       break;
    case nir_intrinsic_image_store:
+   case nir_intrinsic_bindless_image_store:
       visit_store_image(bld_base, instr);
       break;
    case nir_intrinsic_image_atomic:
    case nir_intrinsic_image_atomic_swap:
+   case nir_intrinsic_bindless_image_atomic:
+   case nir_intrinsic_bindless_image_atomic_swap:
       visit_atomic_image(bld_base, instr, result);
       break;
    case nir_intrinsic_image_size:
+   case nir_intrinsic_bindless_image_size:
       visit_image_size(bld_base, instr, result);
       break;
    case nir_intrinsic_image_samples:
+   case nir_intrinsic_bindless_image_samples:
       visit_image_samples(bld_base, instr, result);
       break;
    case nir_intrinsic_load_shared:

src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c

@@ -1713,6 +1713,9 @@ static void emit_image_op(struct lp_build_nir_context *bld_base,
       params->image_index_offset = LLVMBuildExtractElement(gallivm->builder, params->image_index_offset,
                                                            first_active_invocation(bld_base), "");
 
+   if (params->resource)
+      params->resource = build_resource_to_scalar(bld_base, params->resource);
+
    bld->image->emit_op(bld->image,
                        bld->bld_base.base.gallivm,
                        params);
@@ -1841,6 +1844,12 @@ static void emit_tex(struct lp_build_nir_context *bld_base,
                                                              first_active_invocation(bld_base), "");
    }
 
+   if (params->texture_resource)
+      params->texture_resource = build_resource_to_scalar(bld_base, params->texture_resource);
+
+   if (params->sampler_resource)
+      params->sampler_resource = build_resource_to_scalar(bld_base, params->sampler_resource);
+
    params->type = bld_base->base.type;
    bld->sampler->emit_tex_sample(bld->sampler,
                                  bld->bld_base.base.gallivm,