nir,spirv: add sparse texture fetches

Like SPIR-V and GL_ARB_sparse_texture2, these return a residency code. It
is placed in the destination after the rest of the result. If it's zero,
then the texel is resident. Otherwise, it's not resident.

Besides the larger destination and the residency code, sparse fetches
work the same as normal fetches.

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Jason Ekstrand <jason@jlekstrand.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/7774>

src/compiler/nir/nir.h

@@ -2036,6 +2036,12 @@ typedef struct {
     */
    bool is_new_style_shadow;
 
+   /**
+    * If this texture instruction should return a sparse residency code. The
+    * code is in the last component of the result.
+    */
+   bool is_sparse;
+
    /* gather component selector */
    unsigned component : 2;
 
@@ -2096,7 +2102,7 @@ nir_tex_instr_need_sampler(const nir_tex_instr *instr)
 }
 
 static inline unsigned
-nir_tex_instr_dest_size(const nir_tex_instr *instr)
+nir_tex_instr_result_size(const nir_tex_instr *instr)
 {
    switch (instr->op) {
    case nir_texop_txs: {
@@ -2142,6 +2148,13 @@ nir_tex_instr_dest_size(const nir_tex_instr *instr)
    }
 }
 
+static inline unsigned
+nir_tex_instr_dest_size(const nir_tex_instr *instr)
+{
+   /* One more component is needed for the residency code. */
+   return nir_tex_instr_result_size(instr) + instr->is_sparse;
+}
+
 /* Returns true if this texture operation queries something about the texture
  * rather than actually sampling it.
  */

src/compiler/nir/nir_clone.c

@@ -413,6 +413,7 @@ clone_tex(clone_state *state, const nir_tex_instr *tex)
    ntex->is_array = tex->is_array;
    ntex->is_shadow = tex->is_shadow;
    ntex->is_new_style_shadow = tex->is_new_style_shadow;
+   ntex->is_sparse = tex->is_sparse;
    ntex->component = tex->component;
    memcpy(ntex->tg4_offsets, tex->tg4_offsets, sizeof(tex->tg4_offsets));
 

src/compiler/nir/nir_print.c

@@ -1218,6 +1218,10 @@ print_tex_instr(nir_tex_instr *instr, print_state *state)
    if (instr->sampler_non_uniform) {
       fprintf(fp, ", sampler non-uniform");
    }
+
+   if (instr->is_sparse) {
+      fprintf(fp, ", sparse");
+   }
 }
 
 static void

src/compiler/nir/nir_serialize.c

@@ -1456,10 +1456,11 @@ union packed_tex_data {
       unsigned is_array:1;
       unsigned is_shadow:1;
       unsigned is_new_style_shadow:1;
+      unsigned is_sparse:1;
       unsigned component:2;
       unsigned texture_non_uniform:1;
       unsigned sampler_non_uniform:1;
-      unsigned unused:8; /* Mark unused for valgrind. */
+      unsigned unused:7; /* Mark unused for valgrind. */
    } u;
 };
 
@@ -1491,6 +1492,7 @@ write_tex(write_ctx *ctx, const nir_tex_instr *tex)
       .u.is_array = tex->is_array,
       .u.is_shadow = tex->is_shadow,
       .u.is_new_style_shadow = tex->is_new_style_shadow,
+      .u.is_sparse = tex->is_sparse,
       .u.component = tex->component,
       .u.texture_non_uniform = tex->texture_non_uniform,
       .u.sampler_non_uniform = tex->sampler_non_uniform,
@@ -1526,6 +1528,7 @@ read_tex(read_ctx *ctx, union packed_instr header)
    tex->is_array = packed.u.is_array;
    tex->is_shadow = packed.u.is_shadow;
    tex->is_new_style_shadow = packed.u.is_new_style_shadow;
+   tex->is_sparse = packed.u.is_sparse;
    tex->component = packed.u.component;
    tex->texture_non_uniform = packed.u.texture_non_uniform;
    tex->sampler_non_uniform = packed.u.sampler_non_uniform;

src/compiler/spirv/spirv_to_nir.c

@@ -2530,8 +2530,6 @@ static void
 vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
                    const uint32_t *w, unsigned count)
 {
-   struct vtn_type *ret_type = vtn_get_type(b, w[1]);
-
    if (opcode == SpvOpSampledImage) {
       struct vtn_sampled_image si = {
          .image = vtn_get_image(b, w[3], NULL),
@@ -2575,20 +2573,25 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
    nir_texop texop;
    switch (opcode) {
    case SpvOpImageSampleImplicitLod:
+   case SpvOpImageSparseSampleImplicitLod:
    case SpvOpImageSampleDrefImplicitLod:
+   case SpvOpImageSparseSampleDrefImplicitLod:
    case SpvOpImageSampleProjImplicitLod:
    case SpvOpImageSampleProjDrefImplicitLod:
       texop = nir_texop_tex;
       break;
 
    case SpvOpImageSampleExplicitLod:
+   case SpvOpImageSparseSampleExplicitLod:
    case SpvOpImageSampleDrefExplicitLod:
+   case SpvOpImageSparseSampleDrefExplicitLod:
    case SpvOpImageSampleProjExplicitLod:
    case SpvOpImageSampleProjDrefExplicitLod:
       texop = nir_texop_txl;
       break;
 
    case SpvOpImageFetch:
+   case SpvOpImageSparseFetch:
       if (sampler_dim == GLSL_SAMPLER_DIM_MS) {
          texop = nir_texop_txf_ms;
       } else {
@@ -2597,7 +2600,9 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
       break;
 
    case SpvOpImageGather:
+   case SpvOpImageSparseGather:
    case SpvOpImageDrefGather:
+   case SpvOpImageSparseDrefGather:
       texop = nir_texop_tg4;
       break;
 
@@ -2681,16 +2686,23 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
    unsigned coord_components;
    switch (opcode) {
    case SpvOpImageSampleImplicitLod:
+   case SpvOpImageSparseSampleImplicitLod:
    case SpvOpImageSampleExplicitLod:
+   case SpvOpImageSparseSampleExplicitLod:
    case SpvOpImageSampleDrefImplicitLod:
+   case SpvOpImageSparseSampleDrefImplicitLod:
    case SpvOpImageSampleDrefExplicitLod:
+   case SpvOpImageSparseSampleDrefExplicitLod:
    case SpvOpImageSampleProjImplicitLod:
    case SpvOpImageSampleProjExplicitLod:
    case SpvOpImageSampleProjDrefImplicitLod:
    case SpvOpImageSampleProjDrefExplicitLod:
    case SpvOpImageFetch:
+   case SpvOpImageSparseFetch:
    case SpvOpImageGather:
+   case SpvOpImageSparseGather:
    case SpvOpImageDrefGather:
+   case SpvOpImageSparseDrefGather:
    case SpvOpImageQueryLod:
    case SpvOpFragmentFetchAMD:
    case SpvOpFragmentMaskFetchAMD: {
@@ -2747,16 +2759,20 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
    unsigned gather_component = 0;
    switch (opcode) {
    case SpvOpImageSampleDrefImplicitLod:
+   case SpvOpImageSparseSampleDrefImplicitLod:
    case SpvOpImageSampleDrefExplicitLod:
+   case SpvOpImageSparseSampleDrefExplicitLod:
    case SpvOpImageSampleProjDrefImplicitLod:
    case SpvOpImageSampleProjDrefExplicitLod:
    case SpvOpImageDrefGather:
+   case SpvOpImageSparseDrefGather:
       /* These all have an explicit depth value as their next source */
       is_shadow = true;
       (*p++) = vtn_tex_src(b, w[idx++], nir_tex_src_comparator);
       break;
 
    case SpvOpImageGather:
+   case SpvOpImageSparseGather:
       /* This has a component as its next source */
       gather_component = vtn_constant_uint(b, w[idx++]);
       break;
@@ -2765,6 +2781,21 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
       break;
    }
 
+   bool is_sparse = false;
+   switch (opcode) {
+   case SpvOpImageSparseSampleImplicitLod:
+   case SpvOpImageSparseSampleExplicitLod:
+   case SpvOpImageSparseSampleDrefImplicitLod:
+   case SpvOpImageSparseSampleDrefExplicitLod:
+   case SpvOpImageSparseFetch:
+   case SpvOpImageSparseGather:
+   case SpvOpImageSparseDrefGather:
+      is_sparse = true;
+      break;
+   default:
+      break;
+   }
+
    /* For OpImageQuerySizeLod, we always have an LOD */
    if (opcode == SpvOpImageQuerySizeLod)
       (*p++) = vtn_tex_src(b, w[idx++], nir_tex_src_lod);
@@ -2848,6 +2879,14 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
       }
    }
 
+   struct vtn_type *ret_type = vtn_get_type(b, w[1]);
+   struct vtn_type *struct_type = NULL;
+   if (is_sparse) {
+      vtn_assert(glsl_type_is_struct_or_ifc(ret_type->type));
+      struct_type = ret_type;
+      ret_type = struct_type->members[1];
+   }
+
    nir_tex_instr *instr = nir_tex_instr_create(b->shader, p - srcs);
    instr->op = texop;
 
@@ -2857,6 +2896,7 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
    instr->sampler_dim = sampler_dim;
    instr->is_array = is_array;
    instr->is_shadow = is_shadow;
+   instr->is_sparse = is_sparse;
    instr->is_new_style_shadow =
       is_shadow && glsl_get_components(ret_type->type) == 1;
    instr->component = gather_component;
@@ -2913,7 +2953,7 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
                      nir_tex_instr_dest_size(instr), 32, NULL);
 
    vtn_assert(glsl_get_vector_elements(ret_type->type) ==
-              nir_tex_instr_dest_size(instr));
+              nir_tex_instr_result_size(instr));
 
    if (gather_offsets) {
       vtn_fail_if(gather_offsets->type->base_type != vtn_base_type_array ||
@@ -2947,8 +2987,17 @@ vtn_handle_texture(struct vtn_builder *b, SpvOp opcode,
 
    nir_builder_instr_insert(&b->nb, &instr->instr);
 
+   if (is_sparse) {
+      struct vtn_ssa_value *dest = vtn_create_ssa_value(b, struct_type->type);
+      unsigned result_size = glsl_get_vector_elements(ret_type->type);
+      dest->elems[0]->def = nir_channel(&b->nb, &instr->dest.ssa, result_size);
+      dest->elems[1]->def = nir_channels(&b->nb, &instr->dest.ssa,
+                                         BITFIELD_MASK(result_size));
+      vtn_push_ssa_value(b, w[2], dest);
+   } else {
       vtn_push_nir_ssa(b, w[2], &instr->dest.ssa);
    }
+}
 
 static void
 fill_common_atomic_sources(struct vtn_builder *b, SpvOp opcode,
@@ -5208,16 +5257,23 @@ vtn_handle_body_instruction(struct vtn_builder *b, SpvOp opcode,
    case SpvOpSampledImage:
    case SpvOpImage:
    case SpvOpImageSampleImplicitLod:
+   case SpvOpImageSparseSampleImplicitLod:
    case SpvOpImageSampleExplicitLod:
+   case SpvOpImageSparseSampleExplicitLod:
    case SpvOpImageSampleDrefImplicitLod:
+   case SpvOpImageSparseSampleDrefImplicitLod:
    case SpvOpImageSampleDrefExplicitLod:
+   case SpvOpImageSparseSampleDrefExplicitLod:
    case SpvOpImageSampleProjImplicitLod:
    case SpvOpImageSampleProjExplicitLod:
    case SpvOpImageSampleProjDrefImplicitLod:
    case SpvOpImageSampleProjDrefExplicitLod:
    case SpvOpImageFetch:
+   case SpvOpImageSparseFetch:
    case SpvOpImageGather:
+   case SpvOpImageSparseGather:
    case SpvOpImageDrefGather:
+   case SpvOpImageSparseDrefGather:
    case SpvOpImageQueryLod:
    case SpvOpImageQueryLevels:
    case SpvOpImageQuerySamples: