diff --git a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c index ccd5a338abd..f98a667ffaa 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c +++ b/src/gallium/drivers/zink/nir_to_spirv/nir_to_spirv.c @@ -2394,6 +2394,7 @@ emit_image_deref_store(struct ntv_context *ctx, nir_intrinsic_instr *intr) static void emit_image_deref_load(struct ntv_context *ctx, nir_intrinsic_instr *intr) { + bool sparse = intr->intrinsic == nir_intrinsic_image_deref_sparse_load; SpvId img_var = get_src(ctx, &intr->src[0]); nir_deref_instr *deref = nir_src_as_deref(intr->src[0]); nir_variable *var = deref->deref_type == nir_deref_type_var ? deref->var : get_var_from_image(ctx, img_var); @@ -2403,9 +2404,10 @@ emit_image_deref_load(struct ntv_context *ctx, nir_intrinsic_instr *intr) SpvId img = spirv_builder_emit_load(&ctx->builder, img_type, img_var); SpvId coord = get_image_coords(ctx, type, &intr->src[1]); SpvId sample = glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS ? get_src(ctx, &intr->src[2]) : 0; + SpvId dest_type = spirv_builder_type_vector(&ctx->builder, base_type, nir_dest_num_components(intr->dest)); SpvId result = spirv_builder_emit_image_read(&ctx->builder, - spirv_builder_type_vector(&ctx->builder, base_type, nir_dest_num_components(intr->dest)), - img, coord, 0, sample, 0); + dest_type, + img, coord, 0, sample, 0, sparse); store_dest(ctx, &intr->dest, result, nir_type_float); } @@ -2703,6 +2705,7 @@ emit_intrinsic(struct ntv_context *ctx, nir_intrinsic_instr *intr) emit_image_deref_store(ctx, intr); break; + case nir_intrinsic_image_deref_sparse_load: case nir_intrinsic_image_deref_load: emit_image_deref_load(ctx, intr); break; @@ -3101,17 +3104,17 @@ emit_tex(struct ntv_context *ctx, nir_tex_instr *tex) spirv_builder_emit_cap(&ctx->builder, SpvCapabilityImageGatherExtended); result = spirv_builder_emit_image_gather(&ctx->builder, dest_type, load, coord, emit_uint_const(ctx, 32, tex->component), - lod, sample, const_offset, offset, dref); + lod, sample, const_offset, offset, dref, tex->is_sparse); } else result = spirv_builder_emit_image_fetch(&ctx->builder, actual_dest_type, - image, coord, lod, sample, const_offset, offset); + image, coord, lod, sample, const_offset, offset, tex->is_sparse); } else { result = spirv_builder_emit_image_sample(&ctx->builder, actual_dest_type, load, coord, proj != 0, lod, bias, dref, dx, dy, - const_offset, offset); + const_offset, offset, tex->is_sparse); } spirv_builder_emit_decoration(&ctx->builder, result, diff --git a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c index d93d47d11e4..0a02476c4a9 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c +++ b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.c @@ -742,6 +742,15 @@ spirv_builder_emit_vote(struct spirv_builder *b, SpvOp op, SpvId src) spirv_builder_const_uint(b, 32, SpvScopeWorkgroup), src); } +static SpvId +sparse_wrap_result_type(struct spirv_builder *b, SpvId result_type) +{ + SpvId types[2]; + types[0] = spirv_builder_type_uint(b, 32); + types[1] = result_type; + return spirv_builder_type_struct(b, types, 2); +} + SpvId spirv_builder_emit_image_sample(struct spirv_builder *b, SpvId result_type, @@ -754,19 +763,34 @@ spirv_builder_emit_image_sample(struct spirv_builder *b, SpvId dx, SpvId dy, SpvId const_offset, - SpvId offset) + SpvId offset, + bool sparse) { SpvId result = spirv_builder_new_id(b); - int opcode = SpvOpImageSampleImplicitLod; int operands = 5; - if (proj) - opcode += SpvOpImageSampleProjImplicitLod - SpvOpImageSampleImplicitLod; - if (lod || (dx && dy)) - opcode += SpvOpImageSampleExplicitLod - SpvOpImageSampleImplicitLod; - if (dref) { - opcode += SpvOpImageSampleDrefImplicitLod - SpvOpImageSampleImplicitLod; - operands++; + int opcode; + if (sparse) { + opcode = SpvOpImageSparseSampleImplicitLod; + if (proj) + opcode += SpvOpImageSparseSampleProjImplicitLod - SpvOpImageSparseSampleImplicitLod; + if (lod || (dx && dy)) + opcode += SpvOpImageSparseSampleExplicitLod - SpvOpImageSparseSampleImplicitLod; + if (dref) { + opcode += SpvOpImageSparseSampleDrefImplicitLod - SpvOpImageSparseSampleImplicitLod; + operands++; + } + result_type = sparse_wrap_result_type(b, result_type); + } else { + opcode = SpvOpImageSampleImplicitLod; + if (proj) + opcode += SpvOpImageSampleProjImplicitLod - SpvOpImageSampleImplicitLod; + if (lod || (dx && dy)) + opcode += SpvOpImageSampleExplicitLod - SpvOpImageSampleImplicitLod; + if (dref) { + opcode += SpvOpImageSampleDrefImplicitLod - SpvOpImageSampleImplicitLod; + operands++; + } } SpvImageOperandsMask operand_mask = SpvImageOperandsMaskNone; @@ -842,13 +866,16 @@ spirv_builder_emit_image_read(struct spirv_builder *b, SpvId coordinate, SpvId lod, SpvId sample, - SpvId offset) + SpvId offset, + bool sparse) { SpvId result = spirv_builder_new_id(b); SpvImageOperandsMask operand_mask = SpvImageOperandsMaskNone; SpvId extra_operands[5]; int num_extra_operands = 1; + if (sparse) + result_type = sparse_wrap_result_type(b, result_type); if (lod) { extra_operands[num_extra_operands++] = lod; operand_mask |= SpvImageOperandsLodMask; @@ -865,7 +892,7 @@ spirv_builder_emit_image_read(struct spirv_builder *b, extra_operands[0] = operand_mask; spirv_buffer_prepare(&b->instructions, b->mem_ctx, 5 + num_extra_operands); - spirv_buffer_emit_word(&b->instructions, SpvOpImageRead | + spirv_buffer_emit_word(&b->instructions, (sparse ? SpvOpImageSparseRead : SpvOpImageRead) | ((5 + num_extra_operands) << 16)); spirv_buffer_emit_word(&b->instructions, result_type); spirv_buffer_emit_word(&b->instructions, result); @@ -923,10 +950,11 @@ spirv_builder_emit_image_gather(struct spirv_builder *b, SpvId sample, SpvId const_offset, SpvId offset, - SpvId dref) + SpvId dref, + bool sparse) { SpvId result = spirv_builder_new_id(b); - SpvId op = SpvOpImageGather; + SpvId op = sparse ? SpvOpImageSparseGather : SpvOpImageGather; SpvImageOperandsMask operand_mask = SpvImageOperandsMaskNone; SpvId extra_operands[4]; @@ -948,7 +976,9 @@ spirv_builder_emit_image_gather(struct spirv_builder *b, operand_mask |= SpvImageOperandsOffsetMask; } if (dref) - op = SpvOpImageDrefGather; + op = sparse ? SpvOpImageSparseDrefGather : SpvOpImageDrefGather; + if (sparse) + result_type = sparse_wrap_result_type(b, result_type); /* finalize num_extra_operands / extra_operands */ extra_operands[0] = operand_mask; @@ -976,7 +1006,8 @@ spirv_builder_emit_image_fetch(struct spirv_builder *b, SpvId lod, SpvId sample, SpvId const_offset, - SpvId offset) + SpvId offset, + bool sparse) { SpvId result = spirv_builder_new_id(b); @@ -999,12 +1030,14 @@ spirv_builder_emit_image_fetch(struct spirv_builder *b, extra_operands[num_extra_operands++] = offset; operand_mask |= SpvImageOperandsOffsetMask; } + if (sparse) + result_type = sparse_wrap_result_type(b, result_type); /* finalize num_extra_operands / extra_operands */ extra_operands[0] = operand_mask; spirv_buffer_prepare(&b->instructions, b->mem_ctx, 5 + num_extra_operands); - spirv_buffer_emit_word(&b->instructions, SpvOpImageFetch | + spirv_buffer_emit_word(&b->instructions, (sparse ? SpvOpImageSparseFetch : SpvOpImageFetch) | ((5 + num_extra_operands) << 16)); spirv_buffer_emit_word(&b->instructions, result_type); spirv_buffer_emit_word(&b->instructions, result); diff --git a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h index d18c101b394..15a27a17f44 100644 --- a/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h +++ b/src/gallium/drivers/zink/nir_to_spirv/spirv_builder.h @@ -283,7 +283,8 @@ spirv_builder_emit_image_sample(struct spirv_builder *b, SpvId dx, SpvId dy, SpvId const_offset, - SpvId offset); + SpvId offset, + bool sparse); SpvId spirv_builder_emit_image(struct spirv_builder *b, SpvId result_type, @@ -303,7 +304,8 @@ spirv_builder_emit_image_read(struct spirv_builder *b, SpvId coordinate, SpvId lod, SpvId sample, - SpvId offset); + SpvId offset, + bool sparse); void spirv_builder_emit_image_write(struct spirv_builder *b, @@ -322,7 +324,8 @@ spirv_builder_emit_image_fetch(struct spirv_builder *b, SpvId lod, SpvId sample, SpvId const_offset, - SpvId offset); + SpvId offset, + bool sparse); SpvId spirv_builder_emit_image_gather(struct spirv_builder *b, SpvId result_type, @@ -333,7 +336,8 @@ spirv_builder_emit_image_gather(struct spirv_builder *b, SpvId sample, SpvId const_offset, SpvId offset, - SpvId dref); + SpvId dref, + bool sparse); SpvId spirv_builder_emit_image_query_size(struct spirv_builder *b,