diff --git a/src/nouveau/vulkan/nvk_nir_lower_descriptors.c b/src/nouveau/vulkan/nvk_nir_lower_descriptors.c index 896ba563439..7033408128d 100644 --- a/src/nouveau/vulkan/nvk_nir_lower_descriptors.c +++ b/src/nouveau/vulkan/nvk_nir_lower_descriptors.c @@ -942,6 +942,137 @@ lower_msaa_image_intrin(nir_builder *b, nir_intrinsic_instr *intrin, nir_intrinsic_set_image_dim(intrin, GLSL_SAMPLER_DIM_2D); } +static bool +is_edb_buffer_view(nir_deref_instr *deref, + const struct lower_descriptors_ctx *ctx) +{ + if (glsl_get_sampler_dim(deref->type) != GLSL_SAMPLER_DIM_BUF) + return false; + + nir_variable *var = nir_deref_instr_get_variable(deref); + uint8_t set = var->data.descriptor_set; + + return ctx->set_layouts[set]->flags & + VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; +} + +static nir_def * +edb_buffer_view_is_null(nir_builder *b, nir_def *desc) +{ + assert(desc->num_components == 4); + nir_def *index = nir_channel(b, desc, 0); + return nir_ieq_imm(b, index, 0); +} + +static nir_def * +edb_buffer_view_offset_el(nir_builder *b, nir_def *desc) +{ + assert(desc->num_components == 4); + return nir_channel(b, desc, 1); +} + +static nir_def * +edb_buffer_view_size_el(nir_builder *b, nir_def *desc) +{ + assert(desc->num_components == 4); + return nir_channel(b, desc, 2); +} + +static nir_def * +edb_buffer_view_oob_alpha(nir_builder *b, nir_def *desc) +{ + assert(desc->num_components == 4); + return nir_channel(b, desc, 3); +} + +static nir_def * +edb_buffer_view_coord_is_in_bounds(nir_builder *b, nir_def *desc, + nir_def *coord) +{ + assert(desc->num_components == 4); + return nir_ult(b, coord, edb_buffer_view_size_el(b, desc)); +} + +static nir_def * +edb_buffer_view_index(nir_builder *b, nir_def *desc, nir_def *in_bounds) +{ + assert(desc->num_components == 4); + nir_def *index = nir_channel(b, desc, 0); + + /* Use the NULL descriptor for OOB access */ + return nir_bcsel(b, in_bounds, index, nir_imm_int(b, 0)); +} + +static nir_def * +adjust_edb_buffer_view_coord(nir_builder *b, nir_def *desc, nir_def *coord) +{ + return nir_iadd(b, coord, edb_buffer_view_offset_el(b, desc)); +} + +static nir_def * +fixup_edb_buffer_view_result(nir_builder *b, nir_def *desc, nir_def *in_bounds, + nir_def *res, nir_alu_type dest_type) +{ + if (res->num_components < 4) + return res; + + nir_def *is_null = edb_buffer_view_is_null(b, desc); + nir_def *oob_alpha = edb_buffer_view_oob_alpha(b, desc); + + nir_def *a = nir_channel(b, res, 3); + a = nir_bcsel(b, nir_ior(b, in_bounds, is_null), a, oob_alpha); + return nir_vector_insert_imm(b, res, a, 3); +} + +static void +lower_edb_buffer_image_intrin(nir_builder *b, nir_intrinsic_instr *intrin, + const struct lower_descriptors_ctx *ctx) +{ + assert(nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_BUF); + + b->cursor = nir_before_instr(&intrin->instr); + nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); + nir_def *desc = load_resource_deref_desc(b, 4, 32, deref, 0, ctx); + + switch (intrin->intrinsic) { + case nir_intrinsic_image_deref_load: + case nir_intrinsic_image_deref_store: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: { + nir_def *pos = intrin->src[1].ssa; + nir_def *x = nir_channel(b, pos, 0); + + nir_def *in_bounds = edb_buffer_view_coord_is_in_bounds(b, desc, x); + nir_def *index = edb_buffer_view_index(b, desc, in_bounds); + + nir_def *new_x = adjust_edb_buffer_view_coord(b, desc, x); + pos = nir_vector_insert_imm(b, pos, new_x, 0); + nir_src_rewrite(&intrin->src[1], pos); + + if (intrin->intrinsic == nir_intrinsic_image_deref_load) { + b->cursor = nir_after_instr(&intrin->instr); + nir_def *res = &intrin->def; + res = fixup_edb_buffer_view_result(b, desc, in_bounds, res, + nir_intrinsic_dest_type(intrin)); + nir_def_rewrite_uses_after(&intrin->def, res, res->parent_instr); + } + + nir_rewrite_image_intrinsic(intrin, index, true); + break; + } + + case nir_intrinsic_image_deref_size: { + assert(intrin->def.num_components == 1); + nir_def *size_el = nir_channel(b, desc, 2); + nir_def_rewrite_uses(&intrin->def, size_el); + break; + } + + default: + unreachable("Unknown image intrinsic"); + } +} + static bool lower_image_intrin(nir_builder *b, nir_intrinsic_instr *intrin, const struct lower_descriptors_ctx *ctx) @@ -950,6 +1081,8 @@ lower_image_intrin(nir_builder *b, nir_intrinsic_instr *intrin, if (glsl_get_sampler_dim(deref->type) == GLSL_SAMPLER_DIM_MS) { lower_msaa_image_intrin(b, intrin, ctx); + } else if (is_edb_buffer_view(deref, ctx)) { + lower_edb_buffer_image_intrin(b, intrin, ctx); } else { b->cursor = nir_before_instr(&intrin->instr); nir_def *desc = load_resource_deref_desc(b, 1, 32, deref, 0, ctx); @@ -1044,12 +1177,64 @@ try_lower_intrin(nir_builder *b, nir_intrinsic_instr *intrin, } } +static void +lower_edb_buffer_tex_instr(nir_builder *b, nir_tex_instr *tex, + const struct lower_descriptors_ctx *ctx) +{ + assert(tex->sampler_dim == GLSL_SAMPLER_DIM_BUF); + + b->cursor = nir_before_instr(&tex->instr); + + const int texture_src_idx = + nir_tex_instr_src_index(tex, nir_tex_src_texture_deref); + nir_deref_instr *texture = nir_src_as_deref(tex->src[texture_src_idx].src); + + nir_def *plane_ssa = nir_steal_tex_src(tex, nir_tex_src_plane); + ASSERTED const uint32_t plane = + plane_ssa ? nir_src_as_uint(nir_src_for_ssa(plane_ssa)) : 0; + assert(plane == 0); + + nir_def *desc = load_resource_deref_desc(b, 4, 32, texture, 0, ctx); + + switch (tex->op) { + case nir_texop_txf: { + const int coord_src_idx = nir_tex_instr_src_index(tex, nir_tex_src_coord); + assert(coord_src_idx >= 0); + nir_def *coord = tex->src[coord_src_idx].src.ssa; + + nir_def *in_bounds = edb_buffer_view_coord_is_in_bounds(b, desc, coord); + + nir_def *index = edb_buffer_view_index(b, desc, in_bounds); + nir_src_rewrite(&tex->src[texture_src_idx].src, index); + tex->src[texture_src_idx].src_type = nir_tex_src_texture_handle; + + nir_def *new_coord = adjust_edb_buffer_view_coord(b, desc, coord); + nir_src_rewrite(&tex->src[coord_src_idx].src, new_coord); + + b->cursor = nir_after_instr(&tex->instr); + nir_def *res = &tex->def; + res = fixup_edb_buffer_view_result(b, desc, in_bounds, + res, tex->dest_type); + nir_def_rewrite_uses_after(&tex->def, res, res->parent_instr); + break; + } + + case nir_texop_txs: { + assert(tex->def.num_components == 1); + nir_def *size_el = edb_buffer_view_size_el(b, desc); + nir_def_rewrite_uses(&tex->def, size_el); + break; + } + + default: + unreachable("Invalid buffer texture op"); + } +} + static bool lower_tex(nir_builder *b, nir_tex_instr *tex, const struct lower_descriptors_ctx *ctx) { - b->cursor = nir_before_instr(&tex->instr); - const int texture_src_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_deref); const int sampler_src_idx = @@ -1064,6 +1249,13 @@ lower_tex(nir_builder *b, nir_tex_instr *tex, nir_src_as_deref(tex->src[sampler_src_idx].src); assert(texture); + if (is_edb_buffer_view(texture, ctx)) { + lower_edb_buffer_tex_instr(b, tex, ctx); + return true; + } + + b->cursor = nir_before_instr(&tex->instr); + nir_def *plane_ssa = nir_steal_tex_src(tex, nir_tex_src_plane); const uint32_t plane = plane_ssa ? nir_src_as_uint(nir_src_for_ssa(plane_ssa)) : 0;