Merge branch 'nvk-geom-passthrough' into 'main'

nvk: Add support for VK_NV_geometry_shader_passthrough

See merge request mesa/mesa!38819

docs/features.txt

@@ -690,6 +690,7 @@ Khronos extensions that are not part of any Vulkan version:
   VK_GOOGLE_user_type                                   DONE (anv, hasvk, hk, kk, lvp, nvk, panvk, radv, tu, vn)
   VK_IMG_filter_cubic                                   DONE (tu/a650+, vn)
   VK_NV_compute_shader_derivatives                      DONE (anv, hasvk, nvk, radv, tu/a7xx+, vn)
+  VK_NV_geometry_shader_passthrough                     DONE (nvk)
   VK_NVX_image_view_handle                              DONE (nvk)
   VK_EXT_acquire_drm_display                            DONE (anv, hk, nvk, radv, tu, v3dv, vn)
   VK_VALVE_mutable_descriptor_type                      DONE (anv, hasvk, hk, nvk, radv, tu, vn)

src/compiler/nir/nir.h

@@ -686,6 +686,14 @@ typedef struct nir_variable {
        */
       unsigned aliased_shared_memory : 1;
 
+      /**
+       * Whether the input block that this variable represent is
+       * passed through a shader stage unmodified.
+       *
+       * See SPV_NV_geometry_shader_passthrough for details.
+       */
+      unsigned passthrough_nv : 1;
+
       /**
        * Layout qualifier for gl_FragDepth. See nir_depth_layout.
        *
@@ -5666,6 +5674,7 @@ nir_shader *nir_create_passthrough_gs(const nir_shader_compiler_options *options
                                       bool force_line_strip_out,
                                       bool passthrough_prim_id);
 
+bool nir_lower_passthrough_gs(nir_shader *shader);
 bool nir_lower_fragcolor(nir_shader *shader, unsigned max_cbufs);
 bool nir_lower_fragcoord_wtrans(nir_shader *shader);
 bool nir_opt_frag_coord_to_pixel_coord(nir_shader *shader);

src/compiler/nir/nir_gather_info.c

@@ -1125,6 +1125,15 @@ nir_shader_gather_info(nir_shader *shader, nir_function_impl *entrypoint)
       }
    }
 
+   if (shader->info.stage == MESA_SHADER_GEOMETRY) {
+      nir_foreach_shader_in_variable(var, shader) {
+         if (var->data.passthrough_nv) {
+            shader->info.gs.uses_passthrough_nv = true;
+            break;
+         }
+      }
+   }
+
    shader->info.ray_queries = 0;
    nir_foreach_variable_in_shader(var, shader) {
       if (!var->data.ray_query)

src/compiler/nir/nir_passthrough_gs.c

@@ -295,3 +295,96 @@ nir_create_passthrough_gs(const nir_shader_compiler_options *options,
 
    return nir;
 }
+
+static bool
+should_lower_passthrough_gs(nir_shader *nir)
+{
+   nir_function_impl *entrypoint = nir_shader_get_entrypoint(nir);
+   nir_shader_gather_info(nir, entrypoint);
+
+   if (!nir->info.gs.uses_passthrough_nv)
+      return false;
+
+   if (nir->info.gs.uses_end_primitive || nir->info.gs.active_stream_mask != 0 ||
+       nir->info.writes_memory || nir->info.has_transform_feedback_varyings ||
+       nir->info.gs.invocations != 1 || nir->info.gs.vertices_in != nir->info.gs.vertices_out)
+      return true;
+
+   if (BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_INVOCATION_ID) ||
+       BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_PRIMITIVE_ID))
+      return true;
+
+   nir_foreach_block(block, entrypoint) {
+      nir_foreach_instr(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+
+         nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+
+         if (intr->intrinsic == nir_intrinsic_emit_vertex)
+            return true;
+      }
+   }
+
+   return false;
+}
+
+bool
+nir_lower_passthrough_gs(nir_shader *nir)
+{
+   if (nir->info.stage != MESA_SHADER_GEOMETRY)
+      return false;
+
+   if (!should_lower_passthrough_gs(nir))
+      return false;
+
+   nir_variable *in_vars[VARYING_SLOT_MAX * 4];
+   nir_variable *out_vars[VARYING_SLOT_MAX * 4];
+   unsigned num_inputs = 0, num_outputs = 0;
+   nir_foreach_shader_in_variable_safe(var, nir) {
+      if (!var->data.passthrough_nv)
+         continue;
+
+      in_vars[num_inputs++] = var;
+      var->data.passthrough_nv = false;
+
+      nir_variable *out = nir_variable_clone(var, nir);
+
+      char name[100];
+      if (var->name) {
+         snprintf(name, sizeof(name), "out_%s", var->name);
+         nir_variable_set_name(nir, out, name);
+      }
+
+      out->data.mode = nir_var_shader_out;
+      out->type = glsl_get_array_element(var->type);
+      nir_shader_add_variable(nir, out);
+      out_vars[num_outputs++] = out;
+   }
+
+   if (num_inputs == 0)
+      return false;
+
+   nir_function_impl *entrypoint = nir_shader_get_entrypoint(nir);
+   nir_builder b = nir_builder_at(nir_before_impl(entrypoint));
+
+   for (unsigned i = 0; i < nir->info.gs.vertices_out; i++) {
+      for (unsigned j = 0; j < num_inputs; ++j) {
+         nir_deref_instr *value = nir_build_deref_var(&b, in_vars[j]);
+         if (in_vars[j]->data.location != VARYING_SLOT_PRIMITIVE_ID)
+            value = nir_build_deref_array(&b, value, nir_imm_int(&b, i));
+
+         copy_vars(&b, nir_build_deref_var(&b, out_vars[j]), value);
+      }
+
+      nir_emit_vertex(&b, 0);
+   }
+
+   nir_end_primitive(&b, 0);
+
+   nir_progress(true, entrypoint, nir_metadata_none);
+   nir->info.gs.uses_passthrough_nv = false;
+
+   nir_shader_gather_info(nir, entrypoint);
+   return true;
+}

src/compiler/nir/nir_print.c

@@ -914,9 +914,10 @@ print_var_decl(nir_variable *var, print_state *state)
    const char *const per_primitive = (var->data.per_primitive) ? "per_primitive " : "";
    const char *const ray_query = (var->data.ray_query) ? "ray_query " : "";
    const char *const fb_fetch = var->data.fb_fetch_output ? "fb_fetch_output " : "";
-   fprintf(fp, "%s%s%s%s%s%s%s%s%s%s %s ",
+   const char *const passthrough_nv = var->data.passthrough_nv ? "passthrough_nv " : "";
+   fprintf(fp, "%s%s%s%s%s%s%s%s%s%s%s %s ",
            bindless, cent, samp, patch, inv, per_view, per_primitive,
-           ray_query, fb_fetch,
+           ray_query, fb_fetch, passthrough_nv,
            get_variable_mode_str(var->data.mode, false),
            glsl_interp_mode_name(var->data.interpolation));
 
@@ -2864,6 +2865,7 @@ print_shader_info(const struct shader_info *info, FILE *fp)
       fprintf(fp, "invocations: %u\n", info->gs.invocations);
       fprintf(fp, "vertices_in: %u\n", info->gs.vertices_in);
       print_nz_bool(fp, "uses_end_primitive", info->gs.uses_end_primitive);
+      print_nz_bool(fp, "uses_passthrough_nv", info->gs.uses_passthrough_nv);
       fprintf(fp, "active_stream_mask: 0x%02x\n", info->gs.active_stream_mask);
       break;
 

src/compiler/shader_info.h

@@ -389,6 +389,9 @@ typedef struct shader_info {
 
          /** The streams used in this shaders (max. 4) */
          uint8_t active_stream_mask:4;
+
+         /** Whether or not this shader uses PassthroughNV */
+         bool uses_passthrough_nv:1;
       } gs;
 
       struct {

src/compiler/spirv/spirv_to_nir.c

@@ -104,6 +104,7 @@ static const struct spirv_capabilities implemented_capabilities = {
    .GenericPointer = true,
    .Geometry = true,
    .GeometryPointSize = true,
+   .GeometryShaderPassthroughNV = true,
    .GeometryStreams = true,
    .GroupNonUniform = true,
    .GroupNonUniformArithmetic = true,

src/compiler/spirv/vtn_variables.c

@@ -1561,6 +1561,12 @@ apply_var_decoration(struct vtn_builder *b,
                   "NodeMaxPayloadsAMDX decoration only allowed in compute shaders");
       break;
 
+   case SpvDecorationPassthroughNV:
+      vtn_fail_if(b->shader->info.stage != MESA_SHADER_GEOMETRY,
+                  "PassthroughNV decoration only allowed in Geometry shaders");
+      var_data->passthrough_nv = true;
+      break;
+
    default:
       vtn_fail_with_decoration("Unhandled decoration", dec->decoration);
    }

src/nouveau/compiler/nak.h

@@ -195,6 +195,12 @@ struct nak_shader_info {
          uint8_t _pad[9];
       } ts;
 
+      struct {
+         bool passthrough_enable;
+
+         uint8_t _pad[11];
+      } gs;
+
       /* Used to initialize the union for other stages */
       uint8_t _pad[12];
    };

src/nouveau/compiler/nak/api.rs

@@ -308,6 +308,14 @@ impl ShaderBin {
                         },
                     }
                 }
+                ShaderStageInfo::Geometry(gs_info) => {
+                    nak_shader_info__bindgen_ty_1 {
+                        gs: nak_shader_info__bindgen_ty_1__bindgen_ty_4 {
+                            passthrough_enable: gs_info.passthrough_enable,
+                            _pad: Default::default(),
+                        },
+                    }
+                }
                 _ => nak_shader_info__bindgen_ty_1 {
                     _pad: Default::default(),
                 },

src/nouveau/compiler/nak/from_nir.rs

@@ -73,8 +73,7 @@ fn init_info_from_nir(nak: &nak_compiler, nir: &nir_shader) -> ShaderInfo {
                 };
 
                 ShaderStageInfo::Geometry(GeometryShaderInfo {
-                    // TODO: Should be set if VK_NV_geometry_shader_passthrough is in use.
-                    passthrough_enable: false,
+                    passthrough_enable: info_gs.uses_passthrough_nv(),
                     stream_out_mask: info_gs.active_stream_mask(),
                     threads_per_input_primitive: info_gs.invocations,
                     output_topology: output_topology,

src/nouveau/compiler/nak/sph.rs

@@ -553,13 +553,19 @@ pub fn encode_header(
             sph.set_does_interlock(stage.does_interlock);
         }
         ShaderStageInfo::Geometry(stage) => {
-            sph.set_gs_passthrough_enable(stage.passthrough_enable);
+            /* In case of passthrough, the hardware expects max_output_vertex_count to be 1 */
+            if stage.passthrough_enable {
+                sph.set_gs_passthrough_enable(true);
+                sph.set_max_output_vertex_count(1);
+            } else {
+                sph.set_max_output_vertex_count(stage.max_output_vertex_count);
+            }
+
             sph.set_stream_out_mask(stage.stream_out_mask);
             sph.set_threads_per_input_primitive(
                 stage.threads_per_input_primitive,
             );
             sph.set_output_topology(stage.output_topology);
-            sph.set_max_output_vertex_count(stage.max_output_vertex_count);
         }
         ShaderStageInfo::TessellationInit(stage) => {
             sph.set_per_patch_attribute_count(stage.per_patch_attribute_count);

src/nouveau/compiler/nak_nir.c

@@ -331,6 +331,9 @@ nak_preprocess_nir(nir_shader *nir, const struct nak_compiler *nak)
 
    nir_validate_ssa_dominance(nir, "before nak_preprocess_nir");
 
+   if (nir->info.stage == MESA_SHADER_GEOMETRY)
+      NIR_PASS(_, nir, nir_lower_passthrough_gs);
+
    OPT(nir, nir_lower_io_vars_to_temporaries,
        nir_shader_get_entrypoint(nir),
        nir_var_shader_out);

src/nouveau/headers/nv_push.h

@@ -70,6 +70,7 @@ void vk_push_print(FILE *fp, const struct nv_push *push,
 #define SUBC_NVB097 0
 #define SUBC_NVB197 0
 #define SUBC_NVC097 0
+#define SUBC_NVC197 0
 #define SUBC_NVC397 0
 #define SUBC_NVC597 0
 #define SUBC_NVC797 0

src/nouveau/vulkan/nvk_physical_device.c

@@ -302,6 +302,7 @@ nvk_get_device_extensions(const struct nvk_instance *instance,
       .GOOGLE_user_type = true,
       .MESA_image_alignment_control = true,
       .NV_compute_shader_derivatives = info->cls_eng3d >= TURING_A,
+      .NV_geometry_shader_passthrough = info->cls_eng3d >= MAXWELL_B,
       .NV_shader_sm_builtins = true,
       .NVX_image_view_handle = info->cls_eng3d >= MAXWELL_A, /* needs true bindless descriptors */
       .VALVE_mutable_descriptor_type = true,

src/nouveau/vulkan/nvk_shader.c

@@ -31,6 +31,7 @@
 #include "clc597.h"
 #include "nv_push_cl9097.h"
 #include "nv_push_clb197.h"
+#include "nv_push_clc197.h"
 #include "nv_push_clc397.h"
 #include "nv_push_clc797.h"
 
@@ -634,6 +635,9 @@ nvk_max_shader_push_dw(const struct nvk_physical_device *pdev,
    if (stage == MESA_SHADER_TESS_EVAL)
       max_dw_count += 2;
 
+   if (stage == MESA_SHADER_GEOMETRY && pdev->info.cls_eng3d >= PASCAL_B)
+      max_dw_count += 2;
+
    if (stage == MESA_SHADER_FRAGMENT)
       max_dw_count += 13;
 
@@ -694,6 +698,13 @@ nvk_shader_fill_push(struct nvk_device *dev,
                                            shader->info.ts.prims));
    }
 
+   /* On Pascal B and later, we need to specify if GS passthrough is enabled */
+   if (shader->info.stage == MESA_SHADER_GEOMETRY &&
+       pdev->info.cls_eng3d >= PASCAL_B) {
+      max_dw_count += 2;
+      P_IMMD(p, NVC197, SET_GS_MODE, shader->info.gs.passthrough_enable);
+   }
+
    if (shader->info.stage == MESA_SHADER_FRAGMENT) {
       max_dw_count += 13;
 

src/vulkan/runtime/vk_nir.c

@@ -115,6 +115,15 @@ nir_vk_is_not_xfb_output(nir_variable *var, void *data)
    }
 }
 
+static bool
+is_not_xfb_output_or_passthrough(nir_variable *var, void *data)
+{
+   if (var->data.mode == nir_var_shader_in && var->data.passthrough_nv)
+      return false;
+
+   return nir_vk_is_not_xfb_output(var, data);
+}
+
 nir_shader *
 vk_spirv_to_nir(struct vk_device *device,
                 const uint32_t *spirv_data, size_t spirv_size_B,
@@ -184,7 +193,7 @@ vk_spirv_to_nir(struct vk_device *device,
    NIR_PASS(_, nir, nir_split_per_member_structs);
 
    nir_remove_dead_variables_options dead_vars_opts = {
-      .can_remove_var = nir_vk_is_not_xfb_output,
+      .can_remove_var = is_not_xfb_output_or_passthrough,
    };
    NIR_PASS(_, nir, nir_remove_dead_variables,
               nir_var_shader_in | nir_var_shader_out | nir_var_system_value |