nir: Support deref instructions in lower_var_copies

Reviewed-by: Caio Marcelo de Oliveira Filho <caio.oliveira@intel.com>
Acked-by: Rob Clark <robdclark@gmail.com>
Acked-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
Acked-by: Dave Airlie <airlied@redhat.com>
Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>

src/compiler/nir/nir.h

@@ -78,6 +78,7 @@ name(const in_type *parent)                             \
 struct nir_function;
 struct nir_shader;
 struct nir_instr;
+struct nir_builder;
 
 
 /**
@@ -2699,6 +2700,8 @@ bool nir_lower_deref_instrs(nir_shader *shader,
                             enum nir_lower_deref_flags flags);
 
 void nir_lower_var_copy_instr(nir_intrinsic_instr *copy, nir_shader *shader);
+void nir_lower_deref_copy_instr(struct nir_builder *b,
+                                nir_intrinsic_instr *copy);
 bool nir_lower_var_copies(nir_shader *shader);
 
 void nir_fixup_deref_modes(nir_shader *shader);

src/compiler/nir/nir_builder.h

@@ -667,6 +667,54 @@ nir_build_deref_for_chain(nir_builder *b, nir_deref_var *deref_var)
    return tail;
 }
 
+/** Returns a deref that follows another but starting from the given parent
+ *
+ * The new deref will be the same type and take the same array or struct index
+ * as the leader deref but it may have a different parent.  This is very
+ * useful for walking deref paths.
+ */
+static inline nir_deref_instr *
+nir_build_deref_follower(nir_builder *b, nir_deref_instr *parent,
+                         nir_deref_instr *leader)
+{
+   /* If the derefs would have the same parent, don't make a new one */
+   assert(leader->parent.is_ssa);
+   if (leader->parent.ssa == &parent->dest.ssa)
+      return leader;
+
+   UNUSED nir_deref_instr *leader_parent = nir_src_as_deref(leader->parent);
+
+   switch (leader->deref_type) {
+   case nir_deref_type_var:
+      unreachable("A var dereference cannot have a parent");
+      break;
+
+   case nir_deref_type_array:
+   case nir_deref_type_array_wildcard:
+      assert(glsl_type_is_matrix(parent->type) ||
+             glsl_type_is_array(parent->type));
+      assert(glsl_get_length(parent->type) ==
+             glsl_get_length(leader_parent->type));
+
+      if (leader->deref_type == nir_deref_type_array) {
+         assert(leader->arr.index.is_ssa);
+         return nir_build_deref_array(b, parent, leader->arr.index.ssa);
+      } else {
+         return nir_build_deref_array_wildcard(b, parent);
+      }
+
+   case nir_deref_type_struct:
+      assert(glsl_type_is_struct(parent->type));
+      assert(glsl_get_length(parent->type) ==
+             glsl_get_length(leader_parent->type));
+
+      return nir_build_deref_struct(b, parent, leader->strct.index);
+
+   default:
+      unreachable("Invalid deref instruction type");
+   }
+}
+
 static inline nir_ssa_def *
 nir_load_reg(nir_builder *build, nir_register *reg)
 {

src/compiler/nir/nir_lower_var_copies.c

@@ -26,6 +26,8 @@
  */
 
 #include "nir.h"
+#include "nir_builder.h"
+#include "nir_deref.h"
 #include "compiler/nir_types.h"
 
 /*
@@ -154,24 +156,111 @@ nir_lower_var_copy_instr(nir_intrinsic_instr *copy, nir_shader *shader)
                         &copy->variables[1]->deref, shader);
 }
 
+static nir_deref_instr *
+build_deref_to_next_wildcard(nir_builder *b,
+                             nir_deref_instr *parent,
+                             nir_deref_instr ***deref_arr)
+{
+   for (; **deref_arr; (*deref_arr)++) {
+      if ((**deref_arr)->deref_type == nir_deref_type_array_wildcard)
+         return parent;
+
+      parent = nir_build_deref_follower(b, parent, **deref_arr);
+   }
+
+   assert(**deref_arr == NULL);
+   *deref_arr = NULL;
+   return parent;
+}
+
+static void
+emit_deref_copy_load_store(nir_builder *b,
+                           nir_deref_instr *dst_deref,
+                           nir_deref_instr **dst_deref_arr,
+                           nir_deref_instr *src_deref,
+                           nir_deref_instr **src_deref_arr)
+{
+   if (dst_deref_arr || src_deref_arr) {
+      assert(dst_deref_arr && src_deref_arr);
+      dst_deref = build_deref_to_next_wildcard(b, dst_deref, &dst_deref_arr);
+      src_deref = build_deref_to_next_wildcard(b, src_deref, &src_deref_arr);
+   }
+
+   if (dst_deref_arr || src_deref_arr) {
+      assert(dst_deref_arr && src_deref_arr);
+      assert((*dst_deref_arr)->deref_type == nir_deref_type_array_wildcard);
+      assert((*src_deref_arr)->deref_type == nir_deref_type_array_wildcard);
+
+      unsigned length = glsl_get_length(src_deref->type);
+      /* The wildcards should represent the same number of elements */
+      assert(length == glsl_get_length(dst_deref->type));
+      assert(length > 0);
+
+      for (unsigned i = 0; i < length; i++) {
+         nir_ssa_def *index = nir_imm_int(b, i);
+         emit_deref_copy_load_store(b,
+                                    nir_build_deref_array(b, dst_deref, index),
+                                    dst_deref_arr + 1,
+                                    nir_build_deref_array(b, src_deref, index),
+                                    src_deref_arr + 1);
+      }
+   } else {
+      assert(dst_deref->type == src_deref->type);
+      assert(glsl_type_is_vector_or_scalar(dst_deref->type));
+
+      nir_store_deref(b, dst_deref, nir_load_deref(b, src_deref), ~0);
+   }
+}
+
+void
+nir_lower_deref_copy_instr(nir_builder *b, nir_intrinsic_instr *copy)
+{
+   /* Unfortunately, there's just no good way to handle wildcards except to
+    * flip the chain around and walk the list from variable to final pointer.
+    */
+   assert(copy->src[0].is_ssa && copy->src[1].is_ssa);
+   nir_deref_instr *dst = nir_instr_as_deref(copy->src[0].ssa->parent_instr);
+   nir_deref_instr *src = nir_instr_as_deref(copy->src[1].ssa->parent_instr);
+
+   nir_deref_path dst_path, src_path;
+   nir_deref_path_init(&dst_path, dst, NULL);
+   nir_deref_path_init(&src_path, src, NULL);
+
+   b->cursor = nir_before_instr(&copy->instr);
+   emit_deref_copy_load_store(b, dst_path.path[0], &dst_path.path[1],
+                                 src_path.path[0], &src_path.path[1]);
+
+   nir_deref_path_finish(&dst_path);
+   nir_deref_path_finish(&src_path);
+}
+
 static bool
 lower_var_copies_impl(nir_function_impl *impl)
 {
    nir_shader *shader = impl->function->shader;
    bool progress = false;
 
+   nir_builder b;
+   nir_builder_init(&b, impl);
+
    nir_foreach_block(block, impl) {
       nir_foreach_instr_safe(instr, block) {
          if (instr->type != nir_instr_type_intrinsic)
             continue;
 
          nir_intrinsic_instr *copy = nir_instr_as_intrinsic(instr);
-         if (copy->intrinsic != nir_intrinsic_copy_var)
+         if (copy->intrinsic == nir_intrinsic_copy_var)
+            nir_lower_var_copy_instr(copy, shader);
+         else if (copy->intrinsic == nir_intrinsic_copy_deref)
+            nir_lower_deref_copy_instr(&b, copy);
+         else
             continue;
 
-         nir_lower_var_copy_instr(copy, shader);
-
          nir_instr_remove(&copy->instr);
+         if (copy->intrinsic == nir_intrinsic_copy_deref) {
+            nir_deref_instr_remove_if_unused(nir_src_as_deref(copy->src[0]));
+            nir_deref_instr_remove_if_unused(nir_src_as_deref(copy->src[1]));
+         }
          progress = true;
          ralloc_free(copy);
       }
@@ -192,8 +281,6 @@ nir_lower_var_copies(nir_shader *shader)
 {
    bool progress = false;
 
-   nir_assert_lowered_derefs(shader, nir_lower_load_store_derefs);
-
    nir_foreach_function(function, shader) {
       if (function->impl)
          progress |= lower_var_copies_impl(function->impl);