pan/fb: Add a concept of resolve ops and resolve shaders

The resolve sits between the render pass and the write at the end.  This
adds the resolve ops themselves, support in the shader builder for
building the post-frame shaders to handle them, and an optimization pass
that attempts to fold the resolve op into the store and potentially
eliminate the post-frame shader.

Reviewed-by: Lars-Ivar Hesselberg Simonsen <lars-ivar.simonsen@arm.com>
Acked-by: Boris Brezillon <boris.brezillon@collabora.com>
Acked-by: Eric R. Smith <eric.smith@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/39759>

src/panfrost/lib/pan_fb.h

@@ -309,6 +309,82 @@ pan_fb_has_image_load(const struct pan_fb_load *load, bool include_border)
    return false;
 }
 
+/** Describes a resolve operation
+ *
+ * This describes the source data for the resolve operation.  The way in which
+ * MSAA is handled in the source data is defined by the pan_fb_msaa_copy_op.
+ */
+enum ENUM_PACKED pan_fb_resolve_op {
+   /** Do nothing
+    *
+    * In this mode, the framebuffer is left unaffected by the resolve.  If
+    * necessary (sometimes required for Z/S), the old value will be read and
+    * written back to ensure the final value is the same.
+    */
+   PAN_FB_RESOLVE_NONE = 0,
+
+   /** Load from the specified image view */
+   PAN_FB_RESOLVE_IMAGE,
+
+   /** Load from color render target 0 */
+   PAN_FB_RESOLVE_RT_0,
+   PAN_FB_RESOLVE_RT_1,
+   PAN_FB_RESOLVE_RT_2,
+   PAN_FB_RESOLVE_RT_3,
+   PAN_FB_RESOLVE_RT_4,
+   PAN_FB_RESOLVE_RT_5,
+   PAN_FB_RESOLVE_RT_6,
+   PAN_FB_RESOLVE_RT_7,
+
+   /** Load from the depth target
+    *
+    * From the perspective of resolve ops, Z/S are always separate
+    */
+   PAN_FB_RESOLVE_Z,
+
+   /** Load from the stencil target
+    *
+    * From the perspective of resolve ops, Z/S are always separate
+    */
+   PAN_FB_RESOLVE_S,
+   PAN_FB_RESOLVE_OP_COUNT,
+};
+
+static_assert(PAN_FB_RESOLVE_Z == PAN_FB_RESOLVE_RT_0 + PAN_MAX_RTS,
+              "There are PAN_MAX_RTS many RTs");
+
+#define PAN_FB_RESOLVE_RT(rt) \
+   (assert(0 <= (rt) && (rt) < PAN_MAX_RTS), (PAN_FB_RESOLVE_RT_0 + (rt)))
+
+/** Describes a resovle operation on a given target
+ *
+ * For each side of the render area (in-bounds or border), this defines both
+ * a source to copy from (the resolve op) and a MSAA copy op to apply as part
+ * of the copy.  In the common case, a resolve op will read from itself but
+ * that is not strictly a requirement.  Any resolve op can read from any
+ * render target.
+ */
+struct pan_fb_resolve_target {
+   struct pan_fb_resolve_op_msaa {
+      enum pan_fb_resolve_op resolve;
+      enum pan_fb_msaa_copy_op msaa;
+   } in_bounds, border;
+
+   /** For PAN_FB_RESOLVE_IMAGE the image view to load from */
+   const struct pan_image_view *iview;
+};
+
+/** Describes a resolve operation
+ *
+ * A resolve operation is implemented as a post-frame shader (see
+ * pan_fb_resolve_shader_key_fill) and does a copy from render targets to
+ * render targets, applying MSAA copy ops along the way.
+ */
+struct pan_fb_resolve {
+   struct pan_fb_resolve_target rts[PAN_MAX_RTS];
+   struct pan_fb_resolve_target z, s;
+};
+
 struct pan_fb_store_target {
    /** Whether or not to do a store */
    bool store;
@@ -369,6 +445,10 @@ struct pan_fb_store {
 #ifdef PAN_ARCH
 void GENX(pan_align_fb_tiling_area)(struct pan_fb_layout *fb,
                                     const struct pan_fb_store *store);
+
+void GENX(pan_fb_fold_resolve_into_store)(const struct pan_fb_layout *fb,
+                                          struct pan_fb_resolve *resolve,
+                                          struct pan_fb_store *store);
 #endif
 
 struct pan_fb_frame_shaders {
@@ -529,6 +609,12 @@ bool GENX(pan_fb_load_shader_key_fill)(struct pan_fb_shader_key *key,
                                        const struct pan_fb_load *load,
                                        bool zs_prepass);
 
+#if PAN_ARCH >= 6
+bool GENX(pan_fb_resolve_shader_key_fill)(struct pan_fb_shader_key *key,
+                                          const struct pan_fb_layout *fb,
+                                          const struct pan_fb_resolve *resolve);
+#endif
+
 struct nir_shader *
 GENX(pan_get_fb_shader)(const struct pan_fb_shader_key *key,
                         const struct nir_shader_compiler_options *nir_options);

src/panfrost/lib/pan_fb_nir.c

@@ -59,6 +59,27 @@ get_shader_op_for_load(enum pan_fb_load_op op)
    UNREACHABLE("Invalid load op");
 }
 
+static inline enum pan_fb_shader_op
+get_shader_op_for_resolve(enum pan_fb_resolve_op op)
+{
+   switch (op) {
+   case PAN_FB_RESOLVE_NONE:  return PAN_FB_SHADER_PRESERVE;
+   case PAN_FB_RESOLVE_IMAGE: return PAN_FB_SHADER_LOAD_IMAGE;
+   case PAN_FB_RESOLVE_RT_0:  return PAN_FB_SHADER_COPY_RT_0;
+   case PAN_FB_RESOLVE_RT_1:  return PAN_FB_SHADER_COPY_RT_1;
+   case PAN_FB_RESOLVE_RT_2:  return PAN_FB_SHADER_COPY_RT_2;
+   case PAN_FB_RESOLVE_RT_3:  return PAN_FB_SHADER_COPY_RT_3;
+   case PAN_FB_RESOLVE_RT_4:  return PAN_FB_SHADER_COPY_RT_4;
+   case PAN_FB_RESOLVE_RT_5:  return PAN_FB_SHADER_COPY_RT_5;
+   case PAN_FB_RESOLVE_RT_6:  return PAN_FB_SHADER_COPY_RT_6;
+   case PAN_FB_RESOLVE_RT_7:  return PAN_FB_SHADER_COPY_RT_7;
+   case PAN_FB_RESOLVE_Z:     return PAN_FB_SHADER_COPY_Z;
+   case PAN_FB_RESOLVE_S:     return PAN_FB_SHADER_COPY_S;
+   case PAN_FB_RESOLVE_OP_COUNT: UNREACHABLE("Invalid resolve op");
+   }
+   UNREACHABLE("Invalid resolve op");
+}
+
 static enum pan_fb_msaa_copy_op
 reduce_msaa_op(enum pan_fb_msaa_copy_op msaa, enum pan_fb_shader_op op,
                uint8_t fb_sample_count, uint8_t image_sample_count)
@@ -220,6 +241,54 @@ GENX(pan_fb_load_shader_key_fill)(struct pan_fb_shader_key *key,
    }
 }
 
+#if PAN_ARCH >= 6
+static struct pan_fb_shader_key_target
+get_resolve_key_target(enum pipe_format format,
+                       uint8_t fb_sample_count,
+                       bool has_border,
+                       const struct pan_fb_resolve_target *load)
+{
+   const enum pan_fb_shader_op in_bounds_op =
+      get_shader_op_for_resolve(load->in_bounds.resolve);
+   const enum pan_fb_shader_op border_op =
+      get_shader_op_for_resolve(load->border.resolve);
+   return get_key_target(format, fb_sample_count, has_border,
+                         in_bounds_op, border_op,
+                         load->in_bounds.msaa, load->border.msaa,
+                         true, load->iview);
+}
+
+bool
+GENX(pan_fb_resolve_shader_key_fill)(struct pan_fb_shader_key *key,
+                                     const struct pan_fb_layout *fb,
+                                     const struct pan_fb_resolve *resolve)
+{
+   const bool has_border = pan_fb_has_partial_tiles(fb);
+
+   *key = (struct pan_fb_shader_key) {
+      .z = get_resolve_key_target(fb->z_format, fb->sample_count,
+                                  has_border, &resolve->z),
+      .s = get_resolve_key_target(fb->s_format, fb->sample_count,
+                                  has_border, &resolve->s),
+      .z_format = fb->z_format,
+      .fb_sample_count = fb->sample_count,
+   };
+   bool needs_shader = pan_fb_shader_key_target_written(&key->z) ||
+                       pan_fb_shader_key_target_written(&key->s);
+
+   for (unsigned rt = 0; rt < fb->rt_count; rt++) {
+      key->rts[rt] = get_resolve_key_target(fb->rt_formats[rt],
+                                            fb->sample_count,
+                                            has_border,
+                                            &resolve->rts[rt]);
+      if (pan_fb_shader_key_target_written(&key->rts[rt]))
+         needs_shader = true;
+   }
+
+   return needs_shader;
+}
+#endif
+
 static nir_def *
 combine_samples_no_div(nir_builder *b, nir_def **samples, uint8_t sample_count,
                        const nir_alu_type nir_type,