pan/crc: Zero-filled CRC buffer invalidation for v7+

Set the crc_clear_color field in a specific way ensuring that the
all-zero CRC value can't ever be generated by the GPU. For v7+, this
allows to invalidate a CRC buffer by zero-filling it instead of having
to wait for a full frame draw. The drawback is that the CRC buffer
must be mapped on the CPU to do so.

The CRC state is extended with a pan_ptr that must be appropriately
set by the lib user prior to a pan_emit_fbd() call so that it can
access the CRC buffer memory.

Signed-off-by: Loïc Molinari <loic.molinari@collabora.com>

src/gallium/drivers/panfrost/pan_resource.c

@@ -264,6 +264,8 @@ panfrost_resource_import_bo(struct panfrost_resource *rsc,
    if (!rsc->bo)
       return -1;
 
+   pan_crc_state_set_ptr(&rsc->crc_state, &rsc->bo->ptr);
+
    return 0;
 }
 
@@ -1150,6 +1152,7 @@ panfrost_resource_create_with_modifier(struct pipe_screen *screen,
 
       so->bo =
          panfrost_bo_create(dev, so->plane.layout.data_size_B, flags, res_label);
+      pan_crc_state_set_ptr(&so->crc_state, &so->bo->ptr);
 
       if (!so->bo) {
          panfrost_resource_destroy(screen, &so->base);
@@ -1698,6 +1701,7 @@ panfrost_ptr_map(struct pipe_context *pctx, struct pipe_resource *resource,
             panfrost_bo_unreference(rsrc->bo);
             rsrc->bo = newbo;
             rsrc->plane.base = newbo->ptr.gpu;
+            pan_crc_state_set_ptr(&rsrc->crc_state, &newbo->ptr);
 
             if (!copy_resource && drm_is_afbc(rsrc->modifier)) {
                if (panfrost_resource_init_afbc_headers(rsrc))
@@ -1871,6 +1875,7 @@ pan_resource_modifier_convert(struct panfrost_context *ctx,
       rsrc->bo = tmp_rsrc->bo;
       rsrc->plane.base = rsrc->bo->ptr.gpu;
       panfrost_bo_reference(rsrc->bo);
+      pan_crc_state_set_ptr(&rsrc->crc_state, &rsrc->bo->ptr);
 
       rsrc->owns_label = tmp_rsrc->owns_label;
       tmp_rsrc->owns_label = false;
@@ -2218,6 +2223,7 @@ pan_resource_afbcp_commit(struct panfrost_context *ctx,
    panfrost_bo_unreference(prsrc->bo);
    prsrc->bo = prsrc->afbcp->packed_bo;
    prsrc->afbcp->packed_bo = NULL;
+   pan_crc_state_set_ptr(&prsrc->crc_state, &prsrc->bo->ptr);
 
    pan_resource_afbcp_stop(prsrc);
 }
@@ -2329,6 +2335,7 @@ panfrost_ptr_unmap(struct pipe_context *pctx, struct pipe_transfer *transfer)
             prsrc->bo = pan_resource(trans->staging.rsrc)->bo;
             prsrc->plane.base = prsrc->bo->ptr.gpu;
             panfrost_bo_reference(prsrc->bo);
+            pan_crc_state_set_ptr(&prsrc->crc_state, &prsrc->bo->ptr);
 
             prsrc->owns_label = pan_resource(trans->staging.rsrc)->owns_label;
             pan_resource(trans->staging.rsrc)->owns_label = false;

src/panfrost/lib/pan_desc.c

@@ -1084,6 +1084,28 @@ pan_crc_enable(struct pan_crc *crc)
    crc->write = true;
 }
 
+#if PAN_ARCH >= 7
+/* Initialize the CRC buffer by zero'ing it. The all-zero CRC can't collide
+ * thanks to the crc_clear_color field, see pan_crc_clear_color(). Drawback is
+ * the CRC BO must be CPU mapped. */
+static void
+pan_crc_enable_zeroed(struct pan_crc *crc, struct pan_crc_state *state,
+                      const struct pan_image_view *view)
+{
+   const struct pan_image_plane_ref pref =
+      pan_image_view_get_color_plane(view);
+   const struct pan_image_plane *plane = pref.image->planes[pref.plane_idx];
+   const struct pan_image_slice_layout *slice =
+      &plane->layout.slices[view->first_level];
+
+   assert(state->ptr && state->ptr->cpu);
+   memset(state->ptr->cpu + slice->crc.offset_B, 0, slice->crc.size_B);
+
+   pan_crc_enable(crc);
+   state->valid = true;
+}
+#endif
+
 /* Take advantage of a full frame draw to initialize the CRC buffer by
  * forcefully writing back all the tiles and flush the CRC values. Drawback
  * is it only works on full frames. */
@@ -1104,8 +1126,8 @@ static uint64_t
 pan_crc_clear_color(const struct pan_fb_info *fb)
 {
    uint64_t base[4] = { 0, }; /* Compiler auto-vectorization hint */
-   uint64_t crc_clear_flag = 1;
-   uint64_t crc_clear_base = 0;
+   uint64_t crc_clear_flag = 0;
+   uint64_t crc_clear_base = 1ull << 46;
    uint64_t crc_init = 0;
 
    /* When a tile is clear (i.e. no polygons intersect it), the configured
@@ -1117,14 +1139,18 @@ pan_crc_clear_color(const struct pan_fb_info *fb)
     * render on the selected RT. It's done by comparing CRCs in the CRC buffer
     * to the crc_clear_color.
     *
-    * The crc_clear_flag sub-field (bit 63) is flagged set here. It's flipped
-    * by the GPU when writing standard (i.e. non-empty) CRCs.
+    * The crc_clear_flag sub-field (bit 63) is flagged unset here. It's
+    * flipped by the GPU when writing standard (i.e. non-empty) CRCs. This
+    * prevents standard CRCs from using the all-zero CRC value. Empty CRCs
+    * can't use the all-zero CRC value either because crc_clear_base's most
+    * significant bit is flagged set here. This allows to invalidate a CRC
+    * buffer by zero'ing it.
     *
     * v10 introduced the crc_init sub-field (bits 15:0). v7 and v9 can use
     * those as additional crc_clear_base bits. We don't use it for now and
     * keep those 16 bits clear regardless of arch.
     *
-    * This leaves 47 bits in the crc_clear_base sub-field (bits 62:16). Clear
+    * This leaves 46 bits in the crc_clear_base sub-field (bits 62:16). Clear
     * color changes on any RTs must be reflected into this field in order to
     * properly invalidate CRCs stored this way. This is done by hashing the
     * clear value channels of each cleared RT. Each clear color channel value
@@ -1132,17 +1158,17 @@ pan_crc_clear_color(const struct pan_fb_info *fb)
     * hash. Clear values in pan_fb_info struct are expected to be packed with
     * respect to the format and dithering of the underlying RTs so that a
     * change of format (without a clear color change) can generate a different
-    * hash. The prime number 32749 is carefully selected so that the 32 bits
-    * of each clear color channel take at most 47 bits after the mul (the next
-    * prime number 32771 takes at most 48 bits). The resulting hash value is
+    * hash. The prime number 16381 is carefully selected so that the 32 bits
+    * of each clear color channel take at most 46 bits after the mul (the next
+    * prime number 16411 takes at most 47 bits). The resulting hash value is
     * guaranteed not to overflow and can safely be packed. */
 
    for (unsigned i = 0; i < fb->rt_count; ++i)
       if (fb->rts[i].clear)
          for (unsigned j = 0; j < 4; ++j)
-            base[i] ^= 32749 * fb->rts[i].clear_value[j];
+            base[i] ^= 16381 * fb->rts[i].clear_value[j];
 
-   crc_clear_base = (base[0] ^ base[1]) ^ (base[2] ^ base[3]);
+   crc_clear_base |= (base[0] ^ base[1]) ^ (base[2] ^ base[3]);
 
    return (crc_clear_flag << 63) | (crc_clear_base << 16) | crc_init;
 }
@@ -1191,7 +1217,14 @@ pan_get_crc_info(const struct pan_fb_info *fb)
    if (rt->crc_state->valid) {
       pan_crc_enable(&crc);
    } else {
+#if PAN_ARCH >= 7
+      if (rt->crc_state->ptr && rt->crc_state->ptr->cpu)
+         pan_crc_enable_zeroed(&crc, rt->crc_state, rt->view);
+      else
+         pan_crc_maybe_enable_flushed(&crc, rt->crc_state, fb);
+#else
       pan_crc_maybe_enable_flushed(&crc, rt->crc_state, fb);
+#endif
    }
 
 #if PAN_ARCH >= 6

src/panfrost/lib/pan_desc.h

@@ -174,6 +174,9 @@ struct pan_crc {
 };
 
 struct pan_crc_state {
+   /* Pointer to BO mapping. */
+   struct pan_ptr *ptr;
+
    /* Is the CRC buffer valid? Implicitly refers to the first slice. */
    bool valid;
 };
@@ -207,6 +210,12 @@ pan_crc_state_invalidate(struct pan_crc_state *state)
    state->valid = false;
 }
 
+static inline void
+pan_crc_state_set_ptr(struct pan_crc_state *state, struct pan_ptr *ptr)
+{
+   state->ptr = ptr;
+}
+
 static inline bool
 pan_clean_tile_write_rt_enabled(struct pan_clean_tile clean_tile,
                                 unsigned index)