i965/gen6+: Add support for fast depth clears.

Improves citybench high-res performance 3.0% +- 0.4%, n=10.  Improves
Lightsmark 1024x768 performance 0.74% +/- 0.20% (n=78).  No
significant difference on openarena (n=5, didn't fast clear) or nexuiz
(n=3).

Reviewed-by: Ian Romanick <ian.d.romanick@intel.com>
Reviewed-by: Chad Versace <chad.versace@linux.intel.com>

src/mesa/drivers/dri/i965/brw_blorp.cpp

@@ -131,7 +131,6 @@ brw_hiz_op_params::brw_hiz_op_params(struct intel_mipmap_tree *mt,
                                      unsigned int layer,
                                      gen6_hiz_op op)
 {
-   assert(op != GEN6_HIZ_OP_DEPTH_CLEAR); /* Not implemented yet. */
    this->hiz_op = op;
 
    depth.set(mt, level, layer);

src/mesa/drivers/dri/i965/brw_clear.c

@@ -32,10 +32,12 @@
 #include "swrast/swrast.h"
 #include "drivers/common/meta.h"
 
+#include "intel_batchbuffer.h"
 #include "intel_context.h"
 #include "intel_blit.h"
 #include "intel_clear.h"
 #include "intel_fbo.h"
+#include "intel_mipmap_tree.h"
 #include "intel_regions.h"
 
 #define FILE_DEBUG_FLAG DEBUG_BLIT
@@ -74,6 +76,125 @@ debug_mask(const char *name, GLbitfield mask)
    }
 }
 
+/**
+ * Implements fast depth clears on gen6+.
+ *
+ * Fast clears basically work by setting a flag in each of the subspans
+ * represented in the HiZ buffer that says "When you need the depth values for
+ * this subspan, it's the hardware's current clear value."  Then later rendering
+ * can just use the static clear value instead of referencing memory.
+ *
+ * The tricky part of the implementation is that you have to have the clear
+ * value that was used on the depth buffer in place for all further rendering,
+ * at least until a resolve to the real depth buffer happens.
+ */
+static bool
+brw_fast_clear_depth(struct gl_context *ctx)
+{
+   struct intel_context *intel = intel_context(ctx);
+   struct gl_framebuffer *fb = ctx->DrawBuffer;
+   struct intel_renderbuffer *depth_irb =
+      intel_get_renderbuffer(fb, BUFFER_DEPTH);
+   struct intel_mipmap_tree *mt = depth_irb->mt;
+
+   if (intel->gen < 6)
+      return false;
+
+   if (!mt->hiz_mt)
+      return false;
+
+   /* We only handle full buffer clears -- otherwise you'd have to track whether
+    * a previous clear had happened at a different clear value and resolve it
+    * first.
+    */
+   if (ctx->Scissor.Enabled)
+      return false;
+
+   /* The rendered area has to be 8x4 samples, not resolved pixels, so we look
+    * at the miptree slice dimensions instead of renderbuffer size.
+    */
+   if (mt->level[depth_irb->mt_level].width % 8 != 0 ||
+       mt->level[depth_irb->mt_level].height % 4 != 0) {
+      return false;
+   }
+
+   uint32_t depth_clear_value;
+   switch (mt->format) {
+   case MESA_FORMAT_Z32_FLOAT_X24S8:
+   case MESA_FORMAT_S8_Z24:
+      /* From the Sandy Bridge PRM, volume 2 part 1, page 314:
+       *
+       *     "[DevSNB+]: Several cases exist where Depth Buffer Clear cannot be
+       *      enabled (the legacy method of clearing must be performed):
+       *
+       *      - If the depth buffer format is D32_FLOAT_S8X24_UINT or
+       *        D24_UNORM_S8_UINT.
+       */
+      return false;
+
+   case MESA_FORMAT_Z32_FLOAT:
+      depth_clear_value = float_as_int(ctx->Depth.Clear);
+      break;
+
+   case MESA_FORMAT_Z16:
+      /* From the Sandy Bridge PRM, volume 2 part 1, page 314:
+       *
+       *     "[DevSNB+]: Several cases exist where Depth Buffer Clear cannot be
+       *      enabled (the legacy method of clearing must be performed):
+       *
+       *      - DevSNB{W/A}]: When depth buffer format is D16_UNORM and the
+       *        width of the map (LOD0) is not multiple of 16, fast clear
+       *        optimization must be disabled.
+       */
+      if (intel->gen == 6 && (mt->level[depth_irb->mt_level].width % 16) != 0)
+	 return false;
+      /* FALLTHROUGH */
+
+   default:
+      depth_clear_value = fb->_DepthMax * ctx->Depth.Clear;
+      break;
+   }
+
+   /* If we're clearing to a new clear value, then we need to resolve any clear
+    * flags out of the HiZ buffer into the real depth buffer.
+    */
+   if (mt->depth_clear_value != depth_clear_value) {
+      intel_miptree_all_slices_resolve_depth(intel, mt);
+      mt->depth_clear_value = depth_clear_value;
+   }
+
+   /* From the Sandy Bridge PRM, volume 2 part 1, page 313:
+    *
+    *     "If other rendering operations have preceded this clear, a
+    *      PIPE_CONTROL with write cache flush enabled and Z-inhibit disabled
+    *      must be issued before the rectangle primitive used for the depth
+    *      buffer clear operation.
+    */
+   intel_batchbuffer_emit_mi_flush(intel);
+
+   intel_hiz_exec(intel, mt, depth_irb->mt_level, depth_irb->mt_layer,
+		  GEN6_HIZ_OP_DEPTH_CLEAR);
+
+   if (intel->gen == 6) {
+      /* From the Sandy Bridge PRM, volume 2 part 1, page 314:
+       *
+       *     "DevSNB, DevSNB-B{W/A}]: Depth buffer clear pass must be followed
+       *      by a PIPE_CONTROL command with DEPTH_STALL bit set and Then
+       *      followed by Depth FLUSH'
+      */
+      intel_batchbuffer_emit_mi_flush(intel);
+   }
+
+   /* Now, the entire HiZ buffer contains data that needs to be resolved to the
+    * entire depth buffer (so any previous resolve records should get tossed
+    * out).
+    */
+   intel_resolve_map_clear(&mt->hiz_map);
+   intel_renderbuffer_set_needs_depth_resolve(depth_irb);
+
+   return true;
+}
+
 /**
  * Called by ctx->Driver.Clear.
  */
@@ -89,6 +210,15 @@ brw_clear(struct gl_context *ctx, GLbitfield mask)
       intel->front_buffer_dirty = true;
    }
 
+   intel_prepare_render(intel);
+
+   if (mask & BUFFER_BIT_DEPTH) {
+      if (brw_fast_clear_depth(ctx)) {
+	 DBG("fast clear: depth\n");
+	 mask &= ~BUFFER_BIT_DEPTH;
+      }
+   }
+
    GLbitfield tri_mask = mask & (BUFFER_BITS_COLOR |
 				 BUFFER_BIT_STENCIL |
 				 BUFFER_BIT_DEPTH);

src/mesa/drivers/dri/i965/brw_defines.h

@@ -1466,8 +1466,10 @@ enum brw_wm_barycentric_interp_mode {
 #define GEN7_3DSTATE_HIER_DEPTH_BUFFER		0x7807
 
 #define _3DSTATE_CLEAR_PARAMS			0x7910 /* ILK, SNB */
-# define DEPTH_CLEAR_VALID				(1 << 15)
+# define GEN5_DEPTH_CLEAR_VALID				(1 << 15)
 /* DW1: depth clear value */
+/* DW2 */
+# define GEN7_DEPTH_CLEAR_VALID				(1 << 0)
 
 #define _3DSTATE_SO_DECL_LIST			0x7917 /* GEN7+ */
 /* DW1 */

src/mesa/drivers/dri/i965/brw_misc_state.c

@@ -581,8 +581,10 @@ static void emit_depthbuffer(struct brw_context *brw)
 	 intel_emit_post_sync_nonzero_flush(intel);
 
       BEGIN_BATCH(2);
-      OUT_BATCH(_3DSTATE_CLEAR_PARAMS << 16 | (2 - 2));
-      OUT_BATCH(0);
+      OUT_BATCH(_3DSTATE_CLEAR_PARAMS << 16 |
+		GEN5_DEPTH_CLEAR_VALID |
+		(2 - 2));
+      OUT_BATCH(depth_irb ? depth_irb->mt->depth_clear_value : 0);
       ADVANCE_BATCH();
    }
 }

src/mesa/drivers/dri/i965/gen6_blorp.cpp

@@ -711,7 +711,6 @@ gen6_blorp_emit_wm_config(struct brw_context *brw,
    dw2 = dw4 = dw5 = dw6 = 0;
    switch (params->hiz_op) {
    case GEN6_HIZ_OP_DEPTH_CLEAR:
-      assert(!"not implemented");
       dw4 |= GEN6_WM_DEPTH_CLEAR;
       break;
    case GEN6_HIZ_OP_DEPTH_RESOLVE:
@@ -939,8 +938,10 @@ gen6_blorp_emit_clear_params(struct brw_context *brw,
    struct intel_context *intel = &brw->intel;
 
    BEGIN_BATCH(2);
-   OUT_BATCH(_3DSTATE_CLEAR_PARAMS << 16 | (2 - 2));
-   OUT_BATCH(0);
+   OUT_BATCH(_3DSTATE_CLEAR_PARAMS << 16 |
+	     GEN5_DEPTH_CLEAR_VALID |
+	     (2 - 2));
+   OUT_BATCH(params->depth.mt ? params->depth.mt->depth_clear_value : 0);
    ADVANCE_BATCH();
 }
 

src/mesa/drivers/dri/i965/gen7_blorp.cpp

@@ -427,7 +427,6 @@ gen7_blorp_emit_wm_config(struct brw_context *brw,
 
    switch (params->hiz_op) {
    case GEN6_HIZ_OP_DEPTH_CLEAR:
-      assert(!"not implemented");
       dw1 |= GEN7_WM_DEPTH_CLEAR;
       break;
    case GEN6_HIZ_OP_DEPTH_RESOLVE:
@@ -696,8 +695,8 @@ gen7_blorp_emit_clear_params(struct brw_context *brw,
 
    BEGIN_BATCH(3);
    OUT_BATCH(GEN7_3DSTATE_CLEAR_PARAMS << 16 | (3 - 2));
-   OUT_BATCH(0);
-   OUT_BATCH(0);
+   OUT_BATCH(params->depth.mt ? params->depth.mt->depth_clear_value : 0);
+   OUT_BATCH(GEN7_DEPTH_CLEAR_VALID);
    ADVANCE_BATCH();
 }
 

src/mesa/drivers/dri/i965/gen7_misc_state.c

@@ -268,8 +268,8 @@ static void emit_depthbuffer(struct brw_context *brw)
 
    BEGIN_BATCH(3);
    OUT_BATCH(GEN7_3DSTATE_CLEAR_PARAMS << 16 | (3 - 2));
-   OUT_BATCH(0);
-   OUT_BATCH(0);
+   OUT_BATCH(depth_mt ? depth_mt->depth_clear_value : 0);
+   OUT_BATCH(1);
    ADVANCE_BATCH();
 }
 

src/mesa/drivers/dri/intel/intel_mipmap_tree.h

@@ -177,6 +177,11 @@ struct intel_mipmap_tree
    GLuint total_width;
    GLuint total_height;
 
+   /* The 3DSTATE_CLEAR_PARAMS value associated with the last depth clear to
+    * this depth mipmap tree, if any.
+    */
+   uint32_t depth_clear_value;
+
    /* Includes image offset tables:
     */
    struct intel_mipmap_level level[MAX_TEXTURE_LEVELS];