mesa/src/intel/blorp/blorp_blit.c

/*
 * Copyright © 2012 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "blorp_nir_builder.h"
#include "compiler/nir/nir_format_convert.h"

#include "blorp_priv.h"

#include "util/format_rgb9e5.h"
/* header-only include needed for _mesa_unorm_to_float and friends. */
#include "mesa/main/format_utils.h"
#include "util/u_math.h"

#define FILE_DEBUG_FLAG DEBUG_BLORP

static const bool split_blorp_blit_debug = false;

struct brw_blorp_blit_vars {
   /* Input values from brw_blorp_wm_inputs */
   nir_variable *v_discard_rect;
   nir_variable *v_rect_grid;
   nir_variable *v_coord_transform;
   nir_variable *v_src_z;
   nir_variable *v_src_offset;
   nir_variable *v_dst_offset;
   nir_variable *v_src_inv_size;
};

static void
brw_blorp_blit_vars_init(nir_builder *b, struct brw_blorp_blit_vars *v,
                         const struct brw_blorp_blit_prog_key *key)
{
#define LOAD_INPUT(name, type)\
   v->v_##name = BLORP_CREATE_NIR_INPUT(b->shader, name, type);

   LOAD_INPUT(discard_rect, glsl_vec4_type())
   LOAD_INPUT(rect_grid, glsl_vec4_type())
   LOAD_INPUT(coord_transform, glsl_vec4_type())
   LOAD_INPUT(src_z, glsl_float_type())
   LOAD_INPUT(src_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
   LOAD_INPUT(dst_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
   LOAD_INPUT(src_inv_size, glsl_vector_type(GLSL_TYPE_FLOAT, 2))

#undef LOAD_INPUT
}

static nir_ssa_def *
blorp_blit_get_frag_coords(nir_builder *b,
                           const struct brw_blorp_blit_prog_key *key,
                           struct brw_blorp_blit_vars *v)
{
   nir_ssa_def *coord = nir_f2i32(b, nir_load_frag_coord(b));

   /* Account for destination surface intratile offset
    *
    * Transformation parameters giving translation from destination to source
    * coordinates don't take into account possible intra-tile destination
    * offset.  Therefore it has to be first subtracted from the incoming
    * coordinates.  Vertices are set up based on coordinates containing the
    * intra-tile offset.
    */
   if (key->need_dst_offset)
      coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));

   if (key->persample_msaa_dispatch) {
      return nir_vec3(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1),
                      nir_load_sample_id(b));
   } else {
      return nir_vec2(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1));
   }
}

/**
 * Emit code to translate from destination (X, Y) coordinates to source (X, Y)
 * coordinates.
 */
static nir_ssa_def *
blorp_blit_apply_transform(nir_builder *b, nir_ssa_def *src_pos,
                           struct brw_blorp_blit_vars *v)
{
   nir_ssa_def *coord_transform = nir_load_var(b, v->v_coord_transform);

   nir_ssa_def *offset = nir_vec2(b, nir_channel(b, coord_transform, 1),
                                     nir_channel(b, coord_transform, 3));
   nir_ssa_def *mul = nir_vec2(b, nir_channel(b, coord_transform, 0),
                                  nir_channel(b, coord_transform, 2));

   return nir_fadd(b, nir_fmul(b, src_pos, mul), offset);
}

static inline void
blorp_nir_discard_if_outside_rect(nir_builder *b, nir_ssa_def *pos,
                                  struct brw_blorp_blit_vars *v)
{
   nir_ssa_def *c0, *c1, *c2, *c3;
   nir_ssa_def *discard_rect = nir_load_var(b, v->v_discard_rect);
   nir_ssa_def *dst_x0 = nir_channel(b, discard_rect, 0);
   nir_ssa_def *dst_x1 = nir_channel(b, discard_rect, 1);
   nir_ssa_def *dst_y0 = nir_channel(b, discard_rect, 2);
   nir_ssa_def *dst_y1 = nir_channel(b, discard_rect, 3);

   c0 = nir_ult(b, nir_channel(b, pos, 0), dst_x0);
   c1 = nir_uge(b, nir_channel(b, pos, 0), dst_x1);
   c2 = nir_ult(b, nir_channel(b, pos, 1), dst_y0);
   c3 = nir_uge(b, nir_channel(b, pos, 1), dst_y1);

   nir_ssa_def *oob = nir_ior(b, nir_ior(b, c0, c1), nir_ior(b, c2, c3));
   nir_discard_if(b, oob);
}

static nir_tex_instr *
blorp_create_nir_tex_instr(nir_builder *b, struct brw_blorp_blit_vars *v,
                           nir_texop op, nir_ssa_def *pos, unsigned num_srcs,
                           nir_alu_type dst_type)
{
   nir_tex_instr *tex = nir_tex_instr_create(b->shader, num_srcs);

   tex->op = op;

   tex->dest_type = dst_type | 32;
   tex->is_array = false;
   tex->is_shadow = false;

   /* Blorp only has one texture and it's bound at unit 0 */
   tex->texture_index = 0;
   tex->sampler_index = 0;

   /* To properly handle 3-D and 2-D array textures, we pull the Z component
    * from an input.  TODO: This is a bit magic; we should probably make this
    * more explicit in the future.
    */
   assert(pos->num_components >= 2);
   if (op == nir_texop_txf || op == nir_texop_txf_ms || op == nir_texop_txf_ms_mcs) {
      pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
                        nir_f2i32(b, nir_load_var(b, v->v_src_z)));
   } else {
      pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
                        nir_load_var(b, v->v_src_z));
   }

   tex->src[0].src_type = nir_tex_src_coord;
   tex->src[0].src = nir_src_for_ssa(pos);
   tex->coord_components = 3;

   nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);

   return tex;
}

static nir_ssa_def *
blorp_nir_tex(nir_builder *b, struct brw_blorp_blit_vars *v,
              const struct brw_blorp_blit_prog_key *key, nir_ssa_def *pos)
{
   if (key->need_src_offset)
      pos = nir_fadd(b, pos, nir_i2f32(b, nir_load_var(b, v->v_src_offset)));

   /* If the sampler requires normalized coordinates, we need to compensate. */
   if (key->src_coords_normalized)
      pos = nir_fmul(b, pos, nir_load_var(b, v->v_src_inv_size));

   nir_tex_instr *tex =
      blorp_create_nir_tex_instr(b, v, nir_texop_tex, pos, 2,
                                 key->texture_data_type);

   assert(pos->num_components == 2);
   tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
   tex->src[1].src_type = nir_tex_src_lod;
   tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));

   nir_builder_instr_insert(b, &tex->instr);

   return &tex->dest.ssa;
}

static nir_ssa_def *
blorp_nir_txf(nir_builder *b, struct brw_blorp_blit_vars *v,
              nir_ssa_def *pos, nir_alu_type dst_type)
{
   nir_tex_instr *tex =
      blorp_create_nir_tex_instr(b, v, nir_texop_txf, pos, 2, dst_type);

   tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
   tex->src[1].src_type = nir_tex_src_lod;
   tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));

   nir_builder_instr_insert(b, &tex->instr);

   return &tex->dest.ssa;
}

static nir_ssa_def *
blorp_nir_txf_ms(nir_builder *b, struct brw_blorp_blit_vars *v,
                 nir_ssa_def *pos, nir_ssa_def *mcs, nir_alu_type dst_type)
{
   nir_tex_instr *tex =
      blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms, pos,
                                 mcs != NULL ? 3 : 2, dst_type);

   tex->sampler_dim = GLSL_SAMPLER_DIM_MS;

   tex->src[1].src_type = nir_tex_src_ms_index;
   if (pos->num_components == 2) {
      tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
   } else {
      assert(pos->num_components == 3);
      tex->src[1].src = nir_src_for_ssa(nir_channel(b, pos, 2));
   }

   if (mcs) {
      tex->src[2].src_type = nir_tex_src_ms_mcs;
      tex->src[2].src = nir_src_for_ssa(mcs);
   }

   nir_builder_instr_insert(b, &tex->instr);

   return &tex->dest.ssa;
}

static nir_ssa_def *
blorp_blit_txf_ms_mcs(nir_builder *b, struct brw_blorp_blit_vars *v,
                      nir_ssa_def *pos)
{
   nir_tex_instr *tex =
      blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms_mcs,
                                 pos, 1, nir_type_int);

   tex->sampler_dim = GLSL_SAMPLER_DIM_MS;

   nir_builder_instr_insert(b, &tex->instr);

   return &tex->dest.ssa;
}

/**
 * Emit code to compensate for the difference between Y and W tiling.
 *
 * This code modifies the X and Y coordinates according to the formula:
 *
 *   (X', Y', S') = detile(W-MAJOR, tile(Y-MAJOR, X, Y, S))
 *
 * (See brw_blorp_build_nir_shader).
 */
static inline nir_ssa_def *
blorp_nir_retile_y_to_w(nir_builder *b, nir_ssa_def *pos)
{
   assert(pos->num_components == 2);
   nir_ssa_def *x_Y = nir_channel(b, pos, 0);
   nir_ssa_def *y_Y = nir_channel(b, pos, 1);

   /* Given X and Y coordinates that describe an address using Y tiling,
    * translate to the X and Y coordinates that describe the same address
    * using W tiling.
    *
    * If we break down the low order bits of X and Y, using a
    * single letter to represent each low-order bit:
    *
    *   X = A << 7 | 0bBCDEFGH
    *   Y = J << 5 | 0bKLMNP                                       (1)
    *
    * Then we can apply the Y tiling formula to see the memory offset being
    * addressed:
    *
    *   offset = (J * tile_pitch + A) << 12 | 0bBCDKLMNPEFGH       (2)
    *
    * If we apply the W detiling formula to this memory location, that the
    * corresponding X' and Y' coordinates are:
    *
    *   X' = A << 6 | 0bBCDPFH                                     (3)
    *   Y' = J << 6 | 0bKLMNEG
    *
    * Combining (1) and (3), we see that to transform (X, Y) to (X', Y'),
    * we need to make the following computation:
    *
    *   X' = (X & ~0b1011) >> 1 | (Y & 0b1) << 2 | X & 0b1         (4)
    *   Y' = (Y & ~0b1) << 1 | (X & 0b1000) >> 2 | (X & 0b10) >> 1
    */
   nir_ssa_def *x_W = nir_imm_int(b, 0);
   x_W = nir_mask_shift_or(b, x_W, x_Y, 0xfffffff4, -1);
   x_W = nir_mask_shift_or(b, x_W, y_Y, 0x1, 2);
   x_W = nir_mask_shift_or(b, x_W, x_Y, 0x1, 0);

   nir_ssa_def *y_W = nir_imm_int(b, 0);
   y_W = nir_mask_shift_or(b, y_W, y_Y, 0xfffffffe, 1);
   y_W = nir_mask_shift_or(b, y_W, x_Y, 0x8, -2);
   y_W = nir_mask_shift_or(b, y_W, x_Y, 0x2, -1);

   return nir_vec2(b, x_W, y_W);
}

/**
 * Emit code to compensate for the difference between Y and W tiling.
 *
 * This code modifies the X and Y coordinates according to the formula:
 *
 *   (X', Y', S') = detile(Y-MAJOR, tile(W-MAJOR, X, Y, S))
 *
 * (See brw_blorp_build_nir_shader).
 */
static inline nir_ssa_def *
blorp_nir_retile_w_to_y(nir_builder *b, nir_ssa_def *pos)
{
   assert(pos->num_components == 2);
   nir_ssa_def *x_W = nir_channel(b, pos, 0);
   nir_ssa_def *y_W = nir_channel(b, pos, 1);

   /* Applying the same logic as above, but in reverse, we obtain the
    * formulas:
    *
    * X' = (X & ~0b101) << 1 | (Y & 0b10) << 2 | (Y & 0b1) << 1 | X & 0b1
    * Y' = (Y & ~0b11) >> 1 | (X & 0b100) >> 2
    */
   nir_ssa_def *x_Y = nir_imm_int(b, 0);
   x_Y = nir_mask_shift_or(b, x_Y, x_W, 0xfffffffa, 1);
   x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x2, 2);
   x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x1, 1);
   x_Y = nir_mask_shift_or(b, x_Y, x_W, 0x1, 0);

   nir_ssa_def *y_Y = nir_imm_int(b, 0);
   y_Y = nir_mask_shift_or(b, y_Y, y_W, 0xfffffffc, -1);
   y_Y = nir_mask_shift_or(b, y_Y, x_W, 0x4, -2);

   return nir_vec2(b, x_Y, y_Y);
}

/**
 * Emit code to compensate for the difference between MSAA and non-MSAA
 * surfaces.
 *
 * This code modifies the X and Y coordinates according to the formula:
 *
 *   (X', Y', S') = encode_msaa(num_samples, IMS, X, Y, S)
 *
 * (See brw_blorp_blit_program).
 */
static inline nir_ssa_def *
blorp_nir_encode_msaa(nir_builder *b, nir_ssa_def *pos,
                      unsigned num_samples, enum isl_msaa_layout layout)
{
   assert(pos->num_components == 2 || pos->num_components == 3);

   switch (layout) {
   case ISL_MSAA_LAYOUT_NONE:
      assert(pos->num_components == 2);
      return pos;
   case ISL_MSAA_LAYOUT_ARRAY:
      /* No translation needed */
      return pos;
   case ISL_MSAA_LAYOUT_INTERLEAVED: {
      nir_ssa_def *x_in = nir_channel(b, pos, 0);
      nir_ssa_def *y_in = nir_channel(b, pos, 1);
      nir_ssa_def *s_in = pos->num_components == 2 ? nir_imm_int(b, 0) :
                                                     nir_channel(b, pos, 2);

      nir_ssa_def *x_out = nir_imm_int(b, 0);
      nir_ssa_def *y_out = nir_imm_int(b, 0);
      switch (num_samples) {
      case 2:
      case 4:
         /* encode_msaa(2, IMS, X, Y, S) = (X', Y', 0)
          *   where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
          *         Y' = Y
          *
          * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
          *   where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
          *         Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
          */
         x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 1);
         x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
         x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
         if (num_samples == 2) {
            y_out = y_in;
         } else {
            y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
            y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
            y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
         }
         break;

      case 8:
         /* encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
          *   where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
          *              | (X & 0b1)
          *         Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
          */
         x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
         x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
         x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
         x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
         y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
         break;

      case 16:
         /* encode_msaa(16, IMS, X, Y, S) = (X', Y', 0)
          *   where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
          *              | (X & 0b1)
          *         Y' = (Y & ~0b1) << 2 | (S & 0b1000) >> 1 (S & 0b10)
          *              | (Y & 0b1)
          */
         x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
         x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
         x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
         x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 2);
         y_out = nir_mask_shift_or(b, y_out, s_in, 0x8, -1);
         y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
         break;

      default:
         unreachable("Invalid number of samples for IMS layout");
      }

      return nir_vec2(b, x_out, y_out);
   }

   default:
      unreachable("Invalid MSAA layout");
   }
}

/**
 * Emit code to compensate for the difference between MSAA and non-MSAA
 * surfaces.
 *
 * This code modifies the X and Y coordinates according to the formula:
 *
 *   (X', Y', S) = decode_msaa(num_samples, IMS, X, Y, S)
 *
 * (See brw_blorp_blit_program).
 */
static inline nir_ssa_def *
blorp_nir_decode_msaa(nir_builder *b, nir_ssa_def *pos,
                      unsigned num_samples, enum isl_msaa_layout layout)
{
   assert(pos->num_components == 2 || pos->num_components == 3);

   switch (layout) {
   case ISL_MSAA_LAYOUT_NONE:
      /* No translation necessary, and S should already be zero. */
      assert(pos->num_components == 2);
      return pos;
   case ISL_MSAA_LAYOUT_ARRAY:
      /* No translation necessary. */
      return pos;
   case ISL_MSAA_LAYOUT_INTERLEAVED: {
      assert(pos->num_components == 2);

      nir_ssa_def *x_in = nir_channel(b, pos, 0);
      nir_ssa_def *y_in = nir_channel(b, pos, 1);

      nir_ssa_def *x_out = nir_imm_int(b, 0);
      nir_ssa_def *y_out = nir_imm_int(b, 0);
      nir_ssa_def *s_out = nir_imm_int(b, 0);
      switch (num_samples) {
      case 2:
      case 4:
         /* decode_msaa(2, IMS, X, Y, 0) = (X', Y', S)
          *   where X' = (X & ~0b11) >> 1 | (X & 0b1)
          *         S = (X & 0b10) >> 1
          *
          * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
          *   where X' = (X & ~0b11) >> 1 | (X & 0b1)
          *         Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
          *         S = (Y & 0b10) | (X & 0b10) >> 1
          */
         x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffc, -1);
         x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
         if (num_samples == 2) {
            y_out = y_in;
            s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
         } else {
            y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
            y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
            s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
            s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
         }
         break;

      case 8:
         /* decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
          *   where X' = (X & ~0b111) >> 2 | (X & 0b1)
          *         Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
          *         S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
          */
         x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
         x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
         s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
         s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
         s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
         break;

      case 16:
         /* decode_msaa(16, IMS, X, Y, 0) = (X', Y', S)
          *   where X' = (X & ~0b111) >> 2 | (X & 0b1)
          *         Y' = (Y & ~0b111) >> 2 | (Y & 0b1)
          *         S = (Y & 0b100) << 1 | (X & 0b100) |
          *             (Y & 0b10) | (X & 0b10) >> 1
          */
         x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
         x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffff8, -2);
         y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
         s_out = nir_mask_shift_or(b, s_out, y_in, 0x4, 1);
         s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
         s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
         s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
         break;

      default:
         unreachable("Invalid number of samples for IMS layout");
      }

      return nir_vec3(b, x_out, y_out, s_out);
   }

   default:
      unreachable("Invalid MSAA layout");
   }
}

/**
 * Count the number of trailing 1 bits in the given value.  For example:
 *
 * count_trailing_one_bits(0) == 0
 * count_trailing_one_bits(7) == 3
 * count_trailing_one_bits(11) == 2
 */
static inline int count_trailing_one_bits(unsigned value)
{
#ifdef HAVE___BUILTIN_CTZ
   return __builtin_ctz(~value);
#else
   return util_bitcount(value & ~(value + 1));
#endif
}

static nir_ssa_def *
blorp_nir_combine_samples(nir_builder *b, struct brw_blorp_blit_vars *v,
                          nir_ssa_def *pos, unsigned tex_samples,
                          enum isl_aux_usage tex_aux_usage,
                          nir_alu_type dst_type,
                          enum blorp_filter filter)
{
   nir_variable *color =
      nir_local_variable_create(b->impl, glsl_vec4_type(), "color");

   nir_ssa_def *mcs = NULL;
   if (isl_aux_usage_has_mcs(tex_aux_usage))
      mcs = blorp_blit_txf_ms_mcs(b, v, pos);

   nir_op combine_op;
   switch (filter) {
   case BLORP_FILTER_AVERAGE:
      assert(dst_type == nir_type_float);
      combine_op = nir_op_fadd;
      break;

   case BLORP_FILTER_MIN_SAMPLE:
      switch (dst_type) {
      case nir_type_int:   combine_op = nir_op_imin;  break;
      case nir_type_uint:  combine_op = nir_op_umin;  break;
      case nir_type_float: combine_op = nir_op_fmin;  break;
      default: unreachable("Invalid dst_type");
      }
      break;

   case BLORP_FILTER_MAX_SAMPLE:
      switch (dst_type) {
      case nir_type_int:   combine_op = nir_op_imax;  break;
      case nir_type_uint:  combine_op = nir_op_umax;  break;
      case nir_type_float: combine_op = nir_op_fmax;  break;
      default: unreachable("Invalid dst_type");
      }
      break;

   default:
      unreachable("Invalid filter");
   }

   /* If true, we inserted an if statement that we need to pop at at the end.
    */
   bool inserted_if = false;

   /* We add together samples using a binary tree structure, e.g. for 4x MSAA:
    *
    *   result = ((sample[0] + sample[1]) + (sample[2] + sample[3])) / 4
    *
    * This ensures that when all samples have the same value, no numerical
    * precision is lost, since each addition operation always adds two equal
    * values, and summing two equal floating point values does not lose
    * precision.
    *
    * We perform this computation by treating the texture_data array as a
    * stack and performing the following operations:
    *
    * - push sample 0 onto stack
    * - push sample 1 onto stack
    * - add top two stack entries
    * - push sample 2 onto stack
    * - push sample 3 onto stack
    * - add top two stack entries
    * - add top two stack entries
    * - divide top stack entry by 4
    *
    * Note that after pushing sample i onto the stack, the number of add
    * operations we do is equal to the number of trailing 1 bits in i.  This
    * works provided the total number of samples is a power of two, which it
    * always is for i965.
    *
    * For integer formats, we replace the add operations with average
    * operations and skip the final division.
    */
   nir_ssa_def *texture_data[5];
   unsigned stack_depth = 0;
   for (unsigned i = 0; i < tex_samples; ++i) {
      assert(stack_depth == util_bitcount(i)); /* Loop invariant */

      /* Push sample i onto the stack */
      assert(stack_depth < ARRAY_SIZE(texture_data));

      nir_ssa_def *ms_pos = nir_vec3(b, nir_channel(b, pos, 0),
                                        nir_channel(b, pos, 1),
                                        nir_imm_int(b, i));
      texture_data[stack_depth++] = blorp_nir_txf_ms(b, v, ms_pos, mcs, dst_type);

      if (i == 0 && isl_aux_usage_has_mcs(tex_aux_usage)) {
         /* The Ivy Bridge PRM, Vol4 Part1 p27 (Multisample Control Surface)
          * suggests an optimization:
          *
          *     "A simple optimization with probable large return in
          *     performance is to compare the MCS value to zero (indicating
          *     all samples are on sample slice 0), and sample only from
          *     sample slice 0 using ld2dss if MCS is zero."
          *
          * Note that in the case where the MCS value is zero, sampling from
          * sample slice 0 using ld2dss and sampling from sample 0 using
          * ld2dms are equivalent (since all samples are on sample slice 0).
          * Since we have already sampled from sample 0, all we need to do is
          * skip the remaining fetches and averaging if MCS is zero.
          *
          * It's also trivial to detect when the MCS has the magic clear color
          * value.  In this case, the txf we did on sample 0 will return the
          * clear color and we can skip the remaining fetches just like we do
          * when MCS == 0.
          */
         nir_ssa_def *mcs_zero = nir_ieq_imm(b, nir_channel(b, mcs, 0), 0);
         if (tex_samples == 16) {
            mcs_zero = nir_iand(b, mcs_zero,
               nir_ieq_imm(b, nir_channel(b, mcs, 1), 0));
         }
         nir_ssa_def *mcs_clear =
            blorp_nir_mcs_is_clear_color(b, mcs, tex_samples);

         nir_push_if(b, nir_ior(b, mcs_zero, mcs_clear));
         nir_store_var(b, color, texture_data[0], 0xf);

         nir_push_else(b, NULL);
         inserted_if = true;
      }

      for (int j = 0; j < count_trailing_one_bits(i); j++) {
         assert(stack_depth >= 2);
         --stack_depth;

         texture_data[stack_depth - 1] =
            nir_build_alu(b, combine_op,
                             texture_data[stack_depth - 1],
                             texture_data[stack_depth],
                             NULL, NULL);
      }
   }

   /* We should have just 1 sample on the stack now. */
   assert(stack_depth == 1);

   if (filter == BLORP_FILTER_AVERAGE) {
      assert(dst_type == nir_type_float);
      texture_data[0] = nir_fmul(b, texture_data[0],
                                 nir_imm_float(b, 1.0 / tex_samples));
   }

   nir_store_var(b, color, texture_data[0], 0xf);

   if (inserted_if)
      nir_pop_if(b, NULL);

   return nir_load_var(b, color);
}

static nir_ssa_def *
blorp_nir_manual_blend_bilinear(nir_builder *b, nir_ssa_def *pos,
                                unsigned tex_samples,
                                const struct brw_blorp_blit_prog_key *key,
                                struct brw_blorp_blit_vars *v)
{
   nir_ssa_def *pos_xy = nir_channels(b, pos, 0x3);
   nir_ssa_def *rect_grid = nir_load_var(b, v->v_rect_grid);
   nir_ssa_def *scale = nir_imm_vec2(b, key->x_scale, key->y_scale);

   /* Translate coordinates to lay out the samples in a rectangular  grid
    * roughly corresponding to sample locations.
    */
   pos_xy = nir_fmul(b, pos_xy, scale);
   /* Adjust coordinates so that integers represent pixel centers rather
    * than pixel edges.
    */
   pos_xy = nir_fadd(b, pos_xy, nir_imm_float(b, -0.5));
   /* Clamp the X, Y texture coordinates to properly handle the sampling of
    * texels on texture edges.
    */
   pos_xy = nir_fmin(b, nir_fmax(b, pos_xy, nir_imm_float(b, 0.0)),
                        nir_vec2(b, nir_channel(b, rect_grid, 0),
                                    nir_channel(b, rect_grid, 1)));

   /* Store the fractional parts to be used as bilinear interpolation
    * coefficients.
    */
   nir_ssa_def *frac_xy = nir_ffract(b, pos_xy);
   /* Round the float coordinates down to nearest integer */
   pos_xy = nir_fdiv(b, nir_ftrunc(b, pos_xy), scale);

   nir_ssa_def *tex_data[4];
   for (unsigned i = 0; i < 4; ++i) {
      float sample_off_x = (float)(i & 0x1) / key->x_scale;
      float sample_off_y = (float)((i >> 1) & 0x1) / key->y_scale;
      nir_ssa_def *sample_off = nir_imm_vec2(b, sample_off_x, sample_off_y);

      nir_ssa_def *sample_coords = nir_fadd(b, pos_xy, sample_off);
      nir_ssa_def *sample_coords_int = nir_f2i32(b, sample_coords);

      /* The MCS value we fetch has to match up with the pixel that we're
       * sampling from. Since we sample from different pixels in each
       * iteration of this "for" loop, the call to mcs_fetch() should be
       * here inside the loop after computing the pixel coordinates.
       */
      nir_ssa_def *mcs = NULL;
      if (isl_aux_usage_has_mcs(key->tex_aux_usage))
         mcs = blorp_blit_txf_ms_mcs(b, v, sample_coords_int);

      /* Compute sample index and map the sample index to a sample number.
       * Sample index layout shows the numbering of slots in a rectangular
       * grid of samples with in a pixel. Sample number layout shows the
       * rectangular grid of samples roughly corresponding to the real sample
       * locations with in a pixel.
       *
       * In the case of 2x MSAA, the layout of sample indices is reversed from
       * the layout of sample numbers:
       *
       * sample index layout :  ---------    sample number layout :  ---------
       *                        | 0 | 1 |                            | 1 | 0 |
       *                        ---------                            ---------
       *
       * In case of 4x MSAA, layout of sample indices matches the layout of
       * sample numbers:
       *           ---------
       *           | 0 | 1 |
       *           ---------
       *           | 2 | 3 |
       *           ---------
       *
       * In case of 8x MSAA the two layouts don't match.
       * sample index layout :  ---------    sample number layout :  ---------
       *                        | 0 | 1 |                            | 3 | 7 |
       *                        ---------                            ---------
       *                        | 2 | 3 |                            | 5 | 0 |
       *                        ---------                            ---------
       *                        | 4 | 5 |                            | 1 | 2 |
       *                        ---------                            ---------
       *                        | 6 | 7 |                            | 4 | 6 |
       *                        ---------                            ---------
       *
       * Fortunately, this can be done fairly easily as:
       * S' = (0x17306425 >> (S * 4)) & 0xf
       *
       * In the case of 16x MSAA the two layouts don't match.
       * Sample index layout:                Sample number layout:
       * ---------------------               ---------------------
       * |  0 |  1 |  2 |  3 |               | 15 | 10 |  9 |  7 |
       * ---------------------               ---------------------
       * |  4 |  5 |  6 |  7 |               |  4 |  1 |  3 | 13 |
       * ---------------------               ---------------------
       * |  8 |  9 | 10 | 11 |               | 12 |  2 |  0 |  6 |
       * ---------------------               ---------------------
       * | 12 | 13 | 14 | 15 |               | 11 |  8 |  5 | 14 |
       * ---------------------               ---------------------
       *
       * This is equivalent to
       * S' = (0xe58b602cd31479af >> (S * 4)) & 0xf
       */
      nir_ssa_def *frac = nir_ffract(b, sample_coords);
      nir_ssa_def *sample =
         nir_fdot2(b, frac, nir_imm_vec2(b, key->x_scale,
                                            key->x_scale * key->y_scale));
      sample = nir_f2i32(b, sample);

      if (tex_samples == 2) {
         sample = nir_isub(b, nir_imm_int(b, 1), sample);
      } else if (tex_samples == 8) {
         sample = nir_iand(b, nir_ishr(b, nir_imm_int(b, 0x64210573),
                                       nir_ishl(b, sample, nir_imm_int(b, 2))),
                           nir_imm_int(b, 0xf));
      } else if (tex_samples == 16) {
         nir_ssa_def *sample_low =
            nir_iand(b, nir_ishr(b, nir_imm_int(b, 0xd31479af),
                                 nir_ishl(b, sample, nir_imm_int(b, 2))),
                     nir_imm_int(b, 0xf));
         nir_ssa_def *sample_high =
            nir_iand(b, nir_ishr(b, nir_imm_int(b, 0xe58b602c),
                                 nir_ishl(b, nir_iadd(b, sample,
                                                      nir_imm_int(b, -8)),
                                          nir_imm_int(b, 2))),
                     nir_imm_int(b, 0xf));

         sample = nir_bcsel(b, nir_ilt(b, sample, nir_imm_int(b, 8)),
                            sample_low, sample_high);
      }
      nir_ssa_def *pos_ms = nir_vec3(b, nir_channel(b, sample_coords_int, 0),
                                        nir_channel(b, sample_coords_int, 1),
                                        sample);
      tex_data[i] = blorp_nir_txf_ms(b, v, pos_ms, mcs, key->texture_data_type);
   }

   nir_ssa_def *frac_x = nir_channel(b, frac_xy, 0);
   nir_ssa_def *frac_y = nir_channel(b, frac_xy, 1);
   return nir_flrp(b, nir_flrp(b, tex_data[0], tex_data[1], frac_x),
                      nir_flrp(b, tex_data[2], tex_data[3], frac_x),
                      frac_y);
}

/** Perform a color bit-cast operation
 *
 * For copy operations involving CCS, we may need to use different formats for
 * the source and destination surfaces.  The two formats must both be UINT
 * formats and must have the same size but may have different bit layouts.
 * For instance, we may be copying from R8G8B8A8_UINT to R32_UINT or R32_UINT
 * to R16G16_UINT.  This function generates code to shuffle bits around to get
 * us from one to the other.
 */
static nir_ssa_def *
bit_cast_color(struct nir_builder *b, nir_ssa_def *color,
               const struct brw_blorp_blit_prog_key *key)
{
   if (key->src_format == key->dst_format)
      return color;

   const struct isl_format_layout *src_fmtl =
      isl_format_get_layout(key->src_format);
   const struct isl_format_layout *dst_fmtl =
      isl_format_get_layout(key->dst_format);

   /* They must be formats with the same bit size */
   assert(src_fmtl->bpb == dst_fmtl->bpb);

   if (src_fmtl->bpb <= 32) {
      assert(src_fmtl->channels.r.type == ISL_UINT ||
             src_fmtl->channels.r.type == ISL_UNORM);
      assert(dst_fmtl->channels.r.type == ISL_UINT ||
             dst_fmtl->channels.r.type == ISL_UNORM);

      nir_ssa_def *packed = nir_imm_int(b, 0);
      for (unsigned c = 0; c < 4; c++) {
         if (src_fmtl->channels_array[c].bits == 0)
            continue;

         const unsigned chan_start_bit = src_fmtl->channels_array[c].start_bit;
         const unsigned chan_bits = src_fmtl->channels_array[c].bits;

         nir_ssa_def *chan =  nir_channel(b, color, c);
         if (src_fmtl->channels_array[c].type == ISL_UNORM)
            chan = nir_format_float_to_unorm(b, chan, &chan_bits);

         packed = nir_ior(b, packed, nir_shift(b, chan, chan_start_bit));
      }

      nir_ssa_def *chans[4] = { };
      for (unsigned c = 0; c < 4; c++) {
         if (dst_fmtl->channels_array[c].bits == 0) {
            chans[c] = nir_imm_int(b, 0);
            continue;
         }

         const unsigned chan_start_bit = dst_fmtl->channels_array[c].start_bit;
         const unsigned chan_bits = dst_fmtl->channels_array[c].bits;
         chans[c] = nir_iand(b, nir_shift(b, packed, -(int)chan_start_bit),
                                nir_imm_int(b, BITFIELD_MASK(chan_bits)));

         if (dst_fmtl->channels_array[c].type == ISL_UNORM)
            chans[c] = nir_format_unorm_to_float(b, chans[c], &chan_bits);
      }
      color = nir_vec(b, chans, 4);
   } else {
      /* This path only supports UINT formats */
      assert(src_fmtl->channels.r.type == ISL_UINT);
      assert(dst_fmtl->channels.r.type == ISL_UINT);

      const unsigned src_bpc = src_fmtl->channels.r.bits;
      const unsigned dst_bpc = dst_fmtl->channels.r.bits;

      assert(src_fmtl->channels.g.bits == 0 ||
             src_fmtl->channels.g.bits == src_fmtl->channels.r.bits);
      assert(src_fmtl->channels.b.bits == 0 ||
             src_fmtl->channels.b.bits == src_fmtl->channels.r.bits);
      assert(src_fmtl->channels.a.bits == 0 ||
             src_fmtl->channels.a.bits == src_fmtl->channels.r.bits);
      assert(dst_fmtl->channels.g.bits == 0 ||
             dst_fmtl->channels.g.bits == dst_fmtl->channels.r.bits);
      assert(dst_fmtl->channels.b.bits == 0 ||
             dst_fmtl->channels.b.bits == dst_fmtl->channels.r.bits);
      assert(dst_fmtl->channels.a.bits == 0 ||
             dst_fmtl->channels.a.bits == dst_fmtl->channels.r.bits);

      /* Restrict to only the channels we actually have */
      const unsigned src_channels =
         isl_format_get_num_channels(key->src_format);
      color = nir_channels(b, color, (1 << src_channels) - 1);

      color = nir_format_bitcast_uvec_unmasked(b, color, src_bpc, dst_bpc);
   }

   /* Blorp likes to assume that colors are vec4s */
   nir_ssa_def *u = nir_ssa_undef(b, 1, 32);
   nir_ssa_def *chans[4] = { u, u, u, u };
   for (unsigned i = 0; i < color->num_components; i++)
      chans[i] = nir_channel(b, color, i);
   return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
}

static nir_ssa_def *
select_color_channel(struct nir_builder *b, nir_ssa_def *color,
                     nir_alu_type data_type,
                     enum isl_channel_select chan)
{
   if (chan == ISL_CHANNEL_SELECT_ZERO) {
      return nir_imm_int(b, 0);
   } else if (chan == ISL_CHANNEL_SELECT_ONE) {
      switch (data_type) {
      case nir_type_int:
      case nir_type_uint:
         return nir_imm_int(b, 1);
      case nir_type_float:
         return nir_imm_float(b, 1);
      default:
         unreachable("Invalid data type");
      }
   } else {
      assert((unsigned)(chan - ISL_CHANNEL_SELECT_RED) < 4);
      return nir_channel(b, color, chan - ISL_CHANNEL_SELECT_RED);
   }
}

static nir_ssa_def *
swizzle_color(struct nir_builder *b, nir_ssa_def *color,
              struct isl_swizzle swizzle, nir_alu_type data_type)
{
   return nir_vec4(b,
                   select_color_channel(b, color, data_type, swizzle.r),
                   select_color_channel(b, color, data_type, swizzle.g),
                   select_color_channel(b, color, data_type, swizzle.b),
                   select_color_channel(b, color, data_type, swizzle.a));
}

static nir_ssa_def *
convert_color(struct nir_builder *b, nir_ssa_def *color,
              const struct brw_blorp_blit_prog_key *key)
{
   /* All of our color conversions end up generating a single-channel color
    * value that we need to write out.
    */
   nir_ssa_def *value;

   if (key->dst_format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
      /* The destination image is bound as R32_UINT but the data needs to be
       * in R24_UNORM_X8_TYPELESS.  The bottom 24 are the actual data and the
       * top 8 need to be zero.  We can accomplish this by simply multiplying
       * by a factor to scale things down.
       */
      unsigned factor = (1 << 24) - 1;
      value = nir_fsat(b, nir_channel(b, color, 0));
      value = nir_f2i32(b, nir_fmul(b, value, nir_imm_float(b, factor)));
   } else if (key->dst_format == ISL_FORMAT_L8_UNORM_SRGB) {
      value = nir_format_linear_to_srgb(b, nir_channel(b, color, 0));
   } else if (key->dst_format == ISL_FORMAT_R8G8B8_UNORM_SRGB) {
      value = nir_format_linear_to_srgb(b, color);
   } else if (key->dst_format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
      value = nir_format_pack_r9g9b9e5(b, color);
   } else {
      unreachable("Unsupported format conversion");
   }

   nir_ssa_def *out_comps[4];
   for (unsigned i = 0; i < 4; i++) {
      if (i < value->num_components)
         out_comps[i] = nir_channel(b, value, i);
      else
         out_comps[i] = nir_ssa_undef(b, 1, 32);
   }
   return nir_vec(b, out_comps, 4);
}

/**
 * Generator for WM programs used in BLORP blits.
 *
 * The bulk of the work done by the WM program is to wrap and unwrap the
 * coordinate transformations used by the hardware to store surfaces in
 * memory.  The hardware transforms a pixel location (X, Y, S) (where S is the
 * sample index for a multisampled surface) to a memory offset by the
 * following formulas:
 *
 *   offset = tile(tiling_format, encode_msaa(num_samples, layout, X, Y, S))
 *   (X, Y, S) = decode_msaa(num_samples, layout, detile(tiling_format, offset))
 *
 * For a single-sampled surface, or for a multisampled surface using
 * INTEL_MSAA_LAYOUT_UMS, encode_msaa() and decode_msaa are the identity
 * function:
 *
 *   encode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
 *   decode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
 *   encode_msaa(n, UMS, X, Y, S) = (X, Y, S)
 *   decode_msaa(n, UMS, X, Y, S) = (X, Y, S)
 *
 * For a 4x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
 * embeds the sample number into bit 1 of the X and Y coordinates:
 *
 *   encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
 *     where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
 *           Y' = (Y & ~0b1 ) << 1 | (S & 0b10) | (Y & 0b1)
 *   decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
 *     where X' = (X & ~0b11) >> 1 | (X & 0b1)
 *           Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
 *           S = (Y & 0b10) | (X & 0b10) >> 1
 *
 * For an 8x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
 * embeds the sample number into bits 1 and 2 of the X coordinate and bit 1 of
 * the Y coordinate:
 *
 *   encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
 *     where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1 | (X & 0b1)
 *           Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
 *   decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
 *     where X' = (X & ~0b111) >> 2 | (X & 0b1)
 *           Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
 *           S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
 *
 * For X tiling, tile() combines together the low-order bits of the X and Y
 * coordinates in the pattern 0byyyxxxxxxxxx, creating 4k tiles that are 512
 * bytes wide and 8 rows high:
 *
 *   tile(x_tiled, X, Y, S) = A
 *     where A = tile_num << 12 | offset
 *           tile_num = (Y' >> 3) * tile_pitch + (X' >> 9)
 *           offset = (Y' & 0b111) << 9
 *                    | (X & 0b111111111)
 *           X' = X * cpp
 *           Y' = Y + S * qpitch
 *   detile(x_tiled, A) = (X, Y, S)
 *     where X = X' / cpp
 *           Y = Y' % qpitch
 *           S = Y' / qpitch
 *           Y' = (tile_num / tile_pitch) << 3
 *                | (A & 0b111000000000) >> 9
 *           X' = (tile_num % tile_pitch) << 9
 *                | (A & 0b111111111)
 *
 * (In all tiling formulas, cpp is the number of bytes occupied by a single
 * sample ("chars per pixel"), tile_pitch is the number of 4k tiles required
 * to fill the width of the surface, and qpitch is the spacing (in rows)
 * between array slices).
 *
 * For Y tiling, tile() combines together the low-order bits of the X and Y
 * coordinates in the pattern 0bxxxyyyyyxxxx, creating 4k tiles that are 128
 * bytes wide and 32 rows high:
 *
 *   tile(y_tiled, X, Y, S) = A
 *     where A = tile_num << 12 | offset
 *           tile_num = (Y' >> 5) * tile_pitch + (X' >> 7)
 *           offset = (X' & 0b1110000) << 5
 *                    | (Y' & 0b11111) << 4
 *                    | (X' & 0b1111)
 *           X' = X * cpp
 *           Y' = Y + S * qpitch
 *   detile(y_tiled, A) = (X, Y, S)
 *     where X = X' / cpp
 *           Y = Y' % qpitch
 *           S = Y' / qpitch
 *           Y' = (tile_num / tile_pitch) << 5
 *                | (A & 0b111110000) >> 4
 *           X' = (tile_num % tile_pitch) << 7
 *                | (A & 0b111000000000) >> 5
 *                | (A & 0b1111)
 *
 * For W tiling, tile() combines together the low-order bits of the X and Y
 * coordinates in the pattern 0bxxxyyyyxyxyx, creating 4k tiles that are 64
 * bytes wide and 64 rows high (note that W tiling is only used for stencil
 * buffers, which always have cpp = 1 and S=0):
 *
 *   tile(w_tiled, X, Y, S) = A
 *     where A = tile_num << 12 | offset
 *           tile_num = (Y' >> 6) * tile_pitch + (X' >> 6)
 *           offset = (X' & 0b111000) << 6
 *                    | (Y' & 0b111100) << 3
 *                    | (X' & 0b100) << 2
 *                    | (Y' & 0b10) << 2
 *                    | (X' & 0b10) << 1
 *                    | (Y' & 0b1) << 1
 *                    | (X' & 0b1)
 *           X' = X * cpp = X
 *           Y' = Y + S * qpitch
 *   detile(w_tiled, A) = (X, Y, S)
 *     where X = X' / cpp = X'
 *           Y = Y' % qpitch = Y'
 *           S = Y / qpitch = 0
 *           Y' = (tile_num / tile_pitch) << 6
 *                | (A & 0b111100000) >> 3
 *                | (A & 0b1000) >> 2
 *                | (A & 0b10) >> 1
 *           X' = (tile_num % tile_pitch) << 6
 *                | (A & 0b111000000000) >> 6
 *                | (A & 0b10000) >> 2
 *                | (A & 0b100) >> 1
 *                | (A & 0b1)
 *
 * Finally, for a non-tiled surface, tile() simply combines together the X and
 * Y coordinates in the natural way:
 *
 *   tile(untiled, X, Y, S) = A
 *     where A = Y * pitch + X'
 *           X' = X * cpp
 *           Y' = Y + S * qpitch
 *   detile(untiled, A) = (X, Y, S)
 *     where X = X' / cpp
 *           Y = Y' % qpitch
 *           S = Y' / qpitch
 *           X' = A % pitch
 *           Y' = A / pitch
 *
 * (In these formulas, pitch is the number of bytes occupied by a single row
 * of samples).
 */
static nir_shader *
brw_blorp_build_nir_shader(struct blorp_context *blorp, void *mem_ctx,
                           const struct brw_blorp_blit_prog_key *key)
{
   const struct gen_device_info *devinfo = blorp->isl_dev->info;
   nir_ssa_def *src_pos, *dst_pos, *color;

   /* Sanity checks */
   if (key->dst_tiled_w && key->rt_samples > 1) {
      /* If the destination image is W tiled and multisampled, then the thread
       * must be dispatched once per sample, not once per pixel.  This is
       * necessary because after conversion between W and Y tiling, there's no
       * guarantee that all samples corresponding to a single pixel will still
       * be together.
       */
      assert(key->persample_msaa_dispatch);
   }

   if (key->persample_msaa_dispatch) {
      /* It only makes sense to do persample dispatch if the render target is
       * configured as multisampled.
       */
      assert(key->rt_samples > 0);
   }

   /* Make sure layout is consistent with sample count */
   assert((key->tex_layout == ISL_MSAA_LAYOUT_NONE) ==
          (key->tex_samples <= 1));
   assert((key->rt_layout == ISL_MSAA_LAYOUT_NONE) ==
          (key->rt_samples <= 1));
   assert((key->src_layout == ISL_MSAA_LAYOUT_NONE) ==
          (key->src_samples <= 1));
   assert((key->dst_layout == ISL_MSAA_LAYOUT_NONE) ==
          (key->dst_samples <= 1));

   nir_builder b;
   blorp_nir_init_shader(&b, mem_ctx, MESA_SHADER_FRAGMENT, NULL);

   struct brw_blorp_blit_vars v;
   brw_blorp_blit_vars_init(&b, &v, key);

   dst_pos = blorp_blit_get_frag_coords(&b, key, &v);

   /* Render target and texture hardware don't support W tiling until Gfx8. */
   const bool rt_tiled_w = false;
   const bool tex_tiled_w = devinfo->ver >= 8 && key->src_tiled_w;

   /* The address that data will be written to is determined by the
    * coordinates supplied to the WM thread and the tiling and sample count of
    * the render target, according to the formula:
    *
    * (X, Y, S) = decode_msaa(rt_samples, detile(rt_tiling, offset))
    *
    * If the actual tiling and sample count of the destination surface are not
    * the same as the configuration of the render target, then these
    * coordinates are wrong and we have to adjust them to compensate for the
    * difference.
    */
   if (rt_tiled_w != key->dst_tiled_w ||
       key->rt_samples != key->dst_samples ||
       key->rt_layout != key->dst_layout) {
      dst_pos = blorp_nir_encode_msaa(&b, dst_pos, key->rt_samples,
                                      key->rt_layout);
      /* Now (X, Y, S) = detile(rt_tiling, offset) */
      if (rt_tiled_w != key->dst_tiled_w)
         dst_pos = blorp_nir_retile_y_to_w(&b, dst_pos);
      /* Now (X, Y, S) = detile(rt_tiling, offset) */
      dst_pos = blorp_nir_decode_msaa(&b, dst_pos, key->dst_samples,
                                      key->dst_layout);
   }

   nir_ssa_def *comp = NULL;
   if (key->dst_rgb) {
      /* The destination image is bound as a red texture three times as wide
       * as the actual image.  Our shader is effectively running one color
       * component at a time.  We need to save off the component and adjust
       * the destination position.
       */
      assert(dst_pos->num_components == 2);
      nir_ssa_def *dst_x = nir_channel(&b, dst_pos, 0);
      comp = nir_umod(&b, dst_x, nir_imm_int(&b, 3));
      dst_pos = nir_vec2(&b, nir_idiv(&b, dst_x, nir_imm_int(&b, 3)),
                             nir_channel(&b, dst_pos, 1));
   }

   /* Now (X, Y, S) = decode_msaa(dst_samples, detile(dst_tiling, offset)).
    *
    * That is: X, Y and S now contain the true coordinates and sample index of
    * the data that the WM thread should output.
    *
    * If we need to kill pixels that are outside the destination rectangle,
    * now is the time to do it.
    */
   if (key->use_kill)
      blorp_nir_discard_if_outside_rect(&b, dst_pos, &v);

   src_pos = blorp_blit_apply_transform(&b, nir_i2f32(&b, dst_pos), &v);
   if (dst_pos->num_components == 3) {
      /* The sample coordinate is an integer that we want left alone but
       * blorp_blit_apply_transform() blindly applies the transform to all
       * three coordinates.  Grab the original sample index.
       */
      src_pos = nir_vec3(&b, nir_channel(&b, src_pos, 0),
                             nir_channel(&b, src_pos, 1),
                             nir_channel(&b, dst_pos, 2));
   }

   /* If the source image is not multisampled, then we want to fetch sample
    * number 0, because that's the only sample there is.
    */
   if (key->src_samples == 1)
      src_pos = nir_channels(&b, src_pos, 0x3);

   /* X, Y, and S are now the coordinates of the pixel in the source image
    * that we want to texture from.  Exception: if we are blending, then S is
    * irrelevant, because we are going to fetch all samples.
    */
   switch (key->filter) {
   case BLORP_FILTER_NONE:
   case BLORP_FILTER_NEAREST:
   case BLORP_FILTER_SAMPLE_0:
      /* We're going to use texelFetch, so we need integers */
      if (src_pos->num_components == 2) {
         src_pos = nir_f2i32(&b, src_pos);
      } else {
         assert(src_pos->num_components == 3);
         src_pos = nir_vec3(&b, nir_channel(&b, nir_f2i32(&b, src_pos), 0),
                                nir_channel(&b, nir_f2i32(&b, src_pos), 1),
                                nir_channel(&b, src_pos, 2));
      }

      /* We aren't blending, which means we just want to fetch a single
       * sample from the source surface.  The address that we want to fetch
       * from is related to the X, Y and S values according to the formula:
       *
       * (X, Y, S) = decode_msaa(src_samples, detile(src_tiling, offset)).
       *
       * If the actual tiling and sample count of the source surface are
       * not the same as the configuration of the texture, then we need to
       * adjust the coordinates to compensate for the difference.
       */
      if (tex_tiled_w != key->src_tiled_w ||
          key->tex_samples != key->src_samples ||
          key->tex_layout != key->src_layout) {
         src_pos = blorp_nir_encode_msaa(&b, src_pos, key->src_samples,
                                         key->src_layout);
         /* Now (X, Y, S) = detile(src_tiling, offset) */
         if (tex_tiled_w != key->src_tiled_w)
            src_pos = blorp_nir_retile_w_to_y(&b, src_pos);
         /* Now (X, Y, S) = detile(tex_tiling, offset) */
         src_pos = blorp_nir_decode_msaa(&b, src_pos, key->tex_samples,
                                         key->tex_layout);
      }

      if (key->need_src_offset)
         src_pos = nir_iadd(&b, src_pos, nir_load_var(&b, v.v_src_offset));

      /* Now (X, Y, S) = decode_msaa(tex_samples, detile(tex_tiling, offset)).
       *
       * In other words: X, Y, and S now contain values which, when passed to
       * the texturing unit, will cause data to be read from the correct
       * memory location.  So we can fetch the texel now.
       */
      if (key->src_samples == 1) {
         color = blorp_nir_txf(&b, &v, src_pos, key->texture_data_type);
      } else {
         nir_ssa_def *mcs = NULL;
         if (isl_aux_usage_has_mcs(key->tex_aux_usage))
            mcs = blorp_blit_txf_ms_mcs(&b, &v, src_pos);

         color = blorp_nir_txf_ms(&b, &v, src_pos, mcs, key->texture_data_type);
      }
      break;

   case BLORP_FILTER_BILINEAR:
      assert(!key->src_tiled_w);
      assert(key->tex_samples == key->src_samples);
      assert(key->tex_layout == key->src_layout);

      if (key->src_samples == 1) {
         color = blorp_nir_tex(&b, &v, key, src_pos);
      } else {
         assert(!key->use_kill);
         color = blorp_nir_manual_blend_bilinear(&b, src_pos, key->src_samples,
                                                 key, &v);
      }
      break;

   case BLORP_FILTER_AVERAGE:
   case BLORP_FILTER_MIN_SAMPLE:
   case BLORP_FILTER_MAX_SAMPLE:
      assert(!key->src_tiled_w);
      assert(key->tex_samples == key->src_samples);
      assert(key->tex_layout == key->src_layout);

      /* Resolves (effecively) use texelFetch, so we need integers and we
       * don't care about the sample index if we got one.
       */
      src_pos = nir_f2i32(&b, nir_channels(&b, src_pos, 0x3));

      if (devinfo->ver == 6) {
         /* Because gfx6 only supports 4x interleved MSAA, we can do all the
          * blending we need with a single linear-interpolated texture lookup
          * at the center of the sample. The texture coordinates to be odd
          * integers so that they correspond to the center of a 2x2 block
          * representing the four samples that maxe up a pixel.  So we need
          * to multiply our X and Y coordinates each by 2 and then add 1.
          */
         assert(key->src_coords_normalized);
         assert(key->filter == BLORP_FILTER_AVERAGE);
         src_pos = nir_fadd(&b,
                            nir_i2f32(&b, src_pos),
                            nir_imm_float(&b, 0.5f));
         color = blorp_nir_tex(&b, &v, key, src_pos);
      } else {
         /* Gfx7+ hardware doesn't automaticaly blend. */
         color = blorp_nir_combine_samples(&b, &v, src_pos, key->src_samples,
                                           key->tex_aux_usage,
                                           key->texture_data_type,
                                           key->filter);
      }
      break;

   default:
      unreachable("Invalid blorp filter");
   }

   if (!isl_swizzle_is_identity(key->src_swizzle)) {
      color = swizzle_color(&b, color, key->src_swizzle,
                            key->texture_data_type);
   }

   if (!isl_swizzle_is_identity(key->dst_swizzle)) {
      color = swizzle_color(&b, color, isl_swizzle_invert(key->dst_swizzle),
                            nir_type_int);
   }

   if (key->format_bit_cast) {
      assert(isl_swizzle_is_identity(key->src_swizzle));
      assert(isl_swizzle_is_identity(key->dst_swizzle));
      color = bit_cast_color(&b, color, key);
   } else if (key->dst_format) {
      color = convert_color(&b, color, key);
   } else if (key->uint32_to_sint) {
      /* Normally the hardware will take care of converting values from/to
       * the source and destination formats.  But a few cases need help.
       *
       * The Skylake PRM, volume 07, page 658 has a programming note:
       *
       *    "When using SINT or UINT rendertarget surface formats, Blending
       *     must be DISABLED. The Pre-Blend Color Clamp Enable and Color
       *     Clamp Range fields are ignored, and an implied clamp to the
       *     rendertarget surface format is performed."
       *
       * For UINT to SINT blits, our sample operation gives us a uint32_t,
       * but our render target write expects a signed int32_t number.  If we
       * simply passed the value along, the hardware would interpret a value
       * with bit 31 set as a negative value, clamping it to the largest
       * negative number the destination format could represent.  But the
       * actual source value is a positive number, so we want to clamp it
       * to INT_MAX.  To fix this, we explicitly take min(color, INT_MAX).
       */
      color = nir_umin(&b, color, nir_imm_int(&b, INT32_MAX));
   } else if (key->sint32_to_uint) {
      /* Similar to above, but clamping negative numbers to zero. */
      color = nir_imax(&b, color, nir_imm_int(&b, 0));
   }

   if (key->dst_rgb) {
      /* The destination image is bound as a red texture three times as wide
       * as the actual image.  Our shader is effectively running one color
       * component at a time.  We need to pick off the appropriate component
       * from the source color and write that to destination red.
       */
      assert(dst_pos->num_components == 2);

      nir_ssa_def *color_component =
         nir_bcsel(&b, nir_ieq_imm(&b, comp, 0),
                       nir_channel(&b, color, 0),
                       nir_bcsel(&b, nir_ieq_imm(&b, comp, 1),
                                     nir_channel(&b, color, 1),
                                     nir_channel(&b, color, 2)));

      nir_ssa_def *u = nir_ssa_undef(&b, 1, 32);
      color = nir_vec4(&b, color_component, u, u, u);
   }

   if (key->dst_usage == ISL_SURF_USAGE_RENDER_TARGET_BIT) {
      nir_variable *color_out =
         nir_variable_create(b.shader, nir_var_shader_out,
                             glsl_vec4_type(), "gl_FragColor");
      color_out->data.location = FRAG_RESULT_COLOR;
      nir_store_var(&b, color_out, color, 0xf);
   } else if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
      nir_variable *depth_out =
         nir_variable_create(b.shader, nir_var_shader_out,
                             glsl_float_type(), "gl_FragDepth");
      depth_out->data.location = FRAG_RESULT_DEPTH;
      nir_store_var(&b, depth_out, nir_channel(&b, color, 0), 0x1);
   } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
      nir_variable *stencil_out =
         nir_variable_create(b.shader, nir_var_shader_out,
                             glsl_int_type(), "gl_FragStencilRef");
      stencil_out->data.location = FRAG_RESULT_STENCIL;
      nir_store_var(&b, stencil_out, nir_channel(&b, color, 0), 0x1);
   } else {
      unreachable("Invalid destination usage");
   }

   return b.shader;
}

static bool
brw_blorp_get_blit_kernel(struct blorp_batch *batch,
                          struct blorp_params *params,
                          const struct brw_blorp_blit_prog_key *prog_key)
{
   struct blorp_context *blorp = batch->blorp;

   if (blorp->lookup_shader(batch, prog_key, sizeof(*prog_key),
                            &params->wm_prog_kernel, &params->wm_prog_data))
      return true;

   void *mem_ctx = ralloc_context(NULL);

   const unsigned *program;
   struct brw_wm_prog_data prog_data;

   nir_shader *nir = brw_blorp_build_nir_shader(blorp, mem_ctx, prog_key);
   nir->info.name = ralloc_strdup(nir, blorp_shader_type_to_name(prog_key->shader_type));

   struct brw_wm_prog_key wm_key;
   brw_blorp_init_wm_prog_key(&wm_key);
   wm_key.base.tex.compressed_multisample_layout_mask =
      isl_aux_usage_has_mcs(prog_key->tex_aux_usage);
   wm_key.base.tex.msaa_16 = prog_key->tex_samples == 16;
   wm_key.multisample_fbo = prog_key->rt_samples > 1;

   program = blorp_compile_fs(blorp, mem_ctx, nir, &wm_key, false,
                              &prog_data);

   bool result =
      blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
                           prog_key, sizeof(*prog_key),
                           program, prog_data.base.program_size,
                           &prog_data.base, sizeof(prog_data),
                           &params->wm_prog_kernel, &params->wm_prog_data);

   ralloc_free(mem_ctx);
   return result;
}

static void
brw_blorp_setup_coord_transform(struct brw_blorp_coord_transform *xform,
                                GLfloat src0, GLfloat src1,
                                GLfloat dst0, GLfloat dst1,
                                bool mirror)
{
   double scale = (double)(src1 - src0) / (double)(dst1 - dst0);
   if (!mirror) {
      /* When not mirroring a coordinate (say, X), we need:
       *   src_x - src_x0 = (dst_x - dst_x0 + 0.5) * scale
       * Therefore:
       *   src_x = src_x0 + (dst_x - dst_x0 + 0.5) * scale
       *
       * blorp program uses "round toward zero" to convert the
       * transformed floating point coordinates to integer coordinates,
       * whereas the behaviour we actually want is "round to nearest",
       * so 0.5 provides the necessary correction.
       */
      xform->multiplier = scale;
      xform->offset = src0 + (-(double)dst0 + 0.5) * scale;
   } else {
      /* When mirroring X we need:
       *   src_x - src_x0 = dst_x1 - dst_x - 0.5
       * Therefore:
       *   src_x = src_x0 + (dst_x1 -dst_x - 0.5) * scale
       */
      xform->multiplier = -scale;
      xform->offset = src0 + ((double)dst1 - 0.5) * scale;
   }
}

static inline void
surf_get_intratile_offset_px(struct brw_blorp_surface_info *info,
                             uint32_t *tile_x_px, uint32_t *tile_y_px)
{
   if (info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
      struct isl_extent2d px_size_sa =
         isl_get_interleaved_msaa_px_size_sa(info->surf.samples);
      assert(info->tile_x_sa % px_size_sa.width == 0);
      assert(info->tile_y_sa % px_size_sa.height == 0);
      *tile_x_px = info->tile_x_sa / px_size_sa.width;
      *tile_y_px = info->tile_y_sa / px_size_sa.height;
   } else {
      *tile_x_px = info->tile_x_sa;
      *tile_y_px = info->tile_y_sa;
   }
}

void
blorp_surf_convert_to_single_slice(const struct isl_device *isl_dev,
                                   struct brw_blorp_surface_info *info)
{
   bool ok UNUSED;

   /* It would be insane to try and do this on a compressed surface */
   assert(info->aux_usage == ISL_AUX_USAGE_NONE);

   /* Just bail if we have nothing to do. */
   if (info->surf.dim == ISL_SURF_DIM_2D &&
       info->view.base_level == 0 && info->view.base_array_layer == 0 &&
       info->surf.levels == 1 && info->surf.logical_level0_px.array_len == 1)
      return;

   /* If this gets triggered then we've gotten here twice which.  This
    * shouldn't happen thanks to the above early return.
    */
   assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);

   uint32_t layer = 0, z = 0;
   if (info->surf.dim == ISL_SURF_DIM_3D)
      z = info->view.base_array_layer + info->z_offset;
   else
      layer = info->view.base_array_layer;

   uint32_t byte_offset;
   isl_surf_get_image_surf(isl_dev, &info->surf,
                           info->view.base_level, layer, z,
                           &info->surf,
                           &byte_offset, &info->tile_x_sa, &info->tile_y_sa);
   info->addr.offset += byte_offset;

   uint32_t tile_x_px, tile_y_px;
   surf_get_intratile_offset_px(info, &tile_x_px, &tile_y_px);

   /* Instead of using the X/Y Offset fields in RENDER_SURFACE_STATE, we place
    * the image at the tile boundary and offset our sampling or rendering.
    * For this reason, we need to grow the image by the offset to ensure that
    * the hardware doesn't think we've gone past the edge.
    */
   info->surf.logical_level0_px.w += tile_x_px;
   info->surf.logical_level0_px.h += tile_y_px;
   info->surf.phys_level0_sa.w += info->tile_x_sa;
   info->surf.phys_level0_sa.h += info->tile_y_sa;

   /* The view is also different now. */
   info->view.base_level = 0;
   info->view.levels = 1;
   info->view.base_array_layer = 0;
   info->view.array_len = 1;
   info->z_offset = 0;
}

void
blorp_surf_fake_interleaved_msaa(const struct isl_device *isl_dev,
                                 struct brw_blorp_surface_info *info)
{
   assert(info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED);

   /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
   blorp_surf_convert_to_single_slice(isl_dev, info);

   info->surf.logical_level0_px = info->surf.phys_level0_sa;
   info->surf.samples = 1;
   info->surf.msaa_layout = ISL_MSAA_LAYOUT_NONE;
}

void
blorp_surf_retile_w_to_y(const struct isl_device *isl_dev,
                         struct brw_blorp_surface_info *info)
{
   assert(info->surf.tiling == ISL_TILING_W);

   /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
   blorp_surf_convert_to_single_slice(isl_dev, info);

   /* On gfx7+, we don't have interleaved multisampling for color render
    * targets so we have to fake it.
    *
    * TODO: Are we sure we don't also need to fake it on gfx6?
    */
   if (isl_dev->info->ver > 6 &&
       info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
      blorp_surf_fake_interleaved_msaa(isl_dev, info);
   }

   if (isl_dev->info->ver == 6) {
      /* Gfx6 stencil buffers have a very large alignment coming in from the
       * miptree.  It's out-of-bounds for what the surface state can handle.
       * Since we have a single layer and level, it doesn't really matter as
       * long as we don't pass a bogus value into isl_surf_fill_state().
       */
      info->surf.image_alignment_el = isl_extent3d(4, 2, 1);
   }

   /* Now that we've converted everything to a simple 2-D surface with only
    * one miplevel, we can go about retiling it.
    */
   const unsigned x_align = 8, y_align = info->surf.samples != 0 ? 8 : 4;
   info->surf.tiling = ISL_TILING_Y0;
   info->surf.logical_level0_px.width =
      ALIGN(info->surf.logical_level0_px.width, x_align) * 2;
   info->surf.logical_level0_px.height =
      ALIGN(info->surf.logical_level0_px.height, y_align) / 2;
   info->tile_x_sa *= 2;
   info->tile_y_sa /= 2;
}

static bool
can_shrink_surface(const struct brw_blorp_surface_info *surf)
{
   /* The current code doesn't support offsets into the aux buffers. This
    * should be possible, but we need to make sure the offset is page
    * aligned for both the surface and the aux buffer surface. Generally
    * this mean using the page aligned offset for the aux buffer.
    *
    * Currently the cases where we must split the blit are limited to cases
    * where we don't have a aux buffer.
    */
   if (surf->aux_addr.buffer != NULL)
      return false;

   /* We can't support splitting the blit for gen <= 7, because the qpitch
    * size is calculated by the hardware based on the surface height for
    * gen <= 7. In gen >= 8, the qpitch is controlled by the driver.
    */
   if (surf->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY)
      return false;

   return true;
}

static unsigned
get_max_surface_size(const struct gen_device_info *devinfo,
                     const struct brw_blorp_surface_info *surf)
{
   const unsigned max = devinfo->ver >= 7 ? 16384 : 8192;
   if (split_blorp_blit_debug && can_shrink_surface(surf))
      return max >> 4; /* A smaller restriction when debug is enabled */
   else
      return max;
}

struct blt_axis {
   double src0, src1, dst0, dst1;
   bool mirror;
};

struct blt_coords {
   struct blt_axis x, y;
};

static enum isl_format
get_red_format_for_rgb_format(enum isl_format format)
{
   const struct isl_format_layout *fmtl = isl_format_get_layout(format);

   switch (fmtl->channels.r.bits) {
   case 8:
      switch (fmtl->channels.r.type) {
      case ISL_UNORM:
         return ISL_FORMAT_R8_UNORM;
      case ISL_SNORM:
         return ISL_FORMAT_R8_SNORM;
      case ISL_UINT:
         return ISL_FORMAT_R8_UINT;
      case ISL_SINT:
         return ISL_FORMAT_R8_SINT;
      default:
         unreachable("Invalid 8-bit RGB channel type");
      }
   case 16:
      switch (fmtl->channels.r.type) {
      case ISL_UNORM:
         return ISL_FORMAT_R16_UNORM;
      case ISL_SNORM:
         return ISL_FORMAT_R16_SNORM;
      case ISL_SFLOAT:
         return ISL_FORMAT_R16_FLOAT;
      case ISL_UINT:
         return ISL_FORMAT_R16_UINT;
      case ISL_SINT:
         return ISL_FORMAT_R16_SINT;
      default:
         unreachable("Invalid 8-bit RGB channel type");
      }
   case 32:
      switch (fmtl->channels.r.type) {
      case ISL_SFLOAT:
         return ISL_FORMAT_R32_FLOAT;
      case ISL_UINT:
         return ISL_FORMAT_R32_UINT;
      case ISL_SINT:
         return ISL_FORMAT_R32_SINT;
      default:
         unreachable("Invalid 8-bit RGB channel type");
      }
   default:
      unreachable("Invalid number of red channel bits");
   }
}

void
surf_fake_rgb_with_red(const struct isl_device *isl_dev,
                       struct brw_blorp_surface_info *info)
{
   blorp_surf_convert_to_single_slice(isl_dev, info);

   info->surf.logical_level0_px.width *= 3;
   info->surf.phys_level0_sa.width *= 3;
   info->tile_x_sa *= 3;

   enum isl_format red_format =
      get_red_format_for_rgb_format(info->view.format);

   assert(isl_format_get_layout(red_format)->channels.r.type ==
          isl_format_get_layout(info->view.format)->channels.r.type);
   assert(isl_format_get_layout(red_format)->channels.r.bits ==
          isl_format_get_layout(info->view.format)->channels.r.bits);

   info->surf.format = info->view.format = red_format;
}

enum blit_shrink_status {
   BLIT_NO_SHRINK = 0,
   BLIT_SRC_WIDTH_SHRINK   = (1 << 0),
   BLIT_DST_WIDTH_SHRINK   = (1 << 1),
   BLIT_SRC_HEIGHT_SHRINK  = (1 << 2),
   BLIT_DST_HEIGHT_SHRINK  = (1 << 3),
};

/* Try to blit. If the surface parameters exceed the size allowed by hardware,
 * then enum blit_shrink_status will be returned. If BLIT_NO_SHRINK is
 * returned, then the blit was successful.
 */
static enum blit_shrink_status
try_blorp_blit(struct blorp_batch *batch,
               struct blorp_params *params,
               struct brw_blorp_blit_prog_key *wm_prog_key,
               struct blt_coords *coords)
{
   const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;

   if (params->dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
      if (devinfo->ver >= 7) {
         /* We can render as depth on Gfx5 but there's no real advantage since
          * it doesn't support MSAA or HiZ.  On Gfx4, we can't always render
          * to depth due to issues with depth buffers and mip-mapping.  On
          * Gfx6, we can do everything but we have weird offsetting for HiZ
          * and stencil.  It's easier to just render using the color pipe
          * on those platforms.
          */
         wm_prog_key->dst_usage = ISL_SURF_USAGE_DEPTH_BIT;
      } else {
         wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
      }
   } else if (params->dst.surf.usage & ISL_SURF_USAGE_STENCIL_BIT) {
      assert(params->dst.surf.format == ISL_FORMAT_R8_UINT);
      if (devinfo->ver >= 9) {
         wm_prog_key->dst_usage = ISL_SURF_USAGE_STENCIL_BIT;
      } else {
         wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
      }
   } else {
      wm_prog_key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
   }

   if (isl_format_has_sint_channel(params->src.view.format)) {
      wm_prog_key->texture_data_type = nir_type_int;
   } else if (isl_format_has_uint_channel(params->src.view.format)) {
      wm_prog_key->texture_data_type = nir_type_uint;
   } else {
      wm_prog_key->texture_data_type = nir_type_float;
   }

   /* src_samples and dst_samples are the true sample counts */
   wm_prog_key->src_samples = params->src.surf.samples;
   wm_prog_key->dst_samples = params->dst.surf.samples;

   wm_prog_key->tex_aux_usage = params->src.aux_usage;

   /* src_layout and dst_layout indicate the true MSAA layout used by src and
    * dst.
    */
   wm_prog_key->src_layout = params->src.surf.msaa_layout;
   wm_prog_key->dst_layout = params->dst.surf.msaa_layout;

   /* Round floating point values to nearest integer to avoid "off by one texel"
    * kind of errors when blitting.
    */
   params->x0 = params->wm_inputs.discard_rect.x0 = round(coords->x.dst0);
   params->y0 = params->wm_inputs.discard_rect.y0 = round(coords->y.dst0);
   params->x1 = params->wm_inputs.discard_rect.x1 = round(coords->x.dst1);
   params->y1 = params->wm_inputs.discard_rect.y1 = round(coords->y.dst1);

   brw_blorp_setup_coord_transform(&params->wm_inputs.coord_transform[0],
                                   coords->x.src0, coords->x.src1,
                                   coords->x.dst0, coords->x.dst1,
                                   coords->x.mirror);
   brw_blorp_setup_coord_transform(&params->wm_inputs.coord_transform[1],
                                   coords->y.src0, coords->y.src1,
                                   coords->y.dst0, coords->y.dst1,
                                   coords->y.mirror);


   if (devinfo->ver == 4) {
      /* The MinLOD and MinimumArrayElement don't work properly for cube maps.
       * Convert them to a single slice on gfx4.
       */
      if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
         blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, &params->dst);
         wm_prog_key->need_dst_offset = true;
      }

      if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
         blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, &params->src);
         wm_prog_key->need_src_offset = true;
      }
   }

   if (devinfo->ver > 6 &&
       !isl_surf_usage_is_depth_or_stencil(wm_prog_key->dst_usage) &&
       params->dst.surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
      assert(params->dst.surf.samples > 1);

      /* We must expand the rectangle we send through the rendering pipeline,
       * to account for the fact that we are mapping the destination region as
       * single-sampled when it is in fact multisampled.  We must also align
       * it to a multiple of the multisampling pattern, because the
       * differences between multisampled and single-sampled surface formats
       * will mean that pixels are scrambled within the multisampling pattern.
       * TODO: what if this makes the coordinates too large?
       *
       * Note: this only works if the destination surface uses the IMS layout.
       * If it's UMS, then we have no choice but to set up the rendering
       * pipeline as multisampled.
       */
      struct isl_extent2d px_size_sa =
         isl_get_interleaved_msaa_px_size_sa(params->dst.surf.samples);
      params->x0 = ROUND_DOWN_TO(params->x0, 2) * px_size_sa.width;
      params->y0 = ROUND_DOWN_TO(params->y0, 2) * px_size_sa.height;
      params->x1 = ALIGN(params->x1, 2) * px_size_sa.width;
      params->y1 = ALIGN(params->y1, 2) * px_size_sa.height;

      blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, &params->dst);

      wm_prog_key->use_kill = true;
      wm_prog_key->need_dst_offset = true;
   }

   if (params->dst.surf.tiling == ISL_TILING_W &&
       wm_prog_key->dst_usage != ISL_SURF_USAGE_STENCIL_BIT) {
      /* We must modify the rectangle we send through the rendering pipeline
       * (and the size and x/y offset of the destination surface), to account
       * for the fact that we are mapping it as Y-tiled when it is in fact
       * W-tiled.
       *
       * Both Y tiling and W tiling can be understood as organizations of
       * 32-byte sub-tiles; within each 32-byte sub-tile, the layout of pixels
       * is different, but the layout of the 32-byte sub-tiles within the 4k
       * tile is the same (8 sub-tiles across by 16 sub-tiles down, in
       * column-major order).  In Y tiling, the sub-tiles are 16 bytes wide
       * and 2 rows high; in W tiling, they are 8 bytes wide and 4 rows high.
       *
       * Therefore, to account for the layout differences within the 32-byte
       * sub-tiles, we must expand the rectangle so the X coordinates of its
       * edges are multiples of 8 (the W sub-tile width), and its Y
       * coordinates of its edges are multiples of 4 (the W sub-tile height).
       * Then we need to scale the X and Y coordinates of the rectangle to
       * account for the differences in aspect ratio between the Y and W
       * sub-tiles.  We need to modify the layer width and height similarly.
       *
       * A correction needs to be applied when MSAA is in use: since
       * INTEL_MSAA_LAYOUT_IMS uses an interleaving pattern whose height is 4,
       * we need to align the Y coordinates to multiples of 8, so that when
       * they are divided by two they are still multiples of 4.
       *
       * Note: Since the x/y offset of the surface will be applied using the
       * SURFACE_STATE command packet, it will be invisible to the swizzling
       * code in the shader; therefore it needs to be in a multiple of the
       * 32-byte sub-tile size.  Fortunately it is, since the sub-tile is 8
       * pixels wide and 4 pixels high (when viewed as a W-tiled stencil
       * buffer), and the miplevel alignment used for stencil buffers is 8
       * pixels horizontally and either 4 or 8 pixels vertically (see
       * intel_horizontal_texture_alignment_unit() and
       * intel_vertical_texture_alignment_unit()).
       *
       * Note: Also, since the SURFACE_STATE command packet can only apply
       * offsets that are multiples of 4 pixels horizontally and 2 pixels
       * vertically, it is important that the offsets will be multiples of
       * these sizes after they are converted into Y-tiled coordinates.
       * Fortunately they will be, since we know from above that the offsets
       * are a multiple of the 32-byte sub-tile size, and in Y-tiled
       * coordinates the sub-tile is 16 pixels wide and 2 pixels high.
       *
       * TODO: what if this makes the coordinates (or the texture size) too
       * large?
       */
      const unsigned x_align = 8;
      const unsigned y_align = params->dst.surf.samples != 0 ? 8 : 4;
      params->x0 = ROUND_DOWN_TO(params->x0, x_align) * 2;
      params->y0 = ROUND_DOWN_TO(params->y0, y_align) / 2;
      params->x1 = ALIGN(params->x1, x_align) * 2;
      params->y1 = ALIGN(params->y1, y_align) / 2;

      /* Retile the surface to Y-tiled */
      blorp_surf_retile_w_to_y(batch->blorp->isl_dev, &params->dst);

      wm_prog_key->dst_tiled_w = true;
      wm_prog_key->use_kill = true;
      wm_prog_key->need_dst_offset = true;

      if (params->dst.surf.samples > 1) {
         /* If the destination surface is a W-tiled multisampled stencil
          * buffer that we're mapping as Y tiled, then we need to arrange for
          * the WM program to run once per sample rather than once per pixel,
          * because the memory layout of related samples doesn't match between
          * W and Y tiling.
          */
         wm_prog_key->persample_msaa_dispatch = true;
      }
   }

   if (devinfo->ver < 8 && params->src.surf.tiling == ISL_TILING_W) {
      /* On Haswell and earlier, we have to fake W-tiled sources as Y-tiled.
       * Broadwell adds support for sampling from stencil.
       *
       * See the comments above concerning x/y offset alignment for the
       * destination surface.
       *
       * TODO: what if this makes the texture size too large?
       */
      blorp_surf_retile_w_to_y(batch->blorp->isl_dev, &params->src);

      wm_prog_key->src_tiled_w = true;
      wm_prog_key->need_src_offset = true;
   }

   /* tex_samples and rt_samples are the sample counts that are set up in
    * SURFACE_STATE.
    */
   wm_prog_key->tex_samples = params->src.surf.samples;
   wm_prog_key->rt_samples  = params->dst.surf.samples;

   /* tex_layout and rt_layout indicate the MSAA layout the GPU pipeline will
    * use to access the source and destination surfaces.
    */
   wm_prog_key->tex_layout = params->src.surf.msaa_layout;
   wm_prog_key->rt_layout = params->dst.surf.msaa_layout;

   if (params->src.surf.samples > 0 && params->dst.surf.samples > 1) {
      /* We are blitting from a multisample buffer to a multisample buffer, so
       * we must preserve samples within a pixel.  This means we have to
       * arrange for the WM program to run once per sample rather than once
       * per pixel.
       */
      wm_prog_key->persample_msaa_dispatch = true;
   }

   params->num_samples = params->dst.surf.samples;

   if ((wm_prog_key->filter == BLORP_FILTER_AVERAGE ||
        wm_prog_key->filter == BLORP_FILTER_BILINEAR) &&
       batch->blorp->isl_dev->info->ver <= 6) {
      /* Gfx4-5 don't support non-normalized texture coordinates */
      wm_prog_key->src_coords_normalized = true;
      params->wm_inputs.src_inv_size[0] =
         1.0f / minify(params->src.surf.logical_level0_px.width,
                       params->src.view.base_level);
      params->wm_inputs.src_inv_size[1] =
         1.0f / minify(params->src.surf.logical_level0_px.height,
                       params->src.view.base_level);
   }

   if (isl_format_get_layout(params->dst.view.format)->bpb % 3 == 0) {
      /* We can't render to  RGB formats natively because they aren't a
       * power-of-two size.  Instead, we fake them by using a red format
       * with the same channel type and size and emitting shader code to
       * only write one channel at a time.
       */
      params->x0 *= 3;
      params->x1 *= 3;

      /* If it happens to be sRGB, we need to force a conversion */
      if (params->dst.view.format == ISL_FORMAT_R8G8B8_UNORM_SRGB)
         wm_prog_key->dst_format = ISL_FORMAT_R8G8B8_UNORM_SRGB;

      surf_fake_rgb_with_red(batch->blorp->isl_dev, &params->dst);

      wm_prog_key->dst_rgb = true;
      wm_prog_key->need_dst_offset = true;
   } else if (isl_format_is_rgbx(params->dst.view.format)) {
      /* We can handle RGBX formats easily enough by treating them as RGBA */
      params->dst.view.format =
         isl_format_rgbx_to_rgba(params->dst.view.format);
   } else if (params->dst.view.format == ISL_FORMAT_R24_UNORM_X8_TYPELESS &&
              wm_prog_key->dst_usage != ISL_SURF_USAGE_DEPTH_BIT) {
      wm_prog_key->dst_format = params->dst.view.format;
      params->dst.view.format = ISL_FORMAT_R32_UINT;
   } else if (params->dst.view.format == ISL_FORMAT_A4B4G4R4_UNORM) {
      params->dst.view.swizzle =
         isl_swizzle_compose(params->dst.view.swizzle,
                             ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE));
      params->dst.view.format = ISL_FORMAT_B4G4R4A4_UNORM;
   } else if (params->dst.view.format == ISL_FORMAT_L8_UNORM_SRGB) {
      wm_prog_key->dst_format = params->dst.view.format;
      params->dst.view.format = ISL_FORMAT_R8_UNORM;
   } else if (params->dst.view.format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
      wm_prog_key->dst_format = params->dst.view.format;
      params->dst.view.format = ISL_FORMAT_R32_UINT;
   }

   if (devinfo->ver <= 7 && !devinfo->is_haswell &&
       !isl_swizzle_is_identity(params->src.view.swizzle)) {
      wm_prog_key->src_swizzle = params->src.view.swizzle;
      params->src.view.swizzle = ISL_SWIZZLE_IDENTITY;
   } else {
      wm_prog_key->src_swizzle = ISL_SWIZZLE_IDENTITY;
   }

   if (!isl_swizzle_supports_rendering(devinfo, params->dst.view.swizzle)) {
      wm_prog_key->dst_swizzle = params->dst.view.swizzle;
      params->dst.view.swizzle = ISL_SWIZZLE_IDENTITY;
   } else {
      wm_prog_key->dst_swizzle = ISL_SWIZZLE_IDENTITY;
   }

   if (params->src.tile_x_sa || params->src.tile_y_sa) {
      assert(wm_prog_key->need_src_offset);
      surf_get_intratile_offset_px(&params->src,
                                   &params->wm_inputs.src_offset.x,
                                   &params->wm_inputs.src_offset.y);
   }

   if (params->dst.tile_x_sa || params->dst.tile_y_sa) {
      assert(wm_prog_key->need_dst_offset);
      surf_get_intratile_offset_px(&params->dst,
                                   &params->wm_inputs.dst_offset.x,
                                   &params->wm_inputs.dst_offset.y);
      params->x0 += params->wm_inputs.dst_offset.x;
      params->y0 += params->wm_inputs.dst_offset.y;
      params->x1 += params->wm_inputs.dst_offset.x;
      params->y1 += params->wm_inputs.dst_offset.y;
   }

   /* For some texture types, we need to pass the layer through the sampler. */
   params->wm_inputs.src_z = params->src.z_offset;

   if (!brw_blorp_get_blit_kernel(batch, params, wm_prog_key))
      return 0;

   if (!blorp_ensure_sf_program(batch, params))
      return 0;

   unsigned result = 0;
   unsigned max_src_surface_size = get_max_surface_size(devinfo, &params->src);
   if (params->src.surf.logical_level0_px.width > max_src_surface_size)
      result |= BLIT_SRC_WIDTH_SHRINK;
   if (params->src.surf.logical_level0_px.height > max_src_surface_size)
      result |= BLIT_SRC_HEIGHT_SHRINK;

   unsigned max_dst_surface_size = get_max_surface_size(devinfo, &params->dst);
   if (params->dst.surf.logical_level0_px.width > max_dst_surface_size)
      result |= BLIT_DST_WIDTH_SHRINK;
   if (params->dst.surf.logical_level0_px.height > max_dst_surface_size)
      result |= BLIT_DST_HEIGHT_SHRINK;

   if (result == 0) {
      if (wm_prog_key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
         params->depth = params->dst;
         memset(&params->dst, 0, sizeof(params->dst));
      } else if (wm_prog_key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
         params->stencil = params->dst;
         params->stencil_mask = 0xff;
         memset(&params->dst, 0, sizeof(params->dst));
      }

      batch->blorp->exec(batch, params);
   }

   return result;
}

/* Adjust split blit source coordinates for the current destination
 * coordinates.
 */
static void
adjust_split_source_coords(const struct blt_axis *orig,
                           struct blt_axis *split_coords,
                           double scale)
{
   /* When scale is greater than 0, then we are growing from the start, so
    * src0 uses delta0, and src1 uses delta1. When scale is less than 0, the
    * source range shrinks from the end. In that case src0 is adjusted by
    * delta1, and src1 is adjusted by delta0.
    */
   double delta0 = scale * (split_coords->dst0 - orig->dst0);
   double delta1 = scale * (split_coords->dst1 - orig->dst1);
   split_coords->src0 = orig->src0 + (scale >= 0.0 ? delta0 : delta1);
   split_coords->src1 = orig->src1 + (scale >= 0.0 ? delta1 : delta0);
}

static struct isl_extent2d
get_px_size_sa(const struct isl_surf *surf)
{
   static const struct isl_extent2d one_to_one = { .w = 1, .h = 1 };

   if (surf->msaa_layout != ISL_MSAA_LAYOUT_INTERLEAVED)
      return one_to_one;
   else
      return isl_get_interleaved_msaa_px_size_sa(surf->samples);
}

static void
shrink_surface_params(const struct isl_device *dev,
                      struct brw_blorp_surface_info *info,
                      double *x0, double *x1, double *y0, double *y1)
{
   uint32_t byte_offset, x_offset_sa, y_offset_sa, size;
   struct isl_extent2d px_size_sa;
   int adjust;

   blorp_surf_convert_to_single_slice(dev, info);

   px_size_sa = get_px_size_sa(&info->surf);

   /* Because this gets called after we lower compressed images, the tile
    * offsets may be non-zero and we need to incorporate them in our
    * calculations.
    */
   x_offset_sa = (uint32_t)*x0 * px_size_sa.w + info->tile_x_sa;
   y_offset_sa = (uint32_t)*y0 * px_size_sa.h + info->tile_y_sa;
   isl_tiling_get_intratile_offset_sa(info->surf.tiling,
                                      info->surf.format, info->surf.row_pitch_B,
                                      x_offset_sa, y_offset_sa,
                                      &byte_offset,
                                      &info->tile_x_sa, &info->tile_y_sa);

   info->addr.offset += byte_offset;

   adjust = (int)info->tile_x_sa / px_size_sa.w - (int)*x0;
   *x0 += adjust;
   *x1 += adjust;
   info->tile_x_sa = 0;

   adjust = (int)info->tile_y_sa / px_size_sa.h - (int)*y0;
   *y0 += adjust;
   *y1 += adjust;
   info->tile_y_sa = 0;

   size = MIN2((uint32_t)ceil(*x1), info->surf.logical_level0_px.width);
   info->surf.logical_level0_px.width = size;
   info->surf.phys_level0_sa.width = size * px_size_sa.w;

   size = MIN2((uint32_t)ceil(*y1), info->surf.logical_level0_px.height);
   info->surf.logical_level0_px.height = size;
   info->surf.phys_level0_sa.height = size * px_size_sa.h;
}

static void
do_blorp_blit(struct blorp_batch *batch,
              const struct blorp_params *orig_params,
              struct brw_blorp_blit_prog_key *wm_prog_key,
              const struct blt_coords *orig)
{
   struct blorp_params params;
   struct blt_coords blit_coords;
   struct blt_coords split_coords = *orig;
   double w = orig->x.dst1 - orig->x.dst0;
   double h = orig->y.dst1 - orig->y.dst0;
   double x_scale = (orig->x.src1 - orig->x.src0) / w;
   double y_scale = (orig->y.src1 - orig->y.src0) / h;
   if (orig->x.mirror)
      x_scale = -x_scale;
   if (orig->y.mirror)
      y_scale = -y_scale;

   enum blit_shrink_status shrink = BLIT_NO_SHRINK;
   if (split_blorp_blit_debug) {
      if (can_shrink_surface(&orig_params->src))
         shrink |= BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK;
      if (can_shrink_surface(&orig_params->dst))
         shrink |= BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK;
   }

   bool x_done, y_done;
   do {
      params = *orig_params;
      blit_coords = split_coords;

      if (shrink & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK)) {
         shrink_surface_params(batch->blorp->isl_dev, &params.src,
                               &blit_coords.x.src0, &blit_coords.x.src1,
                               &blit_coords.y.src0, &blit_coords.y.src1);
         wm_prog_key->need_src_offset = false;
      }

      if (shrink & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
         shrink_surface_params(batch->blorp->isl_dev, &params.dst,
                               &blit_coords.x.dst0, &blit_coords.x.dst1,
                               &blit_coords.y.dst0, &blit_coords.y.dst1);
         wm_prog_key->need_dst_offset = false;
      }

      enum blit_shrink_status result =
         try_blorp_blit(batch, &params, wm_prog_key, &blit_coords);

      if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK))
         assert(can_shrink_surface(&orig_params->src));

      if (result & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK))
         assert(can_shrink_surface(&orig_params->dst));

      if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_DST_WIDTH_SHRINK)) {
         w /= 2.0;
         assert(w >= 1.0);
         split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
         adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
      }
      if (result & (BLIT_SRC_HEIGHT_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
         h /= 2.0;
         assert(h >= 1.0);
         split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
         adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
      }

      if (result) {
         /* We may get less bits set on result than we had already, so make
          * sure we remember all the ways in which a resize is required.
          */
         shrink |= result;
         continue;
      }

      y_done = (orig->y.dst1 - split_coords.y.dst1 < 0.5);
      x_done = y_done && (orig->x.dst1 - split_coords.x.dst1 < 0.5);
      if (x_done) {
         break;
      } else if (y_done) {
         split_coords.x.dst0 += w;
         split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
         split_coords.y.dst0 = orig->y.dst0;
         split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
         adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
      } else {
         split_coords.y.dst0 += h;
         split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
         adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
      }
   } while (true);
}

void
blorp_blit(struct blorp_batch *batch,
           const struct blorp_surf *src_surf,
           unsigned src_level, float src_layer,
           enum isl_format src_format, struct isl_swizzle src_swizzle,
           const struct blorp_surf *dst_surf,
           unsigned dst_level, unsigned dst_layer,
           enum isl_format dst_format, struct isl_swizzle dst_swizzle,
           float src_x0, float src_y0,
           float src_x1, float src_y1,
           float dst_x0, float dst_y0,
           float dst_x1, float dst_y1,
           enum blorp_filter filter,
           bool mirror_x, bool mirror_y)
{
   struct blorp_params params;
   blorp_params_init(&params);
   params.snapshot_type = INTEL_SNAPSHOT_BLIT;

   /* We cannot handle combined depth and stencil. */
   if (src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
      assert(src_surf->surf->format == ISL_FORMAT_R8_UINT);
   if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
      assert(dst_surf->surf->format == ISL_FORMAT_R8_UINT);

   if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT) {
      assert(src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT);
      /* Prior to Broadwell, we can't render to R8_UINT */
      if (batch->blorp->isl_dev->info->ver < 8) {
         src_format = ISL_FORMAT_R8_UNORM;
         dst_format = ISL_FORMAT_R8_UNORM;
      }
   }

   brw_blorp_surface_info_init(batch->blorp, &params.src, src_surf, src_level,
                               src_layer, src_format, false);
   brw_blorp_surface_info_init(batch->blorp, &params.dst, dst_surf, dst_level,
                               dst_layer, dst_format, true);

   params.src.view.swizzle = src_swizzle;
   params.dst.view.swizzle = dst_swizzle;

   const struct isl_format_layout *src_fmtl =
      isl_format_get_layout(params.src.view.format);

   struct brw_blorp_blit_prog_key wm_prog_key = {
      .shader_type = BLORP_SHADER_TYPE_BLIT,
      .filter = filter,
      .sint32_to_uint = src_fmtl->channels.r.bits == 32 &&
                        isl_format_has_sint_channel(params.src.view.format) &&
                        isl_format_has_uint_channel(params.dst.view.format),
      .uint32_to_sint = src_fmtl->channels.r.bits == 32 &&
                        isl_format_has_uint_channel(params.src.view.format) &&
                        isl_format_has_sint_channel(params.dst.view.format),
   };

   /* Scaling factors used for bilinear filtering in multisample scaled
    * blits.
    */
   if (params.src.surf.samples == 16)
      wm_prog_key.x_scale = 4.0f;
   else
      wm_prog_key.x_scale = 2.0f;
   wm_prog_key.y_scale = params.src.surf.samples / wm_prog_key.x_scale;

   params.wm_inputs.rect_grid.x1 =
      minify(params.src.surf.logical_level0_px.width, src_level) *
      wm_prog_key.x_scale - 1.0f;
   params.wm_inputs.rect_grid.y1 =
      minify(params.src.surf.logical_level0_px.height, src_level) *
      wm_prog_key.y_scale - 1.0f;

   struct blt_coords coords = {
      .x = {
         .src0 = src_x0,
         .src1 = src_x1,
         .dst0 = dst_x0,
         .dst1 = dst_x1,
         .mirror = mirror_x
      },
      .y = {
         .src0 = src_y0,
         .src1 = src_y1,
         .dst0 = dst_y0,
         .dst1 = dst_y1,
         .mirror = mirror_y
      }
   };

   do_blorp_blit(batch, &params, &wm_prog_key, &coords);
}

static enum isl_format
get_copy_format_for_bpb(const struct isl_device *isl_dev, unsigned bpb)
{
   /* The choice of UNORM and UINT formats is very intentional here.  Most
    * of the time, we want to use a UINT format to avoid any rounding error
    * in the blit.  For stencil blits, R8_UINT is required by the hardware.
    * (It's the only format allowed in conjunction with W-tiling.)  Also we
    * intentionally use the 4-channel formats whenever we can.  This is so
    * that, when we do a RGB <-> RGBX copy, the two formats will line up
    * even though one of them is 3/4 the size of the other.  The choice of
    * UNORM vs. UINT is also very intentional because we don't have 8 or
    * 16-bit RGB UINT formats until Sky Lake so we have to use UNORM there.
    * Fortunately, the only time we should ever use two different formats in
    * the table below is for RGB -> RGBA blits and so we will never have any
    * UNORM/UINT mismatch.
    */
   if (ISL_GFX_VER(isl_dev) >= 9) {
      switch (bpb) {
      case 8:  return ISL_FORMAT_R8_UINT;
      case 16: return ISL_FORMAT_R8G8_UINT;
      case 24: return ISL_FORMAT_R8G8B8_UINT;
      case 32: return ISL_FORMAT_R8G8B8A8_UINT;
      case 48: return ISL_FORMAT_R16G16B16_UINT;
      case 64: return ISL_FORMAT_R16G16B16A16_UINT;
      case 96: return ISL_FORMAT_R32G32B32_UINT;
      case 128:return ISL_FORMAT_R32G32B32A32_UINT;
      default:
         unreachable("Unknown format bpb");
      }
   } else {
      switch (bpb) {
      case 8:  return ISL_FORMAT_R8_UINT;
      case 16: return ISL_FORMAT_R8G8_UINT;
      case 24: return ISL_FORMAT_R8G8B8_UNORM;
      case 32: return ISL_FORMAT_R8G8B8A8_UNORM;
      case 48: return ISL_FORMAT_R16G16B16_UNORM;
      case 64: return ISL_FORMAT_R16G16B16A16_UNORM;
      case 96: return ISL_FORMAT_R32G32B32_UINT;
      case 128:return ISL_FORMAT_R32G32B32A32_UINT;
      default:
         unreachable("Unknown format bpb");
      }
   }
}

/** Returns a UINT format that is CCS-compatible with the given format
 *
 * The PRM's say absolutely nothing about how render compression works.  The
 * only thing they provide is a list of formats on which it is and is not
 * supported.  Empirical testing indicates that the compression is only based
 * on the bit-layout of the format and the channel encoding doesn't matter.
 * So, while texture views don't work in general, you can create a view as
 * long as the bit-layout of the formats are the same.
 *
 * Fortunately, for every render compression capable format, the UINT format
 * with the same bit layout also supports render compression.  This means that
 * we only need to handle UINT formats for copy operations.  In order to do
 * copies between formats with different bit layouts, we attach both with a
 * UINT format and use bit_cast_color() to generate code to do the bit-cast
 * operation between the two bit layouts.
 */
static enum isl_format
get_ccs_compatible_copy_format(const struct isl_format_layout *fmtl)
{
   switch (fmtl->format) {
   case ISL_FORMAT_R32G32B32A32_FLOAT:
   case ISL_FORMAT_R32G32B32A32_SINT:
   case ISL_FORMAT_R32G32B32A32_UINT:
   case ISL_FORMAT_R32G32B32A32_UNORM:
   case ISL_FORMAT_R32G32B32A32_SNORM:
   case ISL_FORMAT_R32G32B32X32_FLOAT:
      return ISL_FORMAT_R32G32B32A32_UINT;

   case ISL_FORMAT_R16G16B16A16_UNORM:
   case ISL_FORMAT_R16G16B16A16_SNORM:
   case ISL_FORMAT_R16G16B16A16_SINT:
   case ISL_FORMAT_R16G16B16A16_UINT:
   case ISL_FORMAT_R16G16B16A16_FLOAT:
   case ISL_FORMAT_R16G16B16X16_UNORM:
   case ISL_FORMAT_R16G16B16X16_FLOAT:
      return ISL_FORMAT_R16G16B16A16_UINT;

   case ISL_FORMAT_R32G32_FLOAT:
   case ISL_FORMAT_R32G32_SINT:
   case ISL_FORMAT_R32G32_UINT:
   case ISL_FORMAT_R32G32_UNORM:
   case ISL_FORMAT_R32G32_SNORM:
      return ISL_FORMAT_R32G32_UINT;

   case ISL_FORMAT_B8G8R8A8_UNORM:
   case ISL_FORMAT_B8G8R8A8_UNORM_SRGB:
   case ISL_FORMAT_R8G8B8A8_UNORM:
   case ISL_FORMAT_R8G8B8A8_UNORM_SRGB:
   case ISL_FORMAT_R8G8B8A8_SNORM:
   case ISL_FORMAT_R8G8B8A8_SINT:
   case ISL_FORMAT_R8G8B8A8_UINT:
   case ISL_FORMAT_B8G8R8X8_UNORM:
   case ISL_FORMAT_B8G8R8X8_UNORM_SRGB:
   case ISL_FORMAT_R8G8B8X8_UNORM:
   case ISL_FORMAT_R8G8B8X8_UNORM_SRGB:
      return ISL_FORMAT_R8G8B8A8_UINT;

   case ISL_FORMAT_R16G16_UNORM:
   case ISL_FORMAT_R16G16_SNORM:
   case ISL_FORMAT_R16G16_SINT:
   case ISL_FORMAT_R16G16_UINT:
   case ISL_FORMAT_R16G16_FLOAT:
      return ISL_FORMAT_R16G16_UINT;

   case ISL_FORMAT_R32_SINT:
   case ISL_FORMAT_R32_UINT:
   case ISL_FORMAT_R32_FLOAT:
   case ISL_FORMAT_R32_UNORM:
   case ISL_FORMAT_R32_SNORM:
      return ISL_FORMAT_R32_UINT;

   case ISL_FORMAT_B10G10R10A2_UNORM:
   case ISL_FORMAT_B10G10R10A2_UNORM_SRGB:
   case ISL_FORMAT_R10G10B10A2_UNORM:
   case ISL_FORMAT_R10G10B10A2_UNORM_SRGB:
   case ISL_FORMAT_R10G10B10_FLOAT_A2_UNORM:
   case ISL_FORMAT_R10G10B10A2_UINT:
      return ISL_FORMAT_R10G10B10A2_UINT;

   case ISL_FORMAT_R16_UNORM:
   case ISL_FORMAT_R16_SNORM:
   case ISL_FORMAT_R16_SINT:
   case ISL_FORMAT_R16_UINT:
   case ISL_FORMAT_R16_FLOAT:
      return ISL_FORMAT_R16_UINT;

   case ISL_FORMAT_R8G8_UNORM:
   case ISL_FORMAT_R8G8_SNORM:
   case ISL_FORMAT_R8G8_SINT:
   case ISL_FORMAT_R8G8_UINT:
      return ISL_FORMAT_R8G8_UINT;

   case ISL_FORMAT_B5G5R5X1_UNORM:
   case ISL_FORMAT_B5G5R5X1_UNORM_SRGB:
   case ISL_FORMAT_B5G5R5A1_UNORM:
   case ISL_FORMAT_B5G5R5A1_UNORM_SRGB:
      return ISL_FORMAT_B5G5R5A1_UNORM;

   case ISL_FORMAT_A4B4G4R4_UNORM:
   case ISL_FORMAT_B4G4R4A4_UNORM:
   case ISL_FORMAT_B4G4R4A4_UNORM_SRGB:
      return ISL_FORMAT_B4G4R4A4_UNORM;

   case ISL_FORMAT_B5G6R5_UNORM:
   case ISL_FORMAT_B5G6R5_UNORM_SRGB:
      return ISL_FORMAT_B5G6R5_UNORM;

   case ISL_FORMAT_A1B5G5R5_UNORM:
      return ISL_FORMAT_A1B5G5R5_UNORM;

   case ISL_FORMAT_A8_UNORM:
   case ISL_FORMAT_R8_UNORM:
   case ISL_FORMAT_R8_SNORM:
   case ISL_FORMAT_R8_SINT:
   case ISL_FORMAT_R8_UINT:
      return ISL_FORMAT_R8_UINT;

   default:
      unreachable("Not a compressible format");
   }
}

void
blorp_surf_convert_to_uncompressed(const struct isl_device *isl_dev,
                                   struct brw_blorp_surface_info *info,
                                   uint32_t *x, uint32_t *y,
                                   uint32_t *width, uint32_t *height)
{
   const struct isl_format_layout *fmtl =
      isl_format_get_layout(info->surf.format);

   assert(fmtl->bw > 1 || fmtl->bh > 1);

   /* This is a compressed surface.  We need to convert it to a single
    * slice (because compressed layouts don't perfectly match uncompressed
    * ones with the same bpb) and divide x, y, width, and height by the
    * block size.
    */
   blorp_surf_convert_to_single_slice(isl_dev, info);

   if (width && height) {
#ifndef NDEBUG
      uint32_t right_edge_px = info->tile_x_sa + *x + *width;
      uint32_t bottom_edge_px = info->tile_y_sa + *y + *height;
      assert(*width % fmtl->bw == 0 ||
             right_edge_px == info->surf.logical_level0_px.width);
      assert(*height % fmtl->bh == 0 ||
             bottom_edge_px == info->surf.logical_level0_px.height);
#endif
      *width = DIV_ROUND_UP(*width, fmtl->bw);
      *height = DIV_ROUND_UP(*height, fmtl->bh);
   }

   if (x && y) {
      assert(*x % fmtl->bw == 0);
      assert(*y % fmtl->bh == 0);
      *x /= fmtl->bw;
      *y /= fmtl->bh;
   }

   info->surf.logical_level0_px = isl_surf_get_logical_level0_el(&info->surf);
   info->surf.phys_level0_sa = isl_surf_get_phys_level0_el(&info->surf);

   assert(info->tile_x_sa % fmtl->bw == 0);
   assert(info->tile_y_sa % fmtl->bh == 0);
   info->tile_x_sa /= fmtl->bw;
   info->tile_y_sa /= fmtl->bh;

   /* It's now an uncompressed surface so we need an uncompressed format */
   info->surf.format = get_copy_format_for_bpb(isl_dev, fmtl->bpb);
}

void
blorp_copy(struct blorp_batch *batch,
           const struct blorp_surf *src_surf,
           unsigned src_level, unsigned src_layer,
           const struct blorp_surf *dst_surf,
           unsigned dst_level, unsigned dst_layer,
           uint32_t src_x, uint32_t src_y,
           uint32_t dst_x, uint32_t dst_y,
           uint32_t src_width, uint32_t src_height)
{
   const struct isl_device *isl_dev = batch->blorp->isl_dev;
   struct blorp_params params;

   if (src_width == 0 || src_height == 0)
      return;

   blorp_params_init(&params);
   params.snapshot_type = INTEL_SNAPSHOT_COPY;
   brw_blorp_surface_info_init(batch->blorp, &params.src, src_surf, src_level,
                               src_layer, ISL_FORMAT_UNSUPPORTED, false);
   brw_blorp_surface_info_init(batch->blorp, &params.dst, dst_surf, dst_level,
                               dst_layer, ISL_FORMAT_UNSUPPORTED, true);

   struct brw_blorp_blit_prog_key wm_prog_key = {
      .shader_type = BLORP_SHADER_TYPE_COPY,
      .filter = BLORP_FILTER_NONE,
      .need_src_offset = src_surf->tile_x_sa || src_surf->tile_y_sa,
      .need_dst_offset = dst_surf->tile_x_sa || dst_surf->tile_y_sa,
   };

   const struct isl_format_layout *src_fmtl =
      isl_format_get_layout(params.src.surf.format);
   const struct isl_format_layout *dst_fmtl =
      isl_format_get_layout(params.dst.surf.format);

   assert(params.src.aux_usage == ISL_AUX_USAGE_NONE ||
          params.src.aux_usage == ISL_AUX_USAGE_HIZ ||
          params.src.aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT ||
          params.src.aux_usage == ISL_AUX_USAGE_MCS ||
          params.src.aux_usage == ISL_AUX_USAGE_MCS_CCS ||
          params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
          params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E ||
          params.src.aux_usage == ISL_AUX_USAGE_STC_CCS);

   if (isl_aux_usage_has_hiz(params.src.aux_usage)) {
      /* In order to use HiZ, we have to use the real format for the source.
       * Depth <-> Color copies are not allowed.
       */
      params.src.view.format = params.src.surf.format;
      params.dst.view.format = params.src.surf.format;
   } else if ((params.dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) &&
              isl_dev->info->ver >= 7) {
      /* On Gfx7 and higher, we use actual depth writes for blits into depth
       * buffers so we need the real format.
       */
      params.src.view.format = params.dst.surf.format;
      params.dst.view.format = params.dst.surf.format;
   } else if (params.dst.aux_usage == ISL_AUX_USAGE_CCS_E ||
              params.dst.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
      params.dst.view.format = get_ccs_compatible_copy_format(dst_fmtl);
      if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
          params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
         params.src.view.format = get_ccs_compatible_copy_format(src_fmtl);
      } else if (src_fmtl->bpb == dst_fmtl->bpb) {
         params.src.view.format = params.dst.view.format;
      } else {
         params.src.view.format =
            get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
      }
   } else if (params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
              params.src.aux_usage == ISL_AUX_USAGE_GEN12_CCS_E) {
      params.src.view.format = get_ccs_compatible_copy_format(src_fmtl);
      if (src_fmtl->bpb == dst_fmtl->bpb) {
         params.dst.view.format = params.src.view.format;
      } else {
         params.dst.view.format =
            get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
      }
   } else {
      params.dst.view.format = get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
      params.src.view.format = get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
   }

   if (params.src.view.format != params.dst.view.format) {
      enum isl_format src_cast_format = params.src.view.format;
      enum isl_format dst_cast_format = params.dst.view.format;

      /* The BLORP bitcast code gets confused by RGB formats.  Just treat them
       * as RGBA and then everything will be happy.  This is perfectly safe
       * because BLORP likes to treat things as if they have vec4 colors all
       * the time anyway.
       */
      if (isl_format_get_layout(src_cast_format)->bpb % 3 == 0)
         src_cast_format = isl_format_rgb_to_rgba(src_cast_format);
      if (isl_format_get_layout(dst_cast_format)->bpb % 3 == 0)
         dst_cast_format = isl_format_rgb_to_rgba(dst_cast_format);

      if (src_cast_format != dst_cast_format) {
         wm_prog_key.format_bit_cast = true;
         wm_prog_key.src_format = src_cast_format;
         wm_prog_key.dst_format = dst_cast_format;
      }
   }

   if (src_fmtl->bw > 1 || src_fmtl->bh > 1) {
      blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, &params.src,
                                         &src_x, &src_y,
                                         &src_width, &src_height);
      wm_prog_key.need_src_offset = true;
   }

   if (dst_fmtl->bw > 1 || dst_fmtl->bh > 1) {
      blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, &params.dst,
                                         &dst_x, &dst_y, NULL, NULL);
      wm_prog_key.need_dst_offset = true;
   }

   /* Once both surfaces are stompped to uncompressed as needed, the
    * destination size is the same as the source size.
    */
   uint32_t dst_width = src_width;
   uint32_t dst_height = src_height;

   struct blt_coords coords = {
      .x = {
         .src0 = src_x,
         .src1 = src_x + src_width,
         .dst0 = dst_x,
         .dst1 = dst_x + dst_width,
         .mirror = false
      },
      .y = {
         .src0 = src_y,
         .src1 = src_y + src_height,
         .dst0 = dst_y,
         .dst1 = dst_y + dst_height,
         .mirror = false
      }
   };

   do_blorp_blit(batch, &params, &wm_prog_key, &coords);
}

static enum isl_format
isl_format_for_size(unsigned size_B)
{
   switch (size_B) {
   case 1:  return ISL_FORMAT_R8_UINT;
   case 2:  return ISL_FORMAT_R8G8_UINT;
   case 4:  return ISL_FORMAT_R8G8B8A8_UINT;
   case 8:  return ISL_FORMAT_R16G16B16A16_UINT;
   case 16: return ISL_FORMAT_R32G32B32A32_UINT;
   default:
      unreachable("Not a power-of-two format size");
   }
}

/**
 * Returns the greatest common divisor of a and b that is a power of two.
 */
static uint64_t
gcd_pow2_u64(uint64_t a, uint64_t b)
{
   assert(a > 0 || b > 0);

   unsigned a_log2 = ffsll(a) - 1;
   unsigned b_log2 = ffsll(b) - 1;

   /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
    * case, the MIN2() will take the other one.  If both are 0 then we will
    * hit the assert above.
    */
   return 1 << MIN2(a_log2, b_log2);
}

static void
do_buffer_copy(struct blorp_batch *batch,
               struct blorp_address *src,
               struct blorp_address *dst,
               int width, int height, int block_size)
{
   /* The actual format we pick doesn't matter as blorp will throw it away.
    * The only thing that actually matters is the size.
    */
   enum isl_format format = isl_format_for_size(block_size);

   UNUSED bool ok;
   struct isl_surf surf;
   ok = isl_surf_init(batch->blorp->isl_dev, &surf,
                      .dim = ISL_SURF_DIM_2D,
                      .format = format,
                      .width = width,
                      .height = height,
                      .depth = 1,
                      .levels = 1,
                      .array_len = 1,
                      .samples = 1,
                      .row_pitch_B = width * block_size,
                      .usage = ISL_SURF_USAGE_TEXTURE_BIT |
                               ISL_SURF_USAGE_RENDER_TARGET_BIT,
                      .tiling_flags = ISL_TILING_LINEAR_BIT);
   assert(ok);

   struct blorp_surf src_blorp_surf = {
      .surf = &surf,
      .addr = *src,
   };

   struct blorp_surf dst_blorp_surf = {
      .surf = &surf,
      .addr = *dst,
   };

   blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
              0, 0, 0, 0, width, height);
}

void
blorp_buffer_copy(struct blorp_batch *batch,
                  struct blorp_address src,
                  struct blorp_address dst,
                  uint64_t size)
{
   const struct gen_device_info *devinfo = batch->blorp->isl_dev->info;
   uint64_t copy_size = size;

   /* This is maximum possible width/height our HW can handle */
   uint64_t max_surface_dim = 1 << (devinfo->ver >= 7 ? 14 : 13);

   /* First, we compute the biggest format that can be used with the
    * given offsets and size.
    */
   int bs = 16;
   bs = gcd_pow2_u64(bs, src.offset);
   bs = gcd_pow2_u64(bs, dst.offset);
   bs = gcd_pow2_u64(bs, size);

   /* First, we make a bunch of max-sized copies */
   uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
   while (copy_size >= max_copy_size) {
      do_buffer_copy(batch, &src, &dst, max_surface_dim, max_surface_dim, bs);
      copy_size -= max_copy_size;
      src.offset += max_copy_size;
      dst.offset += max_copy_size;
   }

   /* Now make a max-width copy */
   uint64_t height = copy_size / (max_surface_dim * bs);
   assert(height < max_surface_dim);
   if (height != 0) {
      uint64_t rect_copy_size = height * max_surface_dim * bs;
      do_buffer_copy(batch, &src, &dst, max_surface_dim, height, bs);
      copy_size -= rect_copy_size;
      src.offset += rect_copy_size;
      dst.offset += rect_copy_size;
   }

   /* Finally, make a small copy to finish it off */
   if (copy_size != 0) {
      do_buffer_copy(batch, &src, &dst, copy_size / bs, 1, bs);
   }
}