mesa/src/amd/vulkan/radv_cp_dma.c

/*
 * Copyright © 2016 Red Hat.
 * Copyright © 2016 Bas Nieuwenhuizen
 *
 * based on si_state.c
 * Copyright © 2015 Advanced Micro Devices, Inc.
 *
 * SPDX-License-Identifier: MIT
 */

#include "radv_cp_dma.h"
#include "radv_buffer.h"
#include "radv_cs.h"
#include "radv_debug.h"
#include "radv_shader.h"
#include "radv_sqtt.h"
#include "sid.h"

/* Set this if you want the 3D engine to wait until CP DMA is done.
 * It should be set on the last CP DMA packet. */
#define CP_DMA_SYNC (1 << 0)

/* Set this if the source data was used as a destination in a previous CP DMA
 * packet. It's for preventing a read-after-write (RAW) hazard between two
 * CP DMA packets. */
#define CP_DMA_RAW_WAIT (1 << 1)
#define CP_DMA_USE_L2   (1 << 2)
#define CP_DMA_CLEAR    (1 << 3)

/* Alignment for optimal performance. */
#define SI_CPDMA_ALIGNMENT 32

/* The max number of bytes that can be copied per packet. */
static inline unsigned
cp_dma_max_byte_count(enum amd_gfx_level gfx_level)
{
   unsigned max = gfx_level >= GFX11  ? 32767
                  : gfx_level >= GFX9 ? S_415_BYTE_COUNT_GFX9(~0u)
                                      : S_415_BYTE_COUNT_GFX6(~0u);

   /* make it aligned for optimal performance */
   return max & ~(SI_CPDMA_ALIGNMENT - 1);
}

/* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
 * clear value.
 */
static void
radv_cs_emit_cp_dma(struct radv_device *device, struct radv_cmd_stream *cs, bool predicating, uint64_t dst_va,
                    uint64_t src_va, unsigned size, unsigned flags)
{
   const struct radv_physical_device *pdev = radv_device_physical(device);
   const bool cp_dma_use_L2 = (flags & CP_DMA_USE_L2) && pdev->info.cp_dma_use_L2;
   const bool cp_dma_use_mall = pdev->info.gfx_level == GFX12;
   /* GFX12: TC_L2 means MALL, which should always be set. */
   const bool cp_dma_tc_l2_flag = cp_dma_use_L2 || cp_dma_use_mall;
   uint32_t header = 0, command = 0;

   assert(size <= cp_dma_max_byte_count(pdev->info.gfx_level));

   radeon_check_space(device->ws, cs->b, 9);
   if (pdev->info.gfx_level >= GFX9)
      command |= S_415_BYTE_COUNT_GFX9(size);
   else
      command |= S_415_BYTE_COUNT_GFX6(size);

   /* Sync flags. */
   if (flags & CP_DMA_SYNC)
      header |= S_411_CP_SYNC(1);

   if (flags & CP_DMA_RAW_WAIT)
      command |= S_415_RAW_WAIT(1);

   /* Src and dst flags. */
   if (cp_dma_tc_l2_flag)
      header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2);

   if (flags & CP_DMA_CLEAR)
      header |= S_411_SRC_SEL(V_411_DATA);
   else if (cp_dma_tc_l2_flag)
      header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);

   radeon_begin(cs);
   if (pdev->info.gfx_level >= GFX7) {
      radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating));
      radeon_emit(header);
      radeon_emit(src_va);       /* SRC_ADDR_LO [31:0] */
      radeon_emit(src_va >> 32); /* SRC_ADDR_HI [31:0] */
      radeon_emit(dst_va);       /* DST_ADDR_LO [31:0] */
      radeon_emit(dst_va >> 32); /* DST_ADDR_HI [31:0] */
      radeon_emit(command);
   } else {
      assert(!cp_dma_tc_l2_flag);
      header |= S_411_SRC_ADDR_HI(src_va >> 32);
      radeon_emit(PKT3(PKT3_CP_DMA, 4, predicating));
      radeon_emit(src_va);                  /* SRC_ADDR_LO [31:0] */
      radeon_emit(header);                  /* SRC_ADDR_HI [15:0] + flags. */
      radeon_emit(dst_va);                  /* DST_ADDR_LO [31:0] */
      radeon_emit((dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
      radeon_emit(command);
   }
   radeon_end();
}

static void
radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t src_va, unsigned size, unsigned flags)
{
   struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
   struct radv_cmd_stream *cs = cmd_buffer->cs;
   bool predicating = cmd_buffer->state.predicating;

   radv_cs_emit_cp_dma(device, cs, predicating, dst_va, src_va, size, flags);

   /* CP DMA is executed in ME, but index buffers are read by PFP.
    * This ensures that ME (CP DMA) is idle before PFP starts fetching
    * indices. If we wanted to execute CP DMA in PFP, this packet
    * should precede it.
    */
   if (flags & CP_DMA_SYNC) {
      if (cmd_buffer->qf == RADV_QUEUE_GENERAL) {
         ac_emit_cp_pfp_sync_me(cs->b, cmd_buffer->state.predicating);
      }

      /* CP will see the sync flag and wait for all DMAs to complete. */
      cmd_buffer->state.dma_is_busy = false;
   }

   if (radv_device_fault_detection_enabled(device))
      radv_cmd_buffer_trace_emit(cmd_buffer);
}

/* Emit a CP DMA prefetch.
 * This is useful for warming up caches before draw commands,
 * for example we use it to load shader binaries and VBO descriptors into the cache.
 * Implemented by starting a CP DMA copy where the source and destination are the same.
 *
 * On GPUs where CP DMA uses L2, it loads binaries into L2.
 * On GPUs that have MALL (infinity cache), it loads binaries into MALL.
 *
 * More specifically:
 *
 * | GPU generation  | CP DMA L2 | MALL | Prefetch location |
 * | --------------- | --------- | ---- | ----------------- |
 * | GFX6            | -         | -    | -                 |
 * | GFX7 - 10       | yes       | -    | L2                |
 * | GFX10.3 - 11.5  | yes       | yes  | L2, MALL          |
 * | GFX12           | -         | yes  | MALL              |
 *
 */
void
radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radv_cmd_stream *cs, uint64_t va, unsigned size,
                        bool predicating)
{
   const struct radv_physical_device *pdev = radv_device_physical(device);
   struct radeon_winsys *ws = device->ws;
   enum amd_gfx_level gfx_level = pdev->info.gfx_level;
   uint32_t header = 0, command = 0;

   if (gfx_level >= GFX11)
      size = MIN2(size, 32768 - SI_CPDMA_ALIGNMENT);

   assert(size <= cp_dma_max_byte_count(gfx_level));

   radeon_check_space(ws, cs->b, 9);

   uint64_t aligned_va = va & ~(SI_CPDMA_ALIGNMENT - 1);
   uint64_t aligned_size = ((va + size + SI_CPDMA_ALIGNMENT - 1) & ~(SI_CPDMA_ALIGNMENT - 1)) - aligned_va;

   if (gfx_level >= GFX9) {
      command |= S_415_BYTE_COUNT_GFX9(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX9(1);
      header |= S_411_DST_SEL(V_411_NOWHERE);
   } else {
      command |= S_415_BYTE_COUNT_GFX6(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX6(1);
      header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2);
   }

   header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);

   radeon_begin(cs);
   radeon_emit(PKT3(PKT3_DMA_DATA, 5, predicating));
   radeon_emit(header);
   radeon_emit(aligned_va);       /* SRC_ADDR_LO [31:0] */
   radeon_emit(aligned_va >> 32); /* SRC_ADDR_HI [31:0] */
   radeon_emit(aligned_va);       /* DST_ADDR_LO [31:0] */
   radeon_emit(aligned_va >> 32); /* DST_ADDR_HI [31:0] */
   radeon_emit(command);
   radeon_end();
}

void
radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size)
{
   struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);

   radv_cs_cp_dma_prefetch(device, cmd_buffer->cs, va, size, cmd_buffer->state.predicating);

   if (radv_device_fault_detection_enabled(device))
      radv_cmd_buffer_trace_emit(cmd_buffer);
}

static void
radv_cp_dma_prepare(struct radv_cmd_buffer *cmd_buffer, uint64_t byte_count, uint64_t remaining_size, unsigned *flags)
{

   /* Flush the caches for the first copy only.
    * Also wait for the previous CP DMA operations.
    */
   if (cmd_buffer->state.flush_bits) {
      radv_emit_cache_flush(cmd_buffer);
      *flags |= CP_DMA_RAW_WAIT;
   }

   /* Do the synchronization after the last dma, so that all data
    * is written to memory.
    */
   if (byte_count == remaining_size)
      *flags |= CP_DMA_SYNC;
}

static void
radv_cp_dma_realign_engine(struct radv_cmd_buffer *cmd_buffer, unsigned size)
{
   uint64_t va;
   uint32_t offset;
   unsigned dma_flags = 0;
   unsigned buf_size = SI_CPDMA_ALIGNMENT * 2;
   void *ptr;

   assert(size < SI_CPDMA_ALIGNMENT);

   radv_cmd_buffer_upload_alloc(cmd_buffer, buf_size, &offset, &ptr);

   va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
   va += offset;

   radv_cp_dma_prepare(cmd_buffer, size, size, &dma_flags);

   radv_emit_cp_dma(cmd_buffer, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags);
}

void
radv_cp_dma_copy_memory(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size)
{
   struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
   const struct radv_physical_device *pdev = radv_device_physical(device);
   enum amd_gfx_level gfx_level = pdev->info.gfx_level;
   uint64_t main_src_va, main_dest_va;
   uint64_t skipped_size = 0, realign_size = 0;

   if (!(pdev->info.cp_dma_use_L2 && pdev->info.gfx_level >= GFX9)) {
      /* Invalidate L2 in case "src_va" or "dest_va" were previously written through L2. */
      cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_INV_L2;
   }

   /* Assume that we are not going to sync after the last DMA operation. */
   cmd_buffer->state.dma_is_busy = true;

   if (pdev->info.family <= CHIP_CARRIZO || pdev->info.family == CHIP_STONEY) {
      /* If the size is not aligned, we must add a dummy copy at the end
       * just to align the internal counter. Otherwise, the DMA engine
       * would slow down by an order of magnitude for following copies.
       */
      if (size % SI_CPDMA_ALIGNMENT)
         realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);

      /* If the copy begins unaligned, we must start copying from the next
       * aligned block and the skipped part should be copied after everything
       * else has been copied. Only the src alignment matters, not dst.
       */
      if (src_va % SI_CPDMA_ALIGNMENT) {
         skipped_size = SI_CPDMA_ALIGNMENT - (src_va % SI_CPDMA_ALIGNMENT);
         /* The main part will be skipped if the size is too small. */
         skipped_size = MIN2(skipped_size, size);
         size -= skipped_size;
      }
   }
   main_src_va = src_va + skipped_size;
   main_dest_va = dest_va + skipped_size;

   while (size) {
      unsigned dma_flags = 0;
      unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level));

      if (pdev->info.gfx_level >= GFX9) {
         /* DMA operations via L2 are coherent and faster.
          * TODO: GFX7-GFX8 should also support this but it
          * requires tests/benchmarks.
          *
          * Also enable on GFX9 so we can use L2 at rest on GFX9+. On Raven
          * this didn't seem to be worse.
          *
          * Note that we only use CP DMA for sizes < RADV_BUFFER_OPS_CS_THRESHOLD,
          * which is 4k at the moment, so this is really unlikely to cause
          * significant thrashing.
          */
         dma_flags |= CP_DMA_USE_L2;
      }

      radv_cp_dma_prepare(cmd_buffer, byte_count, size + skipped_size + realign_size, &dma_flags);

      dma_flags &= ~CP_DMA_SYNC;

      radv_emit_cp_dma(cmd_buffer, main_dest_va, main_src_va, byte_count, dma_flags);

      size -= byte_count;
      main_src_va += byte_count;
      main_dest_va += byte_count;
   }

   if (skipped_size) {
      unsigned dma_flags = 0;

      radv_cp_dma_prepare(cmd_buffer, skipped_size, size + skipped_size + realign_size, &dma_flags);

      radv_emit_cp_dma(cmd_buffer, dest_va, src_va, skipped_size, dma_flags);
   }
   if (realign_size)
      radv_cp_dma_realign_engine(cmd_buffer, realign_size);
}

void
radv_cp_dma_fill_memory(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value)
{
   struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
   const struct radv_physical_device *pdev = radv_device_physical(device);

   if (!size)
      return;

   if (!(pdev->info.cp_dma_use_L2 && pdev->info.gfx_level >= GFX9)) {
      /* Invalidate L2 in case "va" was previously written through L2. */
      cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_INV_L2;
   }

   assert(va % 4 == 0 && size % 4 == 0);

   enum amd_gfx_level gfx_level = pdev->info.gfx_level;

   /* Assume that we are not going to sync after the last DMA operation. */
   cmd_buffer->state.dma_is_busy = true;

   while (size) {
      unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level));
      unsigned dma_flags = CP_DMA_CLEAR;

      if (pdev->info.gfx_level >= GFX9) {
         /* DMA operations via L2 are coherent and faster.
          * TODO: GFX7-GFX8 should also support this but it
          * requires tests/benchmarks.
          *
          * Also enable on GFX9 so we can use L2 at rest on GFX9+.
          */
         dma_flags |= CP_DMA_USE_L2;
      }

      radv_cp_dma_prepare(cmd_buffer, byte_count, size, &dma_flags);

      /* Emit the clear packet. */
      radv_emit_cp_dma(cmd_buffer, va, value, byte_count, dma_flags);

      size -= byte_count;
      va += byte_count;
   }
}

void
radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer)
{
   struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
   const struct radv_physical_device *pdev = radv_device_physical(device);

   if (pdev->info.gfx_level < GFX7)
      return;

   if (!cmd_buffer->state.dma_is_busy)
      return;

   /* Issue a dummy DMA that copies zero bytes.
    *
    * The DMA engine will see that there's no work to do and skip this
    * DMA request, however, the CP will see the sync flag and still wait
    * for all DMAs to complete.
    */
   radv_emit_cp_dma(cmd_buffer, 0, 0, 0, CP_DMA_SYNC);

   cmd_buffer->state.dma_is_busy = false;
}