radeonsi: determine secure flag must be set for gfx IB

Reviewed-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4401>

src/gallium/drivers/radeonsi/si_compute.c

@@ -806,6 +806,15 @@ static void si_launch_grid(struct pipe_context *ctx, const struct pipe_grid_info
 
    si_need_gfx_cs_space(sctx);
 
+   /* If we're using a secure context, determine if cs must be secure or not */
+   if (unlikely(sctx->ws->ws_is_secure(sctx->ws))) {
+      bool secure = si_compute_resources_check_encrypted(sctx);
+      if (secure != sctx->ws->cs_is_secure(sctx->gfx_cs)) {
+         si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
+         sctx->ws->cs_set_secure(sctx->gfx_cs, secure);
+      }
+   }
+
    if (sctx->bo_list_add_all_compute_resources)
       si_compute_resources_add_all_to_bo_list(sctx);
 

src/gallium/drivers/radeonsi/si_cp_dma.c

@@ -336,6 +336,17 @@ void si_cp_dma_copy_buffer(struct si_context *sctx, struct pipe_resource *dst,
       }
    }
 
+   /* TMZ handling */
+   if (unlikely(sctx->ws->ws_is_secure(sctx->ws) &&
+                !(user_flags & SI_CPDMA_SKIP_TMZ))) {
+      bool secure = src && (si_resource(src)->flags & RADEON_FLAG_ENCRYPTED);
+      assert(!secure || (!dst || (si_resource(dst)->flags & RADEON_FLAG_ENCRYPTED)));
+      if (secure != sctx->ws->cs_is_secure(sctx->gfx_cs)) {
+         si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
+         sctx->ws->cs_set_secure(sctx->gfx_cs, secure);
+      }
+   }
+
    /* Flush the caches. */
    if ((dst || src) && !(user_flags & SI_CPDMA_SKIP_GFX_SYNC)) {
       sctx->flags |= SI_CONTEXT_PS_PARTIAL_FLUSH | SI_CONTEXT_CS_PARTIAL_FLUSH |

src/gallium/drivers/radeonsi/si_descriptors.c

@@ -54,6 +54,7 @@
  */
 
 #include "si_pipe.h"
+#include "si_compute.h"
 #include "sid.h"
 #include "util/format/u_format.h"
 #include "util/hash_table.h"
@@ -256,6 +257,23 @@ static void si_sampler_views_begin_new_cs(struct si_context *sctx, struct si_sam
    }
 }
 
+static bool si_sampler_views_check_encrypted(struct si_context *sctx, struct si_samplers *samplers,
+                                             unsigned samplers_declared)
+{
+   unsigned mask = samplers->enabled_mask & samplers_declared;
+
+   /* Verify if a samplers uses an encrypted resource */
+   while (mask) {
+      int i = u_bit_scan(&mask);
+      struct si_sampler_view *sview = (struct si_sampler_view *)samplers->views[i];
+
+      struct si_resource *res = si_resource(sview->base.texture);
+      if (res->flags & RADEON_FLAG_ENCRYPTED)
+         return true;
+   }
+   return false;
+}
+
 /* Set buffer descriptor fields that can be changed by reallocations. */
 static void si_set_buf_desc_address(struct si_resource *buf, uint64_t offset, uint32_t *state)
 {
@@ -609,6 +627,24 @@ static void si_image_views_begin_new_cs(struct si_context *sctx, struct si_image
    }
 }
 
+static bool si_image_views_check_encrypted(struct si_context *sctx, struct si_images *images,
+                                           unsigned images_declared)
+{
+   uint mask = images->enabled_mask & images_declared;
+
+   while (mask) {
+      int i = u_bit_scan(&mask);
+      struct pipe_image_view *view = &images->views[i];
+
+      assert(view->resource);
+
+      struct si_texture *tex = (struct si_texture *)view->resource;
+      if (tex->buffer.flags & RADEON_FLAG_ENCRYPTED)
+         return true;
+   }
+   return false;
+}
+
 static void si_disable_shader_image(struct si_context *ctx, unsigned shader, unsigned slot)
 {
    struct si_images *images = &ctx->images[shader];
@@ -955,6 +991,23 @@ static void si_buffer_resources_begin_new_cs(struct si_context *sctx,
    }
 }
 
+static bool si_buffer_resources_check_encrypted(struct si_context *sctx,
+                                                struct si_buffer_resources *buffers)
+{
+   unsigned mask = buffers->enabled_mask;
+
+   while (mask) {
+      int i = u_bit_scan(&mask);
+
+      /* only check for reads */
+      if ((buffers->writable_mask & (1u << i)) == 0 &&
+          (si_resource(buffers->buffers[i])->flags & RADEON_FLAG_ENCRYPTED))
+         return true;
+   }
+
+   return false;
+}
+
 static void si_get_buffer_from_descriptors(struct si_buffer_resources *buffers,
                                            struct si_descriptors *descs, unsigned idx,
                                            struct pipe_resource **buf, unsigned *offset,
@@ -2655,6 +2708,52 @@ void si_release_all_descriptors(struct si_context *sctx)
    si_release_bindless_descriptors(sctx);
 }
 
+bool si_gfx_resources_check_encrypted(struct si_context *sctx)
+{
+   bool use_encrypted_bo = false;
+   struct si_shader_ctx_state *current_shader[SI_NUM_SHADERS] = {
+      [PIPE_SHADER_VERTEX] = &sctx->vs_shader,
+      [PIPE_SHADER_TESS_CTRL] = &sctx->tcs_shader,
+      [PIPE_SHADER_TESS_EVAL] = &sctx->tes_shader,
+      [PIPE_SHADER_GEOMETRY] = &sctx->gs_shader,
+      [PIPE_SHADER_FRAGMENT] = &sctx->ps_shader,
+   };
+
+   for (unsigned i = 0; i < SI_NUM_GRAPHICS_SHADERS && !use_encrypted_bo; i++) {
+      if (!current_shader[i]->cso)
+         continue;
+
+      use_encrypted_bo |=
+         si_buffer_resources_check_encrypted(sctx, &sctx->const_and_shader_buffers[i]);
+      use_encrypted_bo |=
+         si_sampler_views_check_encrypted(sctx, &sctx->samplers[i],
+                                          current_shader[i]->cso->info.samplers_declared);
+      use_encrypted_bo |= si_image_views_check_encrypted(sctx, &sctx->images[i],
+                                          current_shader[i]->cso->info.images_declared);
+   }
+   use_encrypted_bo |= si_buffer_resources_check_encrypted(sctx, &sctx->rw_buffers);
+
+   struct si_state_blend *blend = sctx->queued.named.blend;
+   for (int i = 0; i < sctx->framebuffer.state.nr_cbufs && !use_encrypted_bo; i++) {
+      struct pipe_surface *surf = sctx->framebuffer.state.cbufs[i];
+      if (surf && surf->texture) {
+         struct si_texture *tex = (struct si_texture *)surf->texture;
+         if (!(tex->buffer.flags & RADEON_FLAG_ENCRYPTED))
+            continue;
+         /* Are we reading from this framebuffer (blend) */
+         if ((blend->blend_enable_4bit >> (4 * i)) & 0xf) {
+            /* TODO: blend op */
+            use_encrypted_bo = true;
+         }
+      }
+   }
+
+   /* TODO: we should assert that either use_encrypted_bo is false,
+    * or all writable buffers are encrypted.
+    */
+   return use_encrypted_bo;
+}
+
 void si_gfx_resources_add_all_to_bo_list(struct si_context *sctx)
 {
    for (unsigned i = 0; i < SI_NUM_GRAPHICS_SHADERS; i++) {
@@ -2672,6 +2771,21 @@ void si_gfx_resources_add_all_to_bo_list(struct si_context *sctx)
    sctx->bo_list_add_all_gfx_resources = false;
 }
 
+bool si_compute_resources_check_encrypted(struct si_context *sctx)
+{
+   unsigned sh = PIPE_SHADER_COMPUTE;
+
+   struct si_shader_info* info = &sctx->cs_shader_state.program->sel.info;
+
+   /* TODO: we should assert that either use_encrypted_bo is false,
+    * or all writable buffers are encrypted.
+    */
+   return si_buffer_resources_check_encrypted(sctx, &sctx->const_and_shader_buffers[sh]) ||
+          si_sampler_views_check_encrypted(sctx, &sctx->samplers[sh], info->samplers_declared) ||
+          si_image_views_check_encrypted(sctx, &sctx->images[sh], info->images_declared) ||
+          si_buffer_resources_check_encrypted(sctx, &sctx->rw_buffers);
+}
+
 void si_compute_resources_add_all_to_bo_list(struct si_context *sctx)
 {
    unsigned sh = PIPE_SHADER_COMPUTE;

src/gallium/drivers/radeonsi/si_pipe.h

@@ -1336,9 +1336,10 @@ void si_init_compute_blit_functions(struct si_context *sctx);
 #define SI_CPDMA_SKIP_SYNC_BEFORE    (1 << 2) /* don't wait for DMA before the copy (RAW hazards) */
 #define SI_CPDMA_SKIP_GFX_SYNC       (1 << 3) /* don't flush caches and don't wait for PS/CS */
 #define SI_CPDMA_SKIP_BO_LIST_UPDATE (1 << 4) /* don't update the BO list */
+#define SI_CPDMA_SKIP_TMZ            (1 << 5) /* don't update tmz state */
 #define SI_CPDMA_SKIP_ALL                                                                          \
    (SI_CPDMA_SKIP_CHECK_CS_SPACE | SI_CPDMA_SKIP_SYNC_AFTER | SI_CPDMA_SKIP_SYNC_BEFORE |          \
-    SI_CPDMA_SKIP_GFX_SYNC | SI_CPDMA_SKIP_BO_LIST_UPDATE)
+    SI_CPDMA_SKIP_GFX_SYNC | SI_CPDMA_SKIP_BO_LIST_UPDATE | SI_CPDMA_SKIP_TMZ)
 
 void si_cp_dma_wait_for_idle(struct si_context *sctx);
 void si_cp_dma_clear_buffer(struct si_context *sctx, struct radeon_cmdbuf *cs,

src/gallium/drivers/radeonsi/si_state.h

@@ -501,6 +501,8 @@ bool si_upload_compute_shader_descriptors(struct si_context *sctx);
 void si_release_all_descriptors(struct si_context *sctx);
 void si_gfx_resources_add_all_to_bo_list(struct si_context *sctx);
 void si_compute_resources_add_all_to_bo_list(struct si_context *sctx);
+bool si_gfx_resources_check_encrypted(struct si_context *sctx);
+bool si_compute_resources_check_encrypted(struct si_context *sctx);
 void si_all_descriptors_begin_new_cs(struct si_context *sctx);
 void si_upload_const_buffer(struct si_context *sctx, struct si_resource **buf, const uint8_t *ptr,
                             unsigned size, uint32_t *const_offset);

src/gallium/drivers/radeonsi/si_state_draw.c

@@ -1919,6 +1919,15 @@ static void si_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *i
 
    si_need_gfx_cs_space(sctx);
 
+   /* If we're using a secure context, determine if cs must be secure or not */
+   if (unlikely(sctx->ws->ws_is_secure(sctx->ws))) {
+      bool secure = si_gfx_resources_check_encrypted(sctx);
+      if (secure != sctx->ws->cs_is_secure(sctx->gfx_cs)) {
+         si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
+         sctx->ws->cs_set_secure(sctx->gfx_cs, secure);
+      }
+   }
+
    if (sctx->bo_list_add_all_gfx_resources)
       si_gfx_resources_add_all_to_bo_list(sctx);