nvk: Allocate shaders from a heap

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24326>

src/nouveau/vulkan/nvk_cmd_buffer.c

@@ -255,11 +255,6 @@ nvk_CmdBindPipeline(VkCommandBuffer commandBuffer,
    struct nvk_device *dev = nvk_cmd_buffer_device(cmd);
 
    for (unsigned s = 0; s < ARRAY_SIZE(pipeline->shaders); s++) {
-      if (!pipeline->shaders[s].bo)
-         continue;
-
-      nvk_cmd_buffer_ref_bo(cmd, pipeline->shaders[s].bo);
-
       if (pipeline->shaders[s].slm_size)
          nvk_device_ensure_slm(dev, pipeline->shaders[s].slm_size);
    }

src/nouveau/vulkan/nvk_graphics_pipeline.c

@@ -254,11 +254,11 @@ nvk_graphics_pipeline_create(struct nvk_device *device,
       uint32_t idx = mesa_to_nv9097_shader_type[stage];
 
       P_IMMD(p, NV9097, SET_PIPELINE_SHADER(idx), {
-         .enable  = shader->bo != NULL,
+         .enable  = shader->upload_size > 0,
          .type    = mesa_to_nv9097_shader_type[stage],
       });
 
-      if (shader->bo == NULL)
+      if (shader->upload_size == 0)
          continue;
 
       if (stage != MESA_SHADER_FRAGMENT)

src/nouveau/vulkan/nvk_pipeline.c

@@ -29,8 +29,11 @@ nvk_pipeline_free(struct nvk_device *device,
                   const VkAllocationCallbacks *pAllocator)
 {
    for (uint32_t s = 0; s < ARRAY_SIZE(pipeline->shaders); s++) {
-      if (pipeline->shaders[s].bo)
-         nouveau_ws_bo_destroy(pipeline->shaders[s].bo);
+      if (pipeline->shaders[s].upload_size > 0) {
+         nvk_heap_free(device, &device->shader_heap,
+                       pipeline->shaders[s].upload_addr,
+                       pipeline->shaders[s].upload_size);
+      }
    }
 
    vk_object_free(&device->vk, pAllocator, pipeline);

src/nouveau/vulkan/nvk_queue_drm_nouveau.c

@@ -143,6 +143,15 @@ push_add_queue_state(struct push_builder *pb, struct nvk_queue_state *qs)
       push_add_push(pb, qs->push.bo, 0, qs->push.dw_count);
 }
 
+static void
+push_add_heap(struct push_builder *pb, struct nvk_heap *heap)
+{
+   simple_mtx_lock(&heap->mutex);
+   for (uint32_t i = 0; i < heap->bo_count; i++)
+      push_add_bo(pb, heap->bos[i].bo, NOUVEAU_WS_BO_RDWR);
+   simple_mtx_unlock(&heap->mutex);
+}
+
 VkResult
 nvk_queue_submit_drm_nouveau(struct nvk_queue *queue,
                              struct vk_queue_submit *submit,
@@ -166,6 +175,8 @@ nvk_queue_submit_drm_nouveau(struct nvk_queue *queue,
    } else {
       push_add_queue_state(&pb, &queue->state);
 
+      push_add_heap(&pb, &dev->shader_heap);
+
       simple_mtx_lock(&dev->memory_objects_lock);
       list_for_each_entry(struct nvk_device_memory, mem,
                           &dev->memory_objects, link) {

src/nouveau/vulkan/nvk_shader.c

@@ -722,25 +722,24 @@ nvk_shader_upload(struct nvk_device *dev, struct nvk_shader *shader)
          hdr_size = GF100_SHADER_HEADER_SIZE;
    }
 
-   /* TODO: The I-cache pre-fetches and we don't really know by how much.  So
-    * throw on a bunch just to be sure.
-    */
    uint32_t total_size = shader->code_size + hdr_size;
-   shader->bo = nouveau_ws_bo_new(nvk_device_physical(dev)->dev,
-                                  total_size + 4096, 256,
-                                  NOUVEAU_WS_BO_LOCAL | NOUVEAU_WS_BO_MAP);
+   char *data = malloc(total_size);
+   if (data == NULL)
+      return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
 
-   void *ptr = nouveau_ws_bo_map(shader->bo, NOUVEAU_WS_BO_WR);
-
-   assert(hdr_size <= sizeof(shader->hdr));
-   memcpy(ptr, shader->hdr, hdr_size);
-   memcpy(ptr + hdr_size, shader->code_ptr, shader->code_size);
-   nouveau_ws_bo_unmap(shader->bo, ptr);
+   memcpy(data, shader->hdr, hdr_size);
+   memcpy(data + hdr_size, shader->code_ptr, shader->code_size);
 
 #ifndef NDEBUG
    if (debug_get_bool_option("NV50_PROG_DEBUG", false))
       nvk_shader_dump(shader);
 #endif
 
-   return VK_SUCCESS;
+   VkResult result = nvk_heap_upload(dev, &dev->shader_heap, data,
+                                     total_size, 256, &shader->upload_addr);
+   if (result == VK_SUCCESS)
+      shader->upload_size = total_size;
+   free(data);
+
+   return result;
 }

src/nouveau/vulkan/nvk_shader.h

@@ -22,6 +22,9 @@ struct nvk_shader {
    uint8_t *code_ptr;
    uint32_t code_size;
 
+   uint32_t upload_size;
+   uint64_t upload_addr;
+
    uint8_t num_gprs;
    uint8_t num_barriers;
    uint32_t slm_size;
@@ -56,14 +59,12 @@ struct nvk_shader {
       uint32_t smem_size; /* shared memory (TGSI LOCAL resource) size */
       uint32_t block_size[3];
    } cp;
-
-   struct nouveau_ws_bo *bo;
 };
 
 static inline uint64_t
 nvk_shader_address(const struct nvk_shader *shader)
 {
-   return shader->bo->offset;
+   return shader->upload_addr;
 }
 
 const nir_shader_compiler_options *