diff --git a/src/nouveau/vulkan/meson.build b/src/nouveau/vulkan/meson.build index 09faa5b4876..070c1cf3066 100644 --- a/src/nouveau/vulkan/meson.build +++ b/src/nouveau/vulkan/meson.build @@ -29,6 +29,8 @@ nvk_files = files( 'nvk_format.c', 'nvk_format.h', 'nvk_graphics_pipeline.c', + 'nvk_heap.c', + 'nvk_heap.h', 'nvk_image.c', 'nvk_image.h', 'nvk_image_view.c', diff --git a/src/nouveau/vulkan/nvk_heap.c b/src/nouveau/vulkan/nvk_heap.c new file mode 100644 index 00000000000..76c92f41a7b --- /dev/null +++ b/src/nouveau/vulkan/nvk_heap.c @@ -0,0 +1,188 @@ +#include "nvk_heap.h" + +#include "nvk_device.h" +#include "nvk_physical_device.h" +#include "nvk_queue.h" + +#include "util/macros.h" + +#include "nv_push.h" +#include "nvk_cl90b5.h" + +VkResult +nvk_heap_init(struct nvk_device *device, struct nvk_heap *heap, + enum nouveau_ws_bo_flags bo_flags, + enum nouveau_ws_bo_map_flags map_flags, + uint32_t overalloc) +{ + memset(heap, 0, sizeof(*heap)); + + heap->bo_flags = bo_flags; + if (map_flags) + heap->bo_flags |= NOUVEAU_WS_BO_MAP; + heap->map_flags = map_flags; + heap->overalloc = overalloc; + + simple_mtx_init(&heap->mutex, mtx_plain); + util_vma_heap_init(&heap->heap, 0, 0); + + heap->total_size = 0; + heap->bo_count = 0; + + return VK_SUCCESS; +} + +void +nvk_heap_finish(struct nvk_device *dev, struct nvk_heap *heap) +{ + for (uint32_t bo_idx = 0; bo_idx < heap->bo_count; bo_idx++) { + nouveau_ws_bo_unmap(heap->bos[bo_idx].bo, heap->bos[bo_idx].map); + nouveau_ws_bo_destroy(heap->bos[bo_idx].bo); + } + + util_vma_heap_finish(&heap->heap); + simple_mtx_destroy(&heap->mutex); +} + +static uint64_t +encode_vma(uint32_t bo_idx, uint64_t bo_offset) +{ + assert(bo_idx < UINT16_MAX - 1); + assert(bo_offset < (1ull << 48)); + return ((uint64_t)(bo_idx + 1) << 48) | bo_offset; +} + +static uint32_t +vma_bo_idx(uint64_t offset) +{ + offset = offset >> 48; + assert(offset > 0); + return offset - 1; +} + +static uint64_t +vma_bo_offset(uint64_t offset) +{ + return offset & BITFIELD64_MASK(48); +} + +static VkResult +nvk_heap_grow_locked(struct nvk_device *dev, struct nvk_heap *heap) +{ + if (heap->bo_count >= NVK_HEAP_MAX_BO_COUNT) { + return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, + "Heap has already hit its maximum size"); + } + + /* First two BOs are MIN_SIZE, double after that */ + const uint64_t new_bo_size = + NVK_HEAP_MIN_SIZE << (MAX2(heap->bo_count, 1) - 1); + + heap->bos[heap->bo_count].bo = + nouveau_ws_bo_new_mapped(dev->pdev->dev, + new_bo_size + heap->overalloc, 0, + heap->bo_flags, heap->map_flags, + &heap->bos[heap->bo_count].map); + if (heap->bos[heap->bo_count].bo == NULL) { + return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, + "Failed to allocate a heap BO: %m"); + } + + uint64_t vma = encode_vma(heap->bo_count, 0); + util_vma_heap_free(&heap->heap, vma, new_bo_size); + + heap->total_size += new_bo_size; + heap->bo_count++; + + return VK_SUCCESS; +} + +static VkResult +nvk_heap_alloc_locked(struct nvk_device *dev, struct nvk_heap *heap, + uint64_t size, uint32_t alignment, + uint64_t *addr_out, void **map_out) +{ + while (1) { + uint64_t vma = util_vma_heap_alloc(&heap->heap, size, alignment); + if (vma != 0) { + uint32_t bo_idx = vma_bo_idx(vma); + uint64_t bo_offset = vma_bo_offset(vma); + + assert(bo_idx < heap->bo_count); + assert(heap->bos[bo_idx].bo != NULL); + assert(bo_offset + size + heap->overalloc <= + heap->bos[bo_idx].bo->size); + + *addr_out = heap->bos[bo_idx].bo->offset + bo_offset; + *map_out = (char *)heap->bos[bo_idx].map + bo_offset; + + return VK_SUCCESS; + } + + VkResult result = nvk_heap_grow_locked(dev, heap); + if (result != VK_SUCCESS) + return result; + } +} + +static void +nvk_heap_free_locked(struct nvk_device *dev, struct nvk_heap *heap, + uint64_t addr, uint64_t size) +{ + assert(addr + size > addr); + + for (uint32_t bo_idx = 0; bo_idx < heap->bo_count; bo_idx++) { + if (addr < heap->bos[bo_idx].bo->offset) + continue; + + uint64_t bo_offset = addr - heap->bos[bo_idx].bo->offset; + if (bo_offset >= heap->bos[bo_idx].bo->size) + continue; + + assert(bo_offset + size <= heap->bos[bo_idx].bo->size); + uint64_t vma = encode_vma(bo_idx, bo_offset); + + util_vma_heap_free(&heap->heap, vma, size); + + break; + } +} + +VkResult +nvk_heap_alloc(struct nvk_device *dev, struct nvk_heap *heap, + uint64_t size, uint32_t alignment, + uint64_t *addr_out, void **map_out) +{ + simple_mtx_lock(&heap->mutex); + VkResult result = nvk_heap_alloc_locked(dev, heap, size, alignment, + addr_out, map_out); + simple_mtx_unlock(&heap->mutex); + + return result; +} + +VkResult +nvk_heap_upload(struct nvk_device *dev, struct nvk_heap *heap, + const void *data, size_t size, uint32_t alignment, + uint64_t *addr_out) +{ + simple_mtx_lock(&heap->mutex); + + void *map; + VkResult result = nvk_heap_alloc_locked(dev, heap, size, alignment, + addr_out, &map); + if (result == VK_SUCCESS) + memcpy(map, data, size); + simple_mtx_unlock(&heap->mutex); + + return result; +} + +void +nvk_heap_free(struct nvk_device *dev, struct nvk_heap *heap, + uint64_t addr, uint64_t size) +{ + simple_mtx_lock(&heap->mutex); + nvk_heap_free_locked(dev, heap, addr, size); + simple_mtx_unlock(&heap->mutex); +} diff --git a/src/nouveau/vulkan/nvk_heap.h b/src/nouveau/vulkan/nvk_heap.h new file mode 100644 index 00000000000..22053a65c4f --- /dev/null +++ b/src/nouveau/vulkan/nvk_heap.h @@ -0,0 +1,56 @@ +#ifndef NVK_HEAP_H +#define NVK_HEAP_H 1 + +#include "nvk_private.h" + +#include "nouveau_bo.h" +#include "util/simple_mtx.h" +#include "util/vma.h" + +struct nvk_device; + +#define NVK_HEAP_MIN_SIZE_LOG2 16 +#define NVK_HEAP_MAX_SIZE_LOG2 32 +#define NVK_HEAP_MIN_SIZE (1ull << NVK_HEAP_MIN_SIZE_LOG2) +#define NVK_HEAP_MAX_SIZE (1ull << NVK_HEAP_MAX_SIZE_LOG2) +#define NVK_HEAP_MAX_BO_COUNT (NVK_HEAP_MAX_SIZE_LOG2 - \ + NVK_HEAP_MIN_SIZE_LOG2 + 1) + +struct nvk_heap_bo { + struct nouveau_ws_bo *bo; + void *map; +}; + +struct nvk_heap { + enum nouveau_ws_bo_flags bo_flags; + enum nouveau_ws_bo_map_flags map_flags; + uint32_t overalloc; + + simple_mtx_t mutex; + struct util_vma_heap heap; + + uint64_t total_size; + + uint32_t bo_count; + struct nvk_heap_bo bos[NVK_HEAP_MAX_BO_COUNT]; +}; + +VkResult nvk_heap_init(struct nvk_device *dev, struct nvk_heap *heap, + enum nouveau_ws_bo_flags bo_flags, + enum nouveau_ws_bo_map_flags map_flags, + uint32_t overalloc); + +void nvk_heap_finish(struct nvk_device *dev, struct nvk_heap *heap); + +VkResult nvk_heap_alloc(struct nvk_device *dev, struct nvk_heap *heap, + uint64_t size, uint32_t alignment, + uint64_t *addr_out, void **map_out); + +VkResult nvk_heap_upload(struct nvk_device *dev, struct nvk_heap *heap, + const void *data, size_t size, uint32_t alignment, + uint64_t *addr_out); + +void nvk_heap_free(struct nvk_device *dev, struct nvk_heap *heap, + uint64_t addr, uint64_t size); + +#endif /* define NVK_HEAP_H */