From fa8bbfaa55a43e6e23fbb3882e2c370afe4b0755 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Wed, 11 Jan 2023 08:12:22 +1000 Subject: [PATCH] nouveau/winsys: add support for the vma bind interfaces Reviewed-by: Faith Ekstrand Part-of: --- src/nouveau/winsys/nouveau_bo.c | 149 +++++++++++++++++++++++++++- src/nouveau/winsys/nouveau_bo.h | 18 ++++ src/nouveau/winsys/nouveau_device.c | 29 ++++++ src/nouveau/winsys/nouveau_device.h | 12 +++ 4 files changed, 207 insertions(+), 1 deletion(-) diff --git a/src/nouveau/winsys/nouveau_bo.c b/src/nouveau/winsys/nouveau_bo.c index b4ed0ab4d6b..168234ee0da 100644 --- a/src/nouveau/winsys/nouveau_bo.c +++ b/src/nouveau/winsys/nouveau_bo.c @@ -2,12 +2,122 @@ #include "drm-uapi/nouveau_drm.h" #include "util/hash_table.h" +#include "util/u_math.h" +#include #include #include #include #include +#if NVK_NEW_UAPI == 1 +static void +bo_bind(struct nouveau_ws_device *dev, + uint32_t handle, uint64_t addr, + uint64_t range, uint64_t bo_offset, + uint32_t flags) +{ + int ret; + + struct drm_nouveau_vm_bind_op newbindop = { + .op = DRM_NOUVEAU_VM_BIND_OP_MAP, + .handle = handle, + .addr = addr, + .range = range, + .bo_offset = bo_offset, + .flags = flags, + }; + struct drm_nouveau_vm_bind vmbind = { + .op_count = 1, + .op_ptr = (uint64_t)(uintptr_t)(void *)&newbindop, + }; + ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_VM_BIND, &vmbind, sizeof(vmbind)); + if (ret) + fprintf(stderr, "vm bind failed %d\n", errno); + assert(ret == 0); +} + +static void +bo_unbind(struct nouveau_ws_device *dev, + uint64_t offset, uint64_t range, + uint32_t flags) +{ + struct drm_nouveau_vm_bind_op newbindop = { + .op = DRM_NOUVEAU_VM_BIND_OP_UNMAP, + .addr = offset, + .range = range, + .flags = flags, + }; + struct drm_nouveau_vm_bind vmbind = { + .op_count = 1, + .op_ptr = (uint64_t)(uintptr_t)(void *)&newbindop, + }; + ASSERTED int ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_VM_BIND, &vmbind, sizeof(vmbind)); + assert(ret == 0); +} + +uint64_t +nouveau_ws_alloc_vma(struct nouveau_ws_device *dev, + uint64_t size, uint64_t align, + bool sparse_resident) +{ + uint64_t offset; + simple_mtx_lock(&dev->vma_mutex); + offset = util_vma_heap_alloc(&dev->vma_heap, size, align); + simple_mtx_unlock(&dev->vma_mutex); + + if (dev->debug_flags & NVK_DEBUG_VM) + fprintf(stderr, "alloc vma %" PRIx64 " %" PRIx64 " sparse: %d\n", + offset, size, sparse_resident); + + if (sparse_resident) + bo_bind(dev, 0, offset, size, 0, DRM_NOUVEAU_VM_BIND_SPARSE); + + return offset; +} + +void +nouveau_ws_free_vma(struct nouveau_ws_device *dev, + uint64_t offset, uint64_t size, + bool sparse_resident) +{ + if (dev->debug_flags & NVK_DEBUG_VM) + fprintf(stderr, "free vma %" PRIx64 " %" PRIx64 "\n", + offset, size); + + if (sparse_resident) + bo_unbind(dev, offset, size, DRM_NOUVEAU_VM_BIND_SPARSE); + + simple_mtx_lock(&dev->vma_mutex); + util_vma_heap_free(&dev->vma_heap, offset, size); + simple_mtx_unlock(&dev->vma_mutex); +} + +void +nouveau_ws_bo_unbind_vma(struct nouveau_ws_device *dev, + uint64_t offset, uint64_t range) +{ + if (dev->debug_flags & NVK_DEBUG_VM) + fprintf(stderr, "unbind vma %" PRIx64 " %" PRIx64 "\n", + offset, range); + bo_unbind(dev, offset, range, 0); +} + +void +nouveau_ws_bo_bind_vma(struct nouveau_ws_device *dev, + struct nouveau_ws_bo *bo, + uint64_t addr, + uint64_t range, + uint64_t bo_offset, + uint32_t pte_kind) +{ + if (dev->debug_flags & NVK_DEBUG_VM) + fprintf(stderr, "bind vma %x %" PRIx64 " %" PRIx64 " %" PRIx64 " %d\n", + bo->handle, addr, range, bo_offset, pte_kind); + bo_bind(dev, bo->handle, addr, range, bo_offset, pte_kind); +} +#endif + struct nouveau_ws_bo * nouveau_ws_bo_new(struct nouveau_ws_device *dev, uint64_t size, uint64_t align, @@ -51,7 +161,15 @@ nouveau_ws_bo_new_tiled(struct nouveau_ws_device *dev, if (align == 0) align = 0x1000; + /* Align the size */ + size = ALIGN(size, align); + +#if NVK_NEW_UAPI == 0 req.info.domain = NOUVEAU_GEM_TILE_NONCONTIG; +#else + req.info.domain = 0; +#endif + if (flags & NOUVEAU_WS_BO_GART) req.info.domain |= NOUVEAU_GEM_DOMAIN_GART; else @@ -60,10 +178,12 @@ nouveau_ws_bo_new_tiled(struct nouveau_ws_device *dev, if (flags & NOUVEAU_WS_BO_MAP) req.info.domain |= NOUVEAU_GEM_DOMAIN_MAPPABLE; +#if NVK_NEW_UAPI == 0 assert(pte_kind == 0 || !(flags & NOUVEAU_WS_BO_GART)); assert(tile_mode == 0 || !(flags & NOUVEAU_WS_BO_GART)); req.info.tile_flags = (uint32_t)pte_kind << 8; req.info.tile_mode = tile_mode; +#endif req.info.size = size; req.align = align; @@ -72,14 +192,25 @@ nouveau_ws_bo_new_tiled(struct nouveau_ws_device *dev, int ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_GEM_NEW, &req, sizeof(req)); if (ret == 0) { - bo->size = req.info.size; + bo->size = size; + bo->align = align; +#if NVK_NEW_UAPI == 0 bo->offset = req.info.offset; +#else + bo->offset = -1ULL; +#endif bo->handle = req.info.handle; bo->map_handle = req.info.map_handle; bo->dev = dev; bo->flags = flags; bo->refcnt = 1; +#if NVK_NEW_UAPI == 1 + assert(pte_kind == 0); + bo->offset = nouveau_ws_alloc_vma(dev, bo->size, align, false); + nouveau_ws_bo_bind_vma(dev, bo, bo->offset, bo->size, 0, 0); +#endif + _mesa_hash_table_insert(dev->bos, (void *)(uintptr_t)bo->handle, bo); } else { FREE(bo); @@ -127,6 +258,16 @@ nouveau_ws_bo_from_dma_buf(struct nouveau_ws_device *dev, int fd) bo->flags = flags; bo->refcnt = 1; +#if NVK_NEW_UAPI == 1 + uint64_t align = (1ULL << 12); + if (info.domain & NOUVEAU_GEM_DOMAIN_VRAM) + align = (1ULL << 16); + + assert(bo->size == ALIGN(bo->size, align)); + + bo->offset = nouveau_ws_alloc_vma(dev, bo->size, align, false); + nouveau_ws_bo_bind_vma(dev, bo, bo->offset, bo->size, 0, 0); +#endif _mesa_hash_table_insert(dev->bos, (void *)(uintptr_t)handle, bo); } } @@ -148,6 +289,12 @@ nouveau_ws_bo_destroy(struct nouveau_ws_bo *bo) simple_mtx_lock(&dev->bos_lock); _mesa_hash_table_remove_key(dev->bos, (void *)(uintptr_t)bo->handle); + +#if NVK_NEW_UAPI == 1 + nouveau_ws_bo_unbind_vma(bo->dev, bo->offset, bo->size); + nouveau_ws_free_vma(bo->dev, bo->offset, bo->size, false); +#endif + drmCloseBufferHandle(bo->dev->fd, bo->handle); FREE(bo); diff --git a/src/nouveau/winsys/nouveau_bo.h b/src/nouveau/winsys/nouveau_bo.h index 63fdb909ccc..2191a957909 100644 --- a/src/nouveau/winsys/nouveau_bo.h +++ b/src/nouveau/winsys/nouveau_bo.h @@ -34,6 +34,7 @@ enum nouveau_ws_bo_map_flags { struct nouveau_ws_bo { uint64_t size; uint64_t offset; + uint64_t align; uint64_t map_handle; struct nouveau_ws_device *dev; uint32_t handle; @@ -41,6 +42,23 @@ struct nouveau_ws_bo { atomic_uint_fast32_t refcnt; }; +#if NVK_NEW_UAPI == 1 +uint64_t nouveau_ws_alloc_vma(struct nouveau_ws_device *dev, + uint64_t size, uint64_t align, bool sparse); + +void nouveau_ws_free_vma(struct nouveau_ws_device *dev, + uint64_t offset, uint64_t size, bool sparse); + +void nouveau_ws_bo_bind_vma(struct nouveau_ws_device *dev, + struct nouveau_ws_bo *bo, + uint64_t addr, + uint64_t range, + uint64_t bo_offset, + uint32_t pte_kind); +void nouveau_ws_bo_unbind_vma(struct nouveau_ws_device *dev, + uint64_t offset, uint64_t range); +#endif + struct nouveau_ws_bo *nouveau_ws_bo_new(struct nouveau_ws_device *, uint64_t size, uint64_t align, enum nouveau_ws_bo_flags); diff --git a/src/nouveau/winsys/nouveau_device.c b/src/nouveau/winsys/nouveau_device.c index 478ba8efef0..da9f6301728 100644 --- a/src/nouveau/winsys/nouveau_device.c +++ b/src/nouveau/winsys/nouveau_device.c @@ -130,6 +130,7 @@ nouveau_ws_device_set_dbg_flags(struct nouveau_ws_device *dev) { "push_dump", NVK_DEBUG_PUSH_DUMP }, { "push_sync", NVK_DEBUG_PUSH_SYNC }, { "zero_memory", NVK_DEBUG_ZERO_MEMORY }, + { "vm", NVK_DEBUG_VM }, { NULL, 0 }, }; @@ -258,6 +259,14 @@ nouveau_ws_device_new(drmDevicePtr drm_device) goto out_err; } +#if NVK_NEW_UAPI == 1 + const uint64_t TOP = 1ull << 40; + const uint64_t KERN = 1ull << 39; + util_vma_heap_init(&device->vma_heap, 4096, (TOP - KERN) - 4096); + simple_mtx_init(&device->vma_mutex, mtx_plain); + device->vma_heap.alloc_high = false; +#endif + uint32_t version = ver->version_major << 24 | ver->version_minor << 8 | @@ -265,8 +274,21 @@ nouveau_ws_device_new(drmDevicePtr drm_device) drmFreeVersion(ver); ver = NULL; +#if NVK_NEW_UAPI == 1 + /* don't work on older kernels */ + if (version < 0x01000400) + goto out_err; +#else if (version < 0x01000301) goto out_err; +#endif + +#if NVK_NEW_UAPI == 1 + /* start the new VM mode */ + struct drm_nouveau_vm_init vminit = { TOP-KERN, KERN }; + ASSERTED int ret = drmCommandWrite(fd, DRM_NOUVEAU_VM_INIT, &vminit, sizeof(vminit)); + assert(!ret); +#endif if (nouveau_ws_device_alloc(fd, device)) goto out_err; @@ -345,6 +367,13 @@ nouveau_ws_device_destroy(struct nouveau_ws_device *device) _mesa_hash_table_destroy(device->bos, NULL); simple_mtx_destroy(&device->bos_lock); + +#if NVK_NEW_UAPI == 1 + util_vma_heap_finish(&device->vma_heap); + simple_mtx_destroy(&device->vma_mutex); +#endif + + close(device->fd); FREE(device); } diff --git a/src/nouveau/winsys/nouveau_device.h b/src/nouveau/winsys/nouveau_device.h index 8d78caf45aa..945c143c082 100644 --- a/src/nouveau/winsys/nouveau_device.h +++ b/src/nouveau/winsys/nouveau_device.h @@ -4,6 +4,7 @@ #include "nouveau_private.h" #include "nv_device_info.h" #include "util/simple_mtx.h" +#include "util/vma.h" #include @@ -14,6 +15,8 @@ struct hash_table; extern "C" { #endif +#define NVK_NEW_UAPI 0 + enum nvk_debug { /* dumps all push buffers after submission */ NVK_DEBUG_PUSH_DUMP = 1ull << 0, @@ -28,6 +31,10 @@ enum nvk_debug { /* Zero all client memory allocations */ NVK_DEBUG_ZERO_MEMORY = 1ull << 2, + + /* Dump VM bind/unbinds + */ + NVK_DEBUG_VM = 1ull << 3, }; struct nouveau_ws_device { @@ -41,6 +48,11 @@ struct nouveau_ws_device { simple_mtx_t bos_lock; struct hash_table *bos; + +#if NVK_NEW_UAPI == 1 + struct util_vma_heap vma_heap; + simple_mtx_t vma_mutex; +#endif }; struct nouveau_ws_device *nouveau_ws_device_new(struct _drmDevice *drm_device);