nouveau/winsys: add support for the vma bind interfaces

Reviewed-by: Faith Ekstrand <faith.ekstrand@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24326>

src/nouveau/winsys/nouveau_bo.c

@@ -2,12 +2,122 @@
 
 #include "drm-uapi/nouveau_drm.h"
 #include "util/hash_table.h"
+#include "util/u_math.h"
 
+#include <errno.h>
 #include <fcntl.h>
 #include <stddef.h>
 #include <sys/mman.h>
 #include <xf86drm.h>
 
+#if NVK_NEW_UAPI == 1
+static void
+bo_bind(struct nouveau_ws_device *dev,
+        uint32_t handle, uint64_t addr,
+        uint64_t range, uint64_t bo_offset,
+        uint32_t flags)
+{
+   int ret;
+
+   struct drm_nouveau_vm_bind_op newbindop = {
+      .op = DRM_NOUVEAU_VM_BIND_OP_MAP,
+      .handle = handle,
+      .addr = addr,
+      .range = range,
+      .bo_offset = bo_offset,
+      .flags = flags,
+   };
+   struct drm_nouveau_vm_bind vmbind = {
+      .op_count = 1,
+      .op_ptr = (uint64_t)(uintptr_t)(void *)&newbindop,
+   };
+   ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_VM_BIND, &vmbind, sizeof(vmbind));
+   if (ret)
+      fprintf(stderr, "vm bind failed %d\n", errno);
+   assert(ret == 0);
+}
+
+static void
+bo_unbind(struct nouveau_ws_device *dev,
+          uint64_t offset, uint64_t range,
+          uint32_t flags)
+{
+   struct drm_nouveau_vm_bind_op newbindop = {
+      .op = DRM_NOUVEAU_VM_BIND_OP_UNMAP,
+      .addr = offset,
+      .range = range,
+      .flags = flags,
+   };
+   struct drm_nouveau_vm_bind vmbind = {
+      .op_count = 1,
+      .op_ptr = (uint64_t)(uintptr_t)(void *)&newbindop,
+   };
+   ASSERTED int ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_VM_BIND, &vmbind, sizeof(vmbind));
+   assert(ret == 0);
+}
+
+uint64_t
+nouveau_ws_alloc_vma(struct nouveau_ws_device *dev,
+                     uint64_t size, uint64_t align,
+                     bool sparse_resident)
+{
+   uint64_t offset;
+   simple_mtx_lock(&dev->vma_mutex);
+   offset = util_vma_heap_alloc(&dev->vma_heap, size, align);
+   simple_mtx_unlock(&dev->vma_mutex);
+
+   if (dev->debug_flags & NVK_DEBUG_VM)
+      fprintf(stderr, "alloc vma %" PRIx64 " %" PRIx64 " sparse: %d\n",
+              offset, size, sparse_resident);
+
+   if (sparse_resident)
+      bo_bind(dev, 0, offset, size, 0, DRM_NOUVEAU_VM_BIND_SPARSE);
+
+   return offset;
+}
+
+void
+nouveau_ws_free_vma(struct nouveau_ws_device *dev,
+                    uint64_t offset, uint64_t size,
+                    bool sparse_resident)
+{
+   if (dev->debug_flags & NVK_DEBUG_VM)
+      fprintf(stderr, "free vma %" PRIx64 " %" PRIx64 "\n",
+              offset, size);
+
+   if (sparse_resident)
+      bo_unbind(dev, offset, size, DRM_NOUVEAU_VM_BIND_SPARSE);
+
+   simple_mtx_lock(&dev->vma_mutex);
+   util_vma_heap_free(&dev->vma_heap, offset, size);
+   simple_mtx_unlock(&dev->vma_mutex);
+}
+
+void
+nouveau_ws_bo_unbind_vma(struct nouveau_ws_device *dev,
+                         uint64_t offset, uint64_t range)
+{
+   if (dev->debug_flags & NVK_DEBUG_VM)
+      fprintf(stderr, "unbind vma %" PRIx64 " %" PRIx64 "\n",
+              offset, range);
+   bo_unbind(dev, offset, range, 0);
+}
+
+void
+nouveau_ws_bo_bind_vma(struct nouveau_ws_device *dev,
+                       struct nouveau_ws_bo *bo,
+                       uint64_t addr,
+                       uint64_t range,
+                       uint64_t bo_offset,
+                       uint32_t pte_kind)
+{
+   if (dev->debug_flags & NVK_DEBUG_VM)
+      fprintf(stderr, "bind vma %x %" PRIx64 " %" PRIx64 " %" PRIx64 " %d\n",
+              bo->handle, addr, range, bo_offset, pte_kind);
+   bo_bind(dev, bo->handle, addr, range, bo_offset, pte_kind);
+}
+#endif
+
 struct nouveau_ws_bo *
 nouveau_ws_bo_new(struct nouveau_ws_device *dev,
                   uint64_t size, uint64_t align,
@@ -51,7 +161,15 @@ nouveau_ws_bo_new_tiled(struct nouveau_ws_device *dev,
    if (align == 0)
       align = 0x1000;
 
+   /* Align the size */
+   size = ALIGN(size, align);
+
+#if NVK_NEW_UAPI == 0
    req.info.domain = NOUVEAU_GEM_TILE_NONCONTIG;
+#else
+   req.info.domain = 0;
+#endif
+
    if (flags & NOUVEAU_WS_BO_GART)
       req.info.domain |= NOUVEAU_GEM_DOMAIN_GART;
    else
@@ -60,10 +178,12 @@ nouveau_ws_bo_new_tiled(struct nouveau_ws_device *dev,
    if (flags & NOUVEAU_WS_BO_MAP)
       req.info.domain |= NOUVEAU_GEM_DOMAIN_MAPPABLE;
 
+#if NVK_NEW_UAPI == 0
    assert(pte_kind == 0 || !(flags & NOUVEAU_WS_BO_GART));
    assert(tile_mode == 0 || !(flags & NOUVEAU_WS_BO_GART));
    req.info.tile_flags = (uint32_t)pte_kind << 8;
    req.info.tile_mode = tile_mode;
+#endif
 
    req.info.size = size;
    req.align = align;
@@ -72,14 +192,25 @@ nouveau_ws_bo_new_tiled(struct nouveau_ws_device *dev,
 
    int ret = drmCommandWriteRead(dev->fd, DRM_NOUVEAU_GEM_NEW, &req, sizeof(req));
    if (ret == 0) {
-      bo->size = req.info.size;
+      bo->size = size;
+      bo->align = align;
+#if NVK_NEW_UAPI == 0
       bo->offset = req.info.offset;
+#else
+      bo->offset = -1ULL;
+#endif
       bo->handle = req.info.handle;
       bo->map_handle = req.info.map_handle;
       bo->dev = dev;
       bo->flags = flags;
       bo->refcnt = 1;
 
+#if NVK_NEW_UAPI == 1
+      assert(pte_kind == 0);
+      bo->offset = nouveau_ws_alloc_vma(dev, bo->size, align, false);
+      nouveau_ws_bo_bind_vma(dev, bo, bo->offset, bo->size, 0, 0);
+#endif
+
       _mesa_hash_table_insert(dev->bos, (void *)(uintptr_t)bo->handle, bo);
    } else {
       FREE(bo);
@@ -127,6 +258,16 @@ nouveau_ws_bo_from_dma_buf(struct nouveau_ws_device *dev, int fd)
             bo->flags = flags;
             bo->refcnt = 1;
 
+#if NVK_NEW_UAPI == 1
+            uint64_t align = (1ULL << 12);
+            if (info.domain & NOUVEAU_GEM_DOMAIN_VRAM)
+               align = (1ULL << 16);
+
+            assert(bo->size == ALIGN(bo->size, align));
+
+            bo->offset = nouveau_ws_alloc_vma(dev, bo->size, align, false);
+            nouveau_ws_bo_bind_vma(dev, bo, bo->offset, bo->size, 0, 0);
+#endif
             _mesa_hash_table_insert(dev->bos, (void *)(uintptr_t)handle, bo);
          }
       }
@@ -148,6 +289,12 @@ nouveau_ws_bo_destroy(struct nouveau_ws_bo *bo)
    simple_mtx_lock(&dev->bos_lock);
 
    _mesa_hash_table_remove_key(dev->bos, (void *)(uintptr_t)bo->handle);
+
+#if NVK_NEW_UAPI == 1
+   nouveau_ws_bo_unbind_vma(bo->dev, bo->offset, bo->size);
+   nouveau_ws_free_vma(bo->dev, bo->offset, bo->size, false);
+#endif
+
    drmCloseBufferHandle(bo->dev->fd, bo->handle);
    FREE(bo);
 

src/nouveau/winsys/nouveau_bo.h

@@ -34,6 +34,7 @@ enum nouveau_ws_bo_map_flags {
 struct nouveau_ws_bo {
    uint64_t size;
    uint64_t offset;
+   uint64_t align;
    uint64_t map_handle;
    struct nouveau_ws_device *dev;
    uint32_t handle;
@@ -41,6 +42,23 @@ struct nouveau_ws_bo {
    atomic_uint_fast32_t refcnt;
 };
 
+#if NVK_NEW_UAPI == 1
+uint64_t nouveau_ws_alloc_vma(struct nouveau_ws_device *dev,
+                              uint64_t size, uint64_t align, bool sparse);
+
+void nouveau_ws_free_vma(struct nouveau_ws_device *dev,
+                         uint64_t offset, uint64_t size, bool sparse);
+
+void nouveau_ws_bo_bind_vma(struct nouveau_ws_device *dev,
+                            struct nouveau_ws_bo *bo,
+                            uint64_t addr,
+                            uint64_t range,
+                            uint64_t bo_offset,
+                            uint32_t pte_kind);
+void nouveau_ws_bo_unbind_vma(struct nouveau_ws_device *dev,
+                              uint64_t offset, uint64_t range);
+#endif
+
 struct nouveau_ws_bo *nouveau_ws_bo_new(struct nouveau_ws_device *,
                                         uint64_t size, uint64_t align,
                                         enum nouveau_ws_bo_flags);

src/nouveau/winsys/nouveau_device.c

@@ -130,6 +130,7 @@ nouveau_ws_device_set_dbg_flags(struct nouveau_ws_device *dev)
       { "push_dump", NVK_DEBUG_PUSH_DUMP },
       { "push_sync", NVK_DEBUG_PUSH_SYNC },
       { "zero_memory", NVK_DEBUG_ZERO_MEMORY },
+      { "vm", NVK_DEBUG_VM },
       { NULL, 0 },
    };
 
@@ -258,6 +259,14 @@ nouveau_ws_device_new(drmDevicePtr drm_device)
       goto out_err;
    }
 
+#if NVK_NEW_UAPI == 1
+   const uint64_t TOP = 1ull << 40;
+   const uint64_t KERN = 1ull << 39;
+   util_vma_heap_init(&device->vma_heap, 4096, (TOP - KERN) - 4096);
+   simple_mtx_init(&device->vma_mutex, mtx_plain);
+   device->vma_heap.alloc_high = false;
+#endif
+
    uint32_t version =
       ver->version_major << 24 |
       ver->version_minor << 8  |
@@ -265,8 +274,21 @@ nouveau_ws_device_new(drmDevicePtr drm_device)
    drmFreeVersion(ver);
    ver = NULL;
 
+#if NVK_NEW_UAPI == 1
+   /* don't work on older kernels */
+   if (version < 0x01000400)
+      goto out_err;
+#else
    if (version < 0x01000301)
       goto out_err;
+#endif
+
+#if NVK_NEW_UAPI == 1
+   /* start the new VM mode */
+   struct drm_nouveau_vm_init vminit = { TOP-KERN, KERN };
+   ASSERTED int ret = drmCommandWrite(fd, DRM_NOUVEAU_VM_INIT, &vminit, sizeof(vminit));
+   assert(!ret);
+#endif
 
    if (nouveau_ws_device_alloc(fd, device))
       goto out_err;
@@ -345,6 +367,13 @@ nouveau_ws_device_destroy(struct nouveau_ws_device *device)
 
    _mesa_hash_table_destroy(device->bos, NULL);
    simple_mtx_destroy(&device->bos_lock);
+
+#if NVK_NEW_UAPI == 1
+   util_vma_heap_finish(&device->vma_heap);
+   simple_mtx_destroy(&device->vma_mutex);
+#endif
+
+
    close(device->fd);
    FREE(device);
 }

src/nouveau/winsys/nouveau_device.h

@@ -4,6 +4,7 @@
 #include "nouveau_private.h"
 #include "nv_device_info.h"
 #include "util/simple_mtx.h"
+#include "util/vma.h"
 
 #include <stddef.h>
 
@@ -14,6 +15,8 @@ struct hash_table;
 extern "C" {
 #endif
 
+#define NVK_NEW_UAPI 0
+
 enum nvk_debug {
    /* dumps all push buffers after submission */
    NVK_DEBUG_PUSH_DUMP = 1ull << 0,
@@ -28,6 +31,10 @@ enum nvk_debug {
    /* Zero all client memory allocations
     */
    NVK_DEBUG_ZERO_MEMORY = 1ull << 2,
+
+   /* Dump VM bind/unbinds
+    */
+   NVK_DEBUG_VM = 1ull << 3,
 };
 
 struct nouveau_ws_device {
@@ -41,6 +48,11 @@ struct nouveau_ws_device {
 
    simple_mtx_t bos_lock;
    struct hash_table *bos;
+
+#if NVK_NEW_UAPI == 1
+   struct util_vma_heap vma_heap;
+   simple_mtx_t vma_mutex;
+#endif
 };
 
 struct nouveau_ws_device *nouveau_ws_device_new(struct _drmDevice *drm_device);