diff --git a/src/panfrost/vulkan/meson.build b/src/panfrost/vulkan/meson.build index 4dbc997ca56..6f014d95b11 100644 --- a/src/panfrost/vulkan/meson.build +++ b/src/panfrost/vulkan/meson.build @@ -38,6 +38,7 @@ panvk_tracepoints = custom_target( libpanvk_files = files( 'panvk_buffer.c', 'panvk_cmd_pool.c', + 'panvk_device.c', 'panvk_device_memory.c', 'panvk_host_copy.c', 'panvk_image.c', diff --git a/src/panfrost/vulkan/panvk_device.c b/src/panfrost/vulkan/panvk_device.c new file mode 100644 index 00000000000..65cddcc913e --- /dev/null +++ b/src/panfrost/vulkan/panvk_device.c @@ -0,0 +1,131 @@ +#include "panvk_device.h" + +#include "drm-uapi/panthor_drm.h" + +static uint64_t +panvk_choose_blackhole_size(const struct pan_kmod_vm *vm, uint64_t max_size) +{ + uint64_t blackhole_size = 0; + u_foreach_bit64(pgsize_bit, vm->pgsize_bitmap) { + uint64_t pgsize = (uint64_t)1 << pgsize_bit; + if (blackhole_size > 0 && pgsize > max_size) + break; + blackhole_size = pgsize; + } + + return blackhole_size; +} + +static void +panvk_blackhole_init(const void *_dev) +{ + struct panvk_device *dev = *(void *const *)_dev; + + uint64_t blackhole_size = panvk_choose_blackhole_size(dev->kmod.vm, 0x200000); + + dev->sparse_mem.blackhole = + pan_kmod_bo_alloc(dev->kmod.dev, dev->kmod.vm, blackhole_size, + PAN_KMOD_BO_FLAG_NO_MMAP); +} + +struct pan_kmod_bo * +panvk_get_blackhole(struct panvk_device *dev) +{ + util_call_once_data(&dev->sparse_mem.blackhole_once, panvk_blackhole_init, &dev); + return dev->sparse_mem.blackhole; +} + +struct panvk_vm_binder { + uint32_t drm_fd; + uint32_t vm_id; + + struct drm_panthor_vm_bind_op *ops; + size_t op_count; + size_t op_cap; +}; + +static int +panvk_vm_binder_flush(struct panvk_vm_binder *b) +{ + if (b->op_count == 0) + return 0; + + struct drm_panthor_vm_bind req = { + .vm_id = b->vm_id, + .ops = DRM_PANTHOR_OBJ_ARRAY(b->op_count, b->ops), + }; + int ret = pan_kmod_ioctl(b->drm_fd, DRM_IOCTL_PANTHOR_VM_BIND, &req); + b->op_count = 0; + return ret; +} + +static int +panvk_vm_binder_bind(struct panvk_vm_binder *b, const struct drm_panthor_vm_bind_op *op) +{ + if (b->op_count == b->op_cap) { + int ret = panvk_vm_binder_flush(b); + if (ret) + return ret; + } + assert(b->op_count < b->op_cap); + b->ops[b->op_count++] = *op; + return 0; +} + +VkResult +panvk_map_to_blackhole(struct panvk_device *device, uint64_t address, uint64_t size) +{ + struct pan_kmod_bo *blackhole = panvk_get_blackhole(device); + uint64_t blackhole_size = blackhole->size; + + struct drm_panthor_vm_bind_op ops[16] = {}; + struct panvk_vm_binder binder = { + .drm_fd = device->drm_fd, + .vm_id = device->kmod.vm->handle, + + .ops = ops, + .op_cap = ARRAY_SIZE(ops), + }; + + uint64_t off = 0; + while (off < size) { + uint64_t va = address + off; + uint64_t bo_offset = va & (blackhole_size - 1); + uint64_t range = MIN2(blackhole_size - bo_offset, size - off); + struct drm_panthor_vm_bind_op op = { + .flags = DRM_PANTHOR_VM_BIND_OP_TYPE_MAP, + .bo_handle = blackhole->handle, + .bo_offset = bo_offset, + .va = va, + .size = range, + }; + int ret = panvk_vm_binder_bind(&binder, &op); + if (ret) + goto err_unmap; + + off += range; + } + assert(off == size); + + int ret = panvk_vm_binder_flush(&binder); + if (ret) + goto err_unmap; + + return VK_SUCCESS; + +err_unmap: + { + struct pan_kmod_vm_op unmap = { + .type = PAN_KMOD_VM_OP_TYPE_UNMAP, + .va = { + .start = address, + .size = size, + }, + }; + ASSERTED int ret = + pan_kmod_vm_bind(device->kmod.vm, PAN_KMOD_VM_OP_TYPE_UNMAP, &unmap, 1); + assert(!ret); + } + + return panvk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); +} diff --git a/src/panfrost/vulkan/panvk_device.h b/src/panfrost/vulkan/panvk_device.h index 1eca2154c8b..abe3c699e5b 100644 --- a/src/panfrost/vulkan/panvk_device.h +++ b/src/panfrost/vulkan/panvk_device.h @@ -24,6 +24,7 @@ #include "util/perf/u_trace.h" #include "util/simple_mtx.h" +#include "util/u_call_once.h" #include "util/u_printf.h" #include "util/vma.h" @@ -83,6 +84,11 @@ struct panvk_device { struct panvk_pool exec; } mempools; + struct { + util_once_flag blackhole_once; + struct pan_kmod_bo *blackhole; + } sparse_mem; + /* For each subqueue, maximum size of the register dump region needed by * exception handlers or functions */ uint32_t *dump_region_size; @@ -173,6 +179,11 @@ panvk_as_free(struct panvk_device *device, uint64_t address, uint64_t size) simple_mtx_unlock(&device->as.lock); } +VkResult panvk_map_to_blackhole(struct panvk_device *device, + uint64_t address, uint64_t size); + +struct pan_kmod_bo *panvk_get_blackhole(struct panvk_device *device); + #if PAN_ARCH VkResult panvk_per_arch(create_device)(struct panvk_physical_device *physical_device, diff --git a/src/panfrost/vulkan/panvk_vX_device.c b/src/panfrost/vulkan/panvk_vX_device.c index 5d3637ba446..22878c5e52d 100644 --- a/src/panfrost/vulkan/panvk_vX_device.c +++ b/src/panfrost/vulkan/panvk_vX_device.c @@ -601,6 +601,7 @@ panvk_per_arch(destroy_device)(struct panvk_device *device, panvk_per_arch(device_draw_context_cleanup)(device); #endif panvk_meta_cleanup(device); + pan_kmod_bo_put(device->sparse_mem.blackhole); u_printf_destroy(&device->printf.ctx); panvk_priv_bo_unref(device->printf.bo); panvk_priv_bo_unref(device->tiler_oom.handlers_bo);