diff --git a/src/intel/vulkan/anv_allocator.c b/src/intel/vulkan/anv_allocator.c index 87909c83735..5e40445d2ca 100644 --- a/src/intel/vulkan/anv_allocator.c +++ b/src/intel/vulkan/anv_allocator.c @@ -1355,7 +1355,7 @@ static void anv_bo_unmap_close(struct anv_device *device, struct anv_bo *bo) { if (bo->map && !bo->from_host_ptr) - anv_device_unmap_bo(device, bo, bo->map, bo->size); + anv_device_unmap_bo(device, bo, bo->map, bo->size, false /* replace */); assert(bo->gem_handle != 0); device->kmd_backend->gem_close(device, bo); @@ -1546,7 +1546,8 @@ anv_device_alloc_bo(struct anv_device *device, }; if (alloc_flags & ANV_BO_ALLOC_MAPPED) { - VkResult result = anv_device_map_bo(device, &new_bo, 0, size, &new_bo.map); + VkResult result = anv_device_map_bo(device, &new_bo, 0, size, + NULL, &new_bo.map); if (unlikely(result != VK_SUCCESS)) { device->kmd_backend->gem_close(device, &new_bo); return result; @@ -1585,15 +1586,18 @@ anv_device_map_bo(struct anv_device *device, struct anv_bo *bo, uint64_t offset, size_t size, + void *placed_addr, void **map_out) { assert(!bo->from_host_ptr); assert(size > 0); - void *map = device->kmd_backend->gem_mmap(device, bo, offset, size); + void *map = device->kmd_backend->gem_mmap(device, bo, offset, size, placed_addr); if (unlikely(map == MAP_FAILED)) return vk_errorf(device, VK_ERROR_MEMORY_MAP_FAILED, "mmap failed: %m"); + assert(placed_addr == NULL || map == placed_addr); + assert(map != NULL); VG(VALGRIND_MALLOCLIKE_BLOCK(map, size, 0, 1)); @@ -1603,15 +1607,26 @@ anv_device_map_bo(struct anv_device *device, return VK_SUCCESS; } -void +VkResult anv_device_unmap_bo(struct anv_device *device, struct anv_bo *bo, - void *map, size_t map_size) + void *map, size_t map_size, + bool replace) { assert(!bo->from_host_ptr); - VG(VALGRIND_FREELIKE_BLOCK(map, 0)); - munmap(map, map_size); + if (replace) { + map = mmap(map, map_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (map == MAP_FAILED) { + return vk_errorf(device, VK_ERROR_MEMORY_MAP_FAILED, + "Failed to map over original mapping"); + } + } else { + VG(VALGRIND_FREELIKE_BLOCK(map, 0)); + munmap(map, map_size); + } + return VK_SUCCESS; } VkResult diff --git a/src/intel/vulkan/anv_device.c b/src/intel/vulkan/anv_device.c index fdbee1b2016..bb5c87ec850 100644 --- a/src/intel/vulkan/anv_device.c +++ b/src/intel/vulkan/anv_device.c @@ -367,6 +367,7 @@ get_device_extensions(const struct anv_physical_device *device, .EXT_inline_uniform_block = true, .EXT_line_rasterization = true, .EXT_load_store_op_none = true, + .EXT_map_memory_placed = device->info.has_mmap_offset, /* Enable the extension only if we have support on both the local & * system memory */ @@ -901,6 +902,11 @@ get_features(const struct anv_physical_device *pdevice, .descriptorBufferCaptureReplay = true, .descriptorBufferImageLayoutIgnored = false, .descriptorBufferPushDescriptors = true, + + /* VK_EXT_map_memory_placed */ + .memoryMapPlaced = true, + .memoryMapRangePlaced = false, + .memoryUnmapReserve = true, }; /* The new DOOM and Wolfenstein games require depthBounds without @@ -1582,6 +1588,11 @@ get_properties(const struct anv_physical_device *pdevice, props->lineSubPixelPrecisionBits = 4; } + /* VK_EXT_map_memory_placed */ + { + props->minPlacedMemoryMapAlignment = 4096; + } + /* VK_EXT_mesh_shader */ { /* Bounded by the maximum representable size in @@ -4653,6 +4664,14 @@ VkResult anv_MapMemory2KHR( "Memory object already mapped."); } + void *placed_addr = NULL; + if (pMemoryMapInfo->flags & VK_MEMORY_MAP_PLACED_BIT_EXT) { + const VkMemoryMapPlacedInfoEXT *placed_info = + vk_find_struct_const(pMemoryMapInfo->pNext, MEMORY_MAP_PLACED_INFO_EXT); + assert(placed_info != NULL); + placed_addr = placed_info->pPlacedAddress; + } + /* GEM will fail to map if the offset isn't 4k-aligned. Round down. */ uint64_t map_offset; if (!device->physical->info.has_mmap_offset) @@ -4666,7 +4685,8 @@ VkResult anv_MapMemory2KHR( map_size = align64(map_size, 4096); void *map; - VkResult result = anv_device_map_bo(device, mem->bo, map_offset, map_size, &map); + VkResult result = anv_device_map_bo(device, mem->bo, map_offset, + map_size, placed_addr, &map); if (result != VK_SUCCESS) return result; @@ -4688,7 +4708,11 @@ VkResult anv_UnmapMemory2KHR( if (mem == NULL || mem->vk.host_ptr) return VK_SUCCESS; - anv_device_unmap_bo(device, mem->bo, mem->map, mem->map_size); + VkResult result = + anv_device_unmap_bo(device, mem->bo, mem->map, mem->map_size, + pMemoryUnmapInfo->flags & VK_MEMORY_UNMAP_RESERVE_BIT_EXT); + if (result != VK_SUCCESS) + return result; mem->map = NULL; mem->map_size = 0; diff --git a/src/intel/vulkan/anv_gem_stubs.c b/src/intel/vulkan/anv_gem_stubs.c index 36d0076dad1..a62f06f58c9 100644 --- a/src/intel/vulkan/anv_gem_stubs.c +++ b/src/intel/vulkan/anv_gem_stubs.c @@ -52,7 +52,7 @@ stub_gem_create(struct anv_device *device, static void * stub_gem_mmap(struct anv_device *device, struct anv_bo *bo, uint64_t offset, - uint64_t size) + uint64_t size, void *placed_addr) { return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, bo->gem_handle, offset); diff --git a/src/intel/vulkan/anv_kmd_backend.h b/src/intel/vulkan/anv_kmd_backend.h index a8a205e1d63..595a3f8fc57 100644 --- a/src/intel/vulkan/anv_kmd_backend.h +++ b/src/intel/vulkan/anv_kmd_backend.h @@ -72,7 +72,7 @@ struct anv_kmd_backend { void (*gem_close)(struct anv_device *device, struct anv_bo *bo); /* Returns MAP_FAILED on error */ void *(*gem_mmap)(struct anv_device *device, struct anv_bo *bo, - uint64_t offset, uint64_t size); + uint64_t offset, uint64_t size, void *placed_addr); /* Bind things however you want. */ int (*vm_bind)(struct anv_device *device, struct anv_sparse_submission *submit); diff --git a/src/intel/vulkan/anv_private.h b/src/intel/vulkan/anv_private.h index ac05ccd819b..dc08bb53e26 100644 --- a/src/intel/vulkan/anv_private.h +++ b/src/intel/vulkan/anv_private.h @@ -2013,10 +2013,12 @@ VkResult anv_device_map_bo(struct anv_device *device, struct anv_bo *bo, uint64_t offset, size_t size, + void *placed_addr, void **map_out); -void anv_device_unmap_bo(struct anv_device *device, - struct anv_bo *bo, - void *map, size_t map_size); +VkResult anv_device_unmap_bo(struct anv_device *device, + struct anv_bo *bo, + void *map, size_t map_size, + bool replace); VkResult anv_device_import_bo_from_host_ptr(struct anv_device *device, void *host_ptr, uint32_t size, enum anv_bo_alloc_flags alloc_flags, diff --git a/src/intel/vulkan/genX_cmd_video.c b/src/intel/vulkan/genX_cmd_video.c index 44df84e35b5..def873aefa7 100644 --- a/src/intel/vulkan/genX_cmd_video.c +++ b/src/intel/vulkan/genX_cmd_video.c @@ -603,6 +603,7 @@ anv_h265_decode_video(struct anv_cmd_buffer *cmd_buffer, src_buffer->address.bo, src_buffer->address.offset, frame_info->srcBufferRange, + NULL /* placed_addr */, &slice_map); if (result != VK_SUCCESS) { anv_batch_set_error(&cmd_buffer->batch, result); @@ -626,7 +627,9 @@ anv_h265_decode_video(struct anv_cmd_buffer *cmd_buffer, vk_fill_video_h265_reference_info(frame_info, h265_pic_info, &slice_params[s], ref_slots); } - anv_device_unmap_bo(cmd_buffer->device, src_buffer->address.bo, slice_map, frame_info->srcBufferRange); + anv_device_unmap_bo(cmd_buffer->device, src_buffer->address.bo, + slice_map, frame_info->srcBufferRange, + false /* replace */); for (unsigned s = 0; s < h265_pic_info->sliceSegmentCount; s++) { uint32_t ctb_size = 1 << (sps->log2_diff_max_min_luma_coding_block_size + diff --git a/src/intel/vulkan/i915/anv_kmd_backend.c b/src/intel/vulkan/i915/anv_kmd_backend.c index 442ccb3f350..ec59da05212 100644 --- a/src/intel/vulkan/i915/anv_kmd_backend.c +++ b/src/intel/vulkan/i915/anv_kmd_backend.c @@ -160,7 +160,8 @@ i915_gem_close(struct anv_device *device, struct anv_bo *bo) static void * i915_gem_mmap_offset(struct anv_device *device, struct anv_bo *bo, - uint64_t size, uint32_t flags) + uint64_t size, uint32_t flags, + void *placed_addr) { struct drm_i915_gem_mmap_offset gem_mmap = { .handle = bo->gem_handle, @@ -169,7 +170,8 @@ i915_gem_mmap_offset(struct anv_device *device, struct anv_bo *bo, if (intel_ioctl(device->fd, DRM_IOCTL_I915_GEM_MMAP_OFFSET, &gem_mmap)) return MAP_FAILED; - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + return mmap(placed_addr, size, PROT_READ | PROT_WRITE, + (placed_addr != NULL ? MAP_FIXED : 0) | MAP_SHARED, device->fd, gem_mmap.offset); } @@ -214,12 +216,13 @@ mmap_calc_flags(struct anv_device *device, struct anv_bo *bo) static void * i915_gem_mmap(struct anv_device *device, struct anv_bo *bo, uint64_t offset, - uint64_t size) + uint64_t size, void *placed_addr) { const uint32_t flags = mmap_calc_flags(device, bo); if (likely(device->physical->info.has_mmap_offset)) - return i915_gem_mmap_offset(device, bo, size, flags); + return i915_gem_mmap_offset(device, bo, size, flags, placed_addr); + assert(placed_addr == NULL); return i915_gem_mmap_legacy(device, bo, offset, size, flags); } diff --git a/src/intel/vulkan/xe/anv_kmd_backend.c b/src/intel/vulkan/xe/anv_kmd_backend.c index 4eca33b06fe..031f571bf6e 100644 --- a/src/intel/vulkan/xe/anv_kmd_backend.c +++ b/src/intel/vulkan/xe/anv_kmd_backend.c @@ -99,7 +99,7 @@ xe_gem_close(struct anv_device *device, struct anv_bo *bo) static void * xe_gem_mmap(struct anv_device *device, struct anv_bo *bo, uint64_t offset, - uint64_t size) + uint64_t size, void *placed_addr) { struct drm_xe_gem_mmap_offset args = { .handle = bo->gem_handle, @@ -107,7 +107,8 @@ xe_gem_mmap(struct anv_device *device, struct anv_bo *bo, uint64_t offset, if (intel_ioctl(device->fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &args)) return MAP_FAILED; - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + return mmap(placed_addr, size, PROT_READ | PROT_WRITE, + (placed_addr != NULL ? MAP_FIXED : 0) | MAP_SHARED, device->fd, args.offset); }