zink: add a suballocator

this is an aux/pipebuffer implementation borrowing heavily from the
one in radeonsi. it currently has the following limitations, which
will be resolved in a followup series:
* 32bit address space still explodes
* swapchain images still have separate memory handling

performance in games like Tomb Raider has been observed to increase by
over 1000%

SQUASHED: simplify get_memory_type_index()

now that the heaps are enumerated, this can be reduced to a simple
array index with a fallback

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12146>

src/gallium/drivers/zink/meson.build

@@ -24,6 +24,7 @@ files_libzink = files(
   'nir_to_spirv/spirv_builder.c',
   'zink_batch.c',
   'zink_blit.c',
+  'zink_bo.c',
   'zink_clear.c',
   'zink_compiler.c',
   'zink_context.c',

src/gallium/drivers/zink/zink_bo.h

@@ -0,0 +1,270 @@
+/*
+ * Copyright © 2021 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ * 
+ * Authors:
+ *    Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
+ */
+
+#ifndef ZINK_BO_H
+#define ZINK_BO_H
+#include <vulkan/vulkan.h>
+#include "pipebuffer/pb_cache.h"
+#include "pipebuffer/pb_slab.h"
+#include "zink_batch.h"
+
+#define VK_VIS_VRAM (VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
+enum zink_resource_access {
+   ZINK_RESOURCE_ACCESS_READ = 1,
+   ZINK_RESOURCE_ACCESS_WRITE = 32,
+   ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
+};
+
+
+enum zink_heap {
+   ZINK_HEAP_DEVICE_LOCAL,
+   ZINK_HEAP_DEVICE_LOCAL_SPARSE,
+   ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
+   ZINK_HEAP_HOST_VISIBLE_ANY,
+   ZINK_HEAP_HOST_VISIBLE_COHERENT,
+   ZINK_HEAP_HOST_VISIBLE_CACHED,
+   ZINK_HEAP_MAX,
+};
+
+enum zink_alloc_flag {
+   ZINK_ALLOC_SPARSE = 1<<0,
+   ZINK_ALLOC_NO_SUBALLOC = 1<<1,
+};
+
+
+struct zink_bo {
+   struct pb_buffer base;
+
+   union {
+      struct {
+         void *cpu_ptr; /* for user_ptr and permanent maps */
+         int map_count;
+
+         bool is_user_ptr;
+         bool use_reusable_pool;
+
+         /* Whether buffer_get_handle or buffer_from_handle has been called,
+          * it can only transition from false to true. Protected by lock.
+          */
+         bool is_shared;
+      } real;
+      struct {
+         struct pb_slab_entry entry;
+         struct zink_bo *real;
+      } slab;
+      struct {
+         uint32_t num_va_pages;
+         uint32_t num_backing_pages;
+
+         struct list_head backing;
+
+         /* Commitment information for each page of the virtual memory area. */
+         struct zink_sparse_commitment *commitments;
+      } sparse;
+   } u;
+
+   VkDeviceMemory mem;
+   uint64_t offset;
+
+   uint32_t unique_id;
+
+   simple_mtx_t lock;
+
+   struct zink_batch_usage *reads;
+   struct zink_batch_usage *writes;
+
+   struct pb_cache_entry cache_entry[];
+};
+
+static inline struct zink_bo *
+zink_bo(struct pb_buffer *pbuf)
+{
+   return (struct zink_bo*)pbuf;
+}
+
+static inline enum zink_alloc_flag
+zink_alloc_flags_from_heap(enum zink_heap heap)
+{
+   enum zink_alloc_flag flags = 0;
+   switch (heap) {
+   case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
+      flags |= ZINK_ALLOC_SPARSE;
+      break;
+   default:
+      break;
+   }
+   return flags;
+}
+
+static inline VkMemoryPropertyFlags
+vk_domain_from_heap(enum zink_heap heap)
+{
+   VkMemoryPropertyFlags domains = 0;
+
+   switch (heap) {
+   case ZINK_HEAP_DEVICE_LOCAL:
+   case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
+      domains = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+      break;
+   case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+      break;
+   case ZINK_HEAP_HOST_VISIBLE_ANY:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
+      break;
+   case ZINK_HEAP_HOST_VISIBLE_COHERENT:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
+      break;
+   case ZINK_HEAP_HOST_VISIBLE_CACHED:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
+      break;
+   default:
+      break;
+   }
+   return domains;
+}
+
+static inline enum zink_heap
+zink_heap_from_domain_flags(VkMemoryPropertyFlags domains, enum zink_alloc_flag flags)
+{
+   if (flags & ZINK_ALLOC_SPARSE)
+      return ZINK_HEAP_DEVICE_LOCAL_SPARSE;
+
+   if ((domains & VK_VIS_VRAM) == VK_VIS_VRAM)
+      return ZINK_HEAP_DEVICE_LOCAL_VISIBLE;
+
+   if (domains & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
+      return ZINK_HEAP_DEVICE_LOCAL;
+
+   if (domains & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)
+      return ZINK_HEAP_HOST_VISIBLE_COHERENT;
+
+   if (domains & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
+      return ZINK_HEAP_HOST_VISIBLE_CACHED;
+
+   return ZINK_HEAP_HOST_VISIBLE_ANY;
+}
+
+bool
+zink_bo_init(struct zink_screen *screen);
+
+void
+zink_bo_deinit(struct zink_screen *screen);
+
+struct pb_buffer *
+zink_bo_create(struct zink_screen *screen, uint64_t size, unsigned alignment, enum zink_heap heap, enum zink_alloc_flag flags);
+
+static inline uint64_t
+zink_bo_get_offset(const struct zink_bo *bo)
+{
+   return bo->offset;
+}
+
+static inline VkDeviceMemory
+zink_bo_get_mem(const struct zink_bo *bo)
+{
+   return bo->mem ? bo->mem : bo->u.slab.real->mem;
+}
+
+static inline VkDeviceSize
+zink_bo_get_size(const struct zink_bo *bo)
+{
+   return bo->mem ? bo->base.size : bo->u.slab.real->base.size;
+}
+
+void *
+zink_bo_map(struct zink_screen *screen, struct zink_bo *bo);
+void
+zink_bo_unmap(struct zink_screen *screen, struct zink_bo *bo);
+
+bool
+zink_bo_commit(struct zink_screen *screen, struct zink_resource *res, uint32_t offset, uint32_t size, bool commit);
+
+static inline bool
+zink_bo_has_unflushed_usage(const struct zink_bo *bo)
+{
+   return zink_batch_usage_is_unflushed(bo->reads) ||
+          zink_batch_usage_is_unflushed(bo->writes);
+}
+
+static inline bool
+zink_bo_has_usage(const struct zink_bo *bo)
+{
+   return zink_batch_usage_exists(bo->reads) ||
+          zink_batch_usage_exists(bo->writes);
+}
+
+static inline bool
+zink_bo_usage_matches(const struct zink_bo *bo, const struct zink_batch_state *bs)
+{
+   return zink_batch_usage_matches(bo->reads, bs) ||
+          zink_batch_usage_matches(bo->writes, bs);
+}
+
+static inline bool
+zink_bo_usage_check_completion(struct zink_screen *screen, struct zink_bo *bo, enum zink_resource_access access)
+{
+   if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, bo->reads))
+      return false;
+   if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, bo->writes))
+      return false;
+   return true;
+}
+
+static inline void
+zink_bo_usage_wait(struct zink_context *ctx, struct zink_bo *bo, enum zink_resource_access access)
+{
+   if (access & ZINK_RESOURCE_ACCESS_READ)
+      zink_batch_usage_wait(ctx, bo->reads);
+   if (access & ZINK_RESOURCE_ACCESS_WRITE)
+      zink_batch_usage_wait(ctx, bo->writes);
+}
+
+static inline void
+zink_bo_usage_set(struct zink_bo *bo, struct zink_batch_state *bs, bool write)
+{
+   if (write)
+      zink_batch_usage_set(&bo->writes, bs);
+   else
+      zink_batch_usage_set(&bo->reads, bs);
+}
+
+static inline void
+zink_bo_usage_unset(struct zink_bo *bo, struct zink_batch_state *bs)
+{
+   zink_batch_usage_unset(&bo->reads, bs);
+   zink_batch_usage_unset(&bo->writes, bs);
+}
+
+
+static inline void
+zink_bo_unref(struct zink_screen *screen, struct zink_bo *bo)
+{
+   struct pb_buffer *pbuf = &bo->base;
+   pb_reference_with_winsys(screen, &pbuf, NULL);
+}
+
+#endif

src/gallium/drivers/zink/zink_context.c

@@ -3201,29 +3201,6 @@ rebind_buffer(struct zink_context *ctx, struct zink_resource *res)
    zink_batch_resource_usage_set(&ctx->batch, res, has_write);
 }
 
-static inline struct zink_screen **
-get_screen_ptr_for_commit(uint8_t *mem)
-{
-   return (struct zink_screen**)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind));
-}
-
-static bool
-resource_commit(struct zink_screen *screen, VkBindSparseInfo *sparse)
-{
-   VkQueue queue = screen->threaded ? screen->thread_queue : screen->queue;
-
-   VkResult ret = vkQueueBindSparse(queue, 1, sparse, VK_NULL_HANDLE);
-   return zink_screen_handle_vkresult(screen, ret);
-}
-
-static void
-submit_resource_commit(void *data, void *gdata, int thread_index)
-{
-   struct zink_screen **screen = get_screen_ptr_for_commit(data);
-   resource_commit(*screen, data);
-   free(data);
-}
-
 static bool
 zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
 {
@@ -3235,51 +3212,12 @@ zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsi
    if (zink_resource_has_unflushed_usage(res))
       zink_flush_queue(ctx);
 
-   uint8_t *mem = malloc(sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind) + sizeof(void*));
+   bool ret = zink_bo_commit(screen, res, box->x, box->width, commit);
-   if (!mem)
-      return false;
-   VkBindSparseInfo *sparse = (void*)mem;
-   sparse->sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
-   sparse->pNext = NULL;
-   sparse->waitSemaphoreCount = 0;
-   sparse->bufferBindCount = 1;
-   sparse->imageOpaqueBindCount = 0;
-   sparse->imageBindCount = 0;
-   sparse->signalSemaphoreCount = 0;
-
-   VkSparseBufferMemoryBindInfo *sparse_bind = (void*)(mem + sizeof(VkBindSparseInfo));
-   sparse_bind->buffer = res->obj->buffer;
-   sparse_bind->bindCount = 1;
-   sparse->pBufferBinds = sparse_bind;
-
-   VkSparseMemoryBind *mem_bind = (void*)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo));
-   mem_bind->resourceOffset = box->x;
-   mem_bind->size = box->width;
-   mem_bind->memory = commit ? res->obj->mem : VK_NULL_HANDLE;
-   /* currently sparse buffers allocate memory 1:1 for the max sparse size,
-    * but probably it should dynamically allocate the committed regions;
-    * if this ever changes, update the below line
-    */
-   mem_bind->memoryOffset = box->x;
-   mem_bind->flags = 0;
-   sparse_bind->pBinds = mem_bind;
-
-   struct zink_screen **ptr = get_screen_ptr_for_commit(mem);
-   *ptr = screen;
-
-   if (screen->threaded) {
-      /* this doesn't need any kind of fencing because any access to this resource
-       * will be automagically synchronized by queue dispatch */
-      util_queue_add_job(&screen->flush_queue, mem, NULL, submit_resource_commit, NULL, 0);
-   } else {
-      bool ret = resource_commit(screen, sparse);
    if (!ret)
       check_device_lost(ctx);
-      free(sparse);
+
    return ret;
 }
-   return true;
-}
 
 static void
 rebind_image(struct zink_context *ctx, struct zink_resource *res)

src/gallium/drivers/zink/zink_resource.c

@@ -78,49 +78,6 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object
    sprintf(buf, "zink_resource_object");
 }
 
-static uint32_t
-mem_hash(const void *key)
-{
-   const struct mem_key *mkey = key;
-   return _mesa_hash_data(&mkey->key, sizeof(mkey->key));
-}
-
-static bool
-mem_equals(const void *a, const void *b)
-{
-   const struct mem_key *ma = a;
-   const struct mem_key *mb = b;
-   return !memcmp(&ma->key, &mb->key, sizeof(ma->key));
-}
-
-static void
-cache_or_free_mem(struct zink_screen *screen, struct zink_resource_object *obj)
-{
-   if (obj->mkey.key.heap_index != UINT32_MAX) {
-      simple_mtx_lock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
-      struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&screen->mem[obj->mkey.key.heap_index].resource_mem_cache, obj->mem_hash, &obj->mkey);
-      assert(he);
-      struct util_dynarray *array = he->data;
-      struct mem_key *mkey = (void*)he->key;
-
-      unsigned seen = mkey->seen_count;
-      mkey->seen_count--;
-      if (util_dynarray_num_elements(array, struct mem_cache_entry) < seen) {
-         struct mem_cache_entry mc = { obj->mem, obj->map };
-         screen->mem[obj->mkey.key.heap_index].mem_cache_size += obj->size;
-         if (sizeof(void*) == 4 && obj->map) {
-            vkUnmapMemory(screen->dev, obj->mem);
-            mc.map = NULL;
-         }
-         util_dynarray_append(array, struct mem_cache_entry, mc);
-         simple_mtx_unlock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
-         return;
-      }
-      simple_mtx_unlock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
-   }
-   vkFreeMemory(screen->dev, obj->mem, NULL);
-}
-
 void
 zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_object *obj)
 {
@@ -134,7 +91,10 @@ zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_ob
 
    util_dynarray_fini(&obj->tmp);
    zink_descriptor_set_refs_clear(&obj->desc_set_refs, obj);
-   cache_or_free_mem(screen, obj);
+   if (obj->dedicated)
+      vkFreeMemory(screen->dev, obj->mem, NULL);
+   else
+      zink_bo_unref(screen, obj->bo);
    FREE(obj);
 }
 
@@ -155,38 +115,6 @@ zink_resource_destroy(struct pipe_screen *pscreen,
    FREE(res);
 }
 
-static uint32_t
-get_memory_type_index(struct zink_screen *screen,
-                      const VkMemoryRequirements *reqs,
-                      VkMemoryPropertyFlags props)
-{
-   int32_t idx = -1;
-   for (uint32_t i = 0u; i < VK_MAX_MEMORY_TYPES; i++) {
-      if (((reqs->memoryTypeBits >> i) & 1) == 1) {
-         if ((screen->info.mem_props.memoryTypes[i].propertyFlags & props) == props) {
-            if (!(props & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) &&
-                screen->info.mem_props.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
-               idx = i;
-            } else
-               return i;
-         }
-      }
-   }
-   if (idx >= 0)
-      return idx;
-
-   if (props & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
-      /* if no suitable cached memory can be found, fall back
-       * to non-cached memory instead.
-       */
-      return get_memory_type_index(screen, reqs,
-         props & ~VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
-   }
-
-   unreachable("Unsupported memory-type");
-   return 0;
-}
-
 static VkImageAspectFlags
 aspect_from_format(enum pipe_format fmt)
 {
@@ -636,17 +564,31 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
       flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
 
    VkMemoryAllocateInfo mai = {0};
+   enum zink_alloc_flag aflags = templ->flags & PIPE_RESOURCE_FLAG_SPARSE ? ZINK_ALLOC_SPARSE : 0;
    mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
    mai.allocationSize = reqs.size;
-   mai.memoryTypeIndex = get_memory_type_index(screen, &reqs, flags);
+   enum zink_heap heap = zink_heap_from_domain_flags(flags, aflags);
+   mai.memoryTypeIndex = screen->heap_map[heap];
+   if (unlikely(!(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex)))) {
+      /* not valid based on reqs; demote to more compatible type */
+      switch (heap) {
+      case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
+         heap = ZINK_HEAP_DEVICE_LOCAL;
+         break;
+      case ZINK_HEAP_HOST_VISIBLE_CACHED:
+         heap = ZINK_HEAP_HOST_VISIBLE_ANY;
+         break;
+      default:
+         break;
+      }
+      mai.memoryTypeIndex = screen->heap_map[heap];
+      assert(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex));
+   }
 
    VkMemoryType mem_type = screen->info.mem_props.memoryTypes[mai.memoryTypeIndex];
    obj->coherent = mem_type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
       obj->host_visible = mem_type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
-   if (templ->target == PIPE_BUFFER && !obj->coherent && obj->host_visible) {
-      mai.allocationSize = reqs.size = align(reqs.size, screen->info.props.limits.nonCoherentAtomSize);
-   }
 
    VkMemoryDedicatedAllocateInfo ded_alloc_info = {
       .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
@@ -695,46 +637,33 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
       mai.pNext = &memory_wsi_info;
    }
 
-   if (!mai.pNext && !(templ->flags & (PIPE_RESOURCE_FLAG_MAP_COHERENT | PIPE_RESOURCE_FLAG_SPARSE))) {
+   if (!mai.pNext) {
-      obj->mkey.key.reqs = reqs;
+      unsigned alignment = MAX2(reqs.alignment, 256);
-      obj->mkey.key.heap_index = mai.memoryTypeIndex;
+      if (templ->usage == PIPE_USAGE_STAGING && obj->is_buffer)
-      obj->mem_hash = mem_hash(&obj->mkey);
+         alignment = MAX2(alignment, screen->info.props.limits.minMemoryMapAlignment);
-      simple_mtx_lock(&screen->mem[mai.memoryTypeIndex].mem_cache_mtx);
+      obj->alignment = alignment;
-
+      obj->bo = zink_bo(zink_bo_create(screen, reqs.size, alignment, heap, 0));
-      struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&screen->mem[mai.memoryTypeIndex].resource_mem_cache, obj->mem_hash, &obj->mkey);
+      if (!obj->bo)
-      struct mem_key *mkey;
+         goto fail2;
-      if (he) {
+      if (aflags == ZINK_ALLOC_SPARSE) {
-         struct util_dynarray *array = he->data;
+         obj->size = templ->width0;
-         mkey = (void*)he->key;
+      } else {
-         if (array && util_dynarray_num_elements(array, struct mem_cache_entry)) {
+         obj->offset = zink_bo_get_offset(obj->bo);
-            struct mem_cache_entry mc = util_dynarray_pop(array, struct mem_cache_entry);
+         obj->mem = zink_bo_get_mem(obj->bo);
-            obj->mem = mc.mem;
+         obj->size = zink_bo_get_size(obj->bo);
-            obj->map = mc.map;
-            screen->mem[mai.memoryTypeIndex].mem_cache_size -= reqs.size;
-            screen->mem[mai.memoryTypeIndex].mem_cache_count--;
       }
    } else {
-         mkey = ralloc(screen, struct mem_key);
+      obj->dedicated = true;
-         memcpy(&mkey->key, &obj->mkey.key, sizeof(obj->mkey.key));
+      obj->offset = 0;
-         mkey->seen_count = 0;
+      obj->size = reqs.size;
-         struct util_dynarray *array = rzalloc(screen, struct util_dynarray);
-         util_dynarray_init(array, screen);
-         _mesa_hash_table_insert_pre_hashed(&screen->mem[mai.memoryTypeIndex].resource_mem_cache, obj->mem_hash, mkey, array);
    }
-      mkey->seen_count++;
-      simple_mtx_unlock(&screen->mem[mai.memoryTypeIndex].mem_cache_mtx);
-   } else
-      obj->mkey.key.heap_index = UINT32_MAX;
 
    /* TODO: sparse buffers should probably allocate multiple regions of memory instead of giant blobs? */
-   if (!obj->mem && vkAllocateMemory(screen->dev, &mai, NULL, &obj->mem) != VK_SUCCESS) {
+   if (obj->dedicated && vkAllocateMemory(screen->dev, &mai, NULL, &obj->mem) != VK_SUCCESS) {
       debug_printf("vkAllocateMemory failed\n");
       goto fail2;
    }
 
-   obj->offset = 0;
-   obj->size = reqs.size;
-
    if (templ->target == PIPE_BUFFER) {
       if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
          if (vkBindBufferMemory(screen->dev, obj->buffer, obj->mem, obj->offset) != VK_SUCCESS)
@@ -1109,6 +1038,8 @@ map_resource(struct zink_screen *screen, struct zink_resource *res)
    if (res->obj->map)
       return res->obj->map;
    assert(res->obj->host_visible);
+   if (!res->obj->dedicated)
+      return zink_bo_map(screen, res->obj->bo);
    result = vkMapMemory(screen->dev, res->obj->mem, res->obj->offset,
                         res->obj->size, 0, &res->obj->map);
    if (zink_screen_handle_vkresult(screen, result))
@@ -1120,6 +1051,9 @@ static void
 unmap_resource(struct zink_screen *screen, struct zink_resource *res)
 {
    res->obj->map = NULL;
+   if (!res->obj->dedicated)
+      zink_bo_unmap(screen, res->obj->bo);
+   else
       vkUnmapMemory(screen->dev, res->obj->mem);
 }
 
@@ -1672,14 +1606,6 @@ zink_screen_resource_init(struct pipe_screen *pscreen)
       pscreen->resource_from_handle = zink_resource_from_handle;
    }
    pscreen->resource_get_param = zink_resource_get_param;
-
-   screen->mem = rzalloc_array(screen, struct zink_mem_cache, screen->info.mem_props.memoryTypeCount);
-   if (!screen->mem)
-      return false;
-   for (uint32_t i = 0; i < screen->info.mem_props.memoryTypeCount; ++i) {
-      simple_mtx_init(&screen->mem[i].mem_cache_mtx, mtx_plain);
-      _mesa_hash_table_init(&screen->mem[i].resource_mem_cache, screen, mem_hash, mem_equals);
-   }
    return true;
 }
 

src/gallium/drivers/zink/zink_resource.h

@@ -28,7 +28,7 @@ struct pipe_screen;
 struct sw_displaytarget;
 struct zink_batch;
 struct zink_context;
-
+struct zink_bo;
 #define ZINK_RESOURCE_USAGE_STREAMOUT (1 << 10) //much greater than ZINK_DESCRIPTOR_TYPES
 
 #include "util/simple_mtx.h"
@@ -44,12 +44,6 @@ struct zink_context;
 
 #define ZINK_MAP_TEMPORARY (PIPE_MAP_DRV_PRV << 0)
 
-enum zink_resource_access {
-   ZINK_RESOURCE_ACCESS_READ = 1,
-   ZINK_RESOURCE_ACCESS_WRITE = 32,
-   ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
-};
-
 struct mem_key {
    unsigned seen_count;
    struct {
@@ -70,9 +64,9 @@ struct zink_resource_object {
    bool transfer_dst;
    VkImageAspectFlags modifier_aspect;
 
+   bool dedicated;
+   struct zink_bo *bo;
    VkDeviceMemory mem;
-   uint32_t mem_hash;
-   struct mem_key mkey;
    VkDeviceSize offset, size, alignment;
 
    VkSampleLocationsInfoEXT zs_evaluate;
@@ -187,75 +181,98 @@ bool
 zink_resource_object_init_storage(struct zink_context *ctx, struct zink_resource *res);
 
 #ifndef __cplusplus
+#include "zink_bo.h"
 
 static inline bool
 zink_resource_usage_is_unflushed(const struct zink_resource *res)
 {
+   if (res->obj->dedicated)
       return zink_batch_usage_is_unflushed(res->obj->reads) ||
              zink_batch_usage_is_unflushed(res->obj->writes);
+   return zink_bo_has_unflushed_usage(res->obj->bo);
 }
 
 static inline bool
 zink_resource_usage_is_unflushed_write(const struct zink_resource *res)
 {
+   if (res->obj->dedicated)
       return zink_batch_usage_is_unflushed(res->obj->writes);
+   return zink_batch_usage_is_unflushed(res->obj->bo->writes);
 }
 
 
 static inline bool
 zink_resource_usage_matches(const struct zink_resource *res, const struct zink_batch_state *bs)
 {
+   if (res->obj->dedicated)
       return zink_batch_usage_matches(res->obj->reads, bs) ||
              zink_batch_usage_matches(res->obj->writes, bs);
+   return zink_bo_usage_matches(res->obj->bo, bs);
 }
 
 static inline bool
 zink_resource_has_usage(const struct zink_resource *res)
 {
+   if (res->obj->dedicated)
       return zink_batch_usage_exists(res->obj->reads) ||
              zink_batch_usage_exists(res->obj->writes);
+   return zink_bo_has_usage(res->obj->bo);
 }
 
 static inline bool
 zink_resource_has_unflushed_usage(const struct zink_resource *res)
 {
+   if (res->obj->dedicated)
       return zink_batch_usage_is_unflushed(res->obj->reads) ||
              zink_batch_usage_is_unflushed(res->obj->writes);
+   return zink_bo_has_unflushed_usage(res->obj->bo);
 }
 
 static inline bool
 zink_resource_usage_check_completion(struct zink_screen *screen, struct zink_resource *res, enum zink_resource_access access)
 {
+   if (res->obj->dedicated) {
       if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, res->obj->reads))
          return false;
       if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, res->obj->writes))
          return false;
       return true;
    }
+   return zink_bo_usage_check_completion(screen, res->obj->bo, access);
+}
 
 static inline void
 zink_resource_usage_wait(struct zink_context *ctx, struct zink_resource *res, enum zink_resource_access access)
 {
+   if (res->obj->dedicated) {
       if (access & ZINK_RESOURCE_ACCESS_READ)
          zink_batch_usage_wait(ctx, res->obj->reads);
       if (access & ZINK_RESOURCE_ACCESS_WRITE)
          zink_batch_usage_wait(ctx, res->obj->writes);
+   } else
+      zink_bo_usage_wait(ctx, res->obj->bo, access);
 }
 
 static inline void
 zink_resource_usage_set(struct zink_resource *res, struct zink_batch_state *bs, bool write)
 {
+   if (res->obj->dedicated) {
       if (write)
          zink_batch_usage_set(&res->obj->writes, bs);
       else
          zink_batch_usage_set(&res->obj->reads, bs);
+   } else
+      zink_bo_usage_set(res->obj->bo, bs, write);
 }
 
 static inline void
 zink_resource_object_usage_unset(struct zink_resource_object *obj, struct zink_batch_state *bs)
 {
+   if (obj->dedicated) {
       zink_batch_usage_unset(&obj->reads, bs);
       zink_batch_usage_unset(&obj->writes, bs);
+   } else
+      zink_bo_usage_unset(obj->bo, bs);
 }
 
 #endif

src/gallium/drivers/zink/zink_screen.c

@@ -585,8 +585,8 @@ zink_get_param(struct pipe_screen *pscreen, enum pipe_cap param)
       return screen->info.feats.features.shaderCullDistance;
 
    case PIPE_CAP_SPARSE_BUFFER_PAGE_SIZE:
-      /* this is the spec minimum */
+      
-      return screen->info.feats.features.sparseBinding ? 64 * 1024 : 0;
+      return screen->info.feats.features.sparseBinding ? ZINK_SPARSE_BUFFER_PAGE_SIZE : 0;
 
    case PIPE_CAP_VIEWPORT_SUBPIXEL_BITS:
       return screen->info.props.limits.viewportSubPixelBits;
@@ -1037,16 +1037,6 @@ zink_is_format_supported(struct pipe_screen *pscreen,
    return true;
 }
 
-static void
-resource_cache_entry_destroy(struct zink_screen *screen, struct hash_entry *he)
-{
-   struct util_dynarray *array = (void*)he->data;
-   util_dynarray_foreach(array, struct mem_cache_entry, mc) {
-      vkFreeMemory(screen->dev, mc->mem, NULL);
-   }
-   util_dynarray_fini(array);
-}
-
 static void
 zink_destroy_screen(struct pipe_screen *pscreen)
 {
@@ -1087,15 +1077,7 @@ zink_destroy_screen(struct pipe_screen *pscreen)
    }
 #endif
    disk_cache_destroy(screen->disk_cache);
-
+   zink_bo_deinit(screen);
-   for (uint32_t i = 0; i < screen->info.mem_props.memoryHeapCount; ++i) {
-      simple_mtx_lock(&screen->mem[i].mem_cache_mtx);
-      hash_table_foreach(&screen->mem[i].resource_mem_cache, he)
-         resource_cache_entry_destroy(screen, he);
-      simple_mtx_unlock(&screen->mem[i].mem_cache_mtx);
-      simple_mtx_destroy(&screen->mem[i].mem_cache_mtx);
-   }
-
    util_live_shader_cache_deinit(&screen->shaders);
 
    if (screen->sem)
@@ -1892,6 +1874,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
 
    if (!zink_screen_resource_init(&screen->base))
       goto fail;
+   zink_bo_init(screen);
    zink_screen_fence_init(&screen->base);
 
    zink_screen_init_compiler(screen);
@@ -1921,6 +1904,25 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
    if (screen->info.have_KHR_timeline_semaphore)
       zink_screen_init_semaphore(screen);
 
+   memset(&screen->heap_map, UINT8_MAX, sizeof(screen->heap_map));
+   for (enum zink_heap i = 0; i < ZINK_HEAP_MAX; i++) {
+      for (unsigned j = 0; j < screen->info.mem_props.memoryTypeCount; j++) {
+         VkMemoryPropertyFlags domains = vk_domain_from_heap(i);
+         if ((screen->info.mem_props.memoryTypes[j].propertyFlags & domains) == domains) {
+            assert(screen->heap_map[i] == UINT8_MAX);
+            screen->heap_map[i] = j;
+            break;
+         }
+      }
+
+      /* not found: use compatible heap */
+      if (screen->heap_map[i] == UINT8_MAX) {
+         /* only cached mem has a failure case for now */
+         assert(i == ZINK_HEAP_HOST_VISIBLE_CACHED);
+         screen->heap_map[i] = screen->heap_map[ZINK_HEAP_HOST_VISIBLE_ANY];
+      }
+   }
+
    simple_mtx_init(&screen->surface_mtx, mtx_plain);
    simple_mtx_init(&screen->bufferview_mtx, mtx_plain);
    simple_mtx_init(&screen->framebuffer_mtx, mtx_plain);

src/gallium/drivers/zink/zink_screen.h

@@ -37,7 +37,8 @@
 #include "util/simple_mtx.h"
 #include "util/u_queue.h"
 #include "util/u_live_shader_cache.h"
-
+#include "pipebuffer/pb_cache.h"
+#include "pipebuffer/pb_slab.h"
 #include <vulkan/vulkan.h>
 
 extern uint32_t zink_debug;
@@ -50,11 +51,16 @@ struct zink_program;
 struct zink_shader;
 enum zink_descriptor_type;
 
+/* this is the spec minimum */
+#define ZINK_SPARSE_BUFFER_PAGE_SIZE (64 * 1024)
+
 #define ZINK_DEBUG_NIR 0x1
 #define ZINK_DEBUG_SPIRV 0x2
 #define ZINK_DEBUG_TGSI 0x4
 #define ZINK_DEBUG_VALIDATION 0x8
 
+#define NUM_SLAB_ALLOCATORS 3
+
 enum zink_descriptor_mode {
    ZINK_DESCRIPTOR_MODE_AUTO,
    ZINK_DESCRIPTOR_MODE_LAZY,
@@ -66,13 +72,6 @@ struct zink_modifier_prop {
     VkDrmFormatModifierPropertiesEXT*    pDrmFormatModifierProperties;
 };
 
-struct zink_mem_cache {
-   simple_mtx_t mem_cache_mtx;
-   struct hash_table resource_mem_cache;
-   uint64_t mem_cache_size;
-   unsigned mem_cache_count;
-};
-
 struct zink_screen {
    struct pipe_screen base;
    bool threaded;
@@ -99,7 +98,15 @@ struct zink_screen {
 
    struct util_live_shader_cache shaders;
 
-   struct zink_mem_cache *mem;
+   struct {
+      struct pb_cache bo_cache;
+      struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
+      unsigned min_alloc_size;
+      struct hash_table *bo_export_table;
+      simple_mtx_t bo_export_table_lock;
+      uint32_t next_bo_unique_id;
+   } pb;
+   uint8_t heap_map[VK_MAX_MEMORY_TYPES];
 
    uint64_t total_video_mem;
    uint64_t clamp_video_mem;