fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2025-12-21 22:20:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

zink: add a suballocator

Browse source 
this is an aux/pipebuffer implementation borrowing heavily from the
one in radeonsi. it currently has the following limitations, which
will be resolved in a followup series:
* 32bit address space still explodes
* swapchain images still have separate memory handling

performance in games like Tomb Raider has been observed to increase by
over 1000%

SQUASHED: simplify get_memory_type_index()

now that the heaps are enumerated, this can be reduced to a simple
array index with a fallback

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12146>
This commit is contained in:
Mike Blumenkrantz 2021-05-13 07:33:44 -04:00 • committed by Marge Bot
parent 5df677e996
commit 40fdb3212c
8 changed files with 1425 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/gallium/drivers/zink/meson.build
View file

@ -24,6 +24,7 @@ files_libzink = files(
'nir_to_spirv/spirv_builder.c',
'zink_batch.c',
'zink_blit.c',
'zink_bo.c',
'zink_clear.c',
'zink_compiler.c',
'zink_context.c',

1014
src/gallium/drivers/zink/zink_bo.c Normal file
View file

File diff suppressed because it is too large Load diff

270
src/gallium/drivers/zink/zink_bo.h Normal file
View file

@ -0,0 +1,270 @@
/*
* Copyright © 2021 Valve Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*
* Authors:
* Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
*/
#ifndef ZINK_BO_H
#define ZINK_BO_H
#include <vulkan/vulkan.h>
#include "pipebuffer/pb_cache.h"
#include "pipebuffer/pb_slab.h"
#include "zink_batch.h"
#define VK_VIS_VRAM (VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
enum zink_resource_access {
ZINK_RESOURCE_ACCESS_READ = 1,
ZINK_RESOURCE_ACCESS_WRITE = 32,
ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
};
enum zink_heap {
ZINK_HEAP_DEVICE_LOCAL,
ZINK_HEAP_DEVICE_LOCAL_SPARSE,
ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
ZINK_HEAP_HOST_VISIBLE_ANY,
ZINK_HEAP_HOST_VISIBLE_COHERENT,
ZINK_HEAP_HOST_VISIBLE_CACHED,
ZINK_HEAP_MAX,
};
enum zink_alloc_flag {
ZINK_ALLOC_SPARSE = 1<<0,
ZINK_ALLOC_NO_SUBALLOC = 1<<1,
};
struct zink_bo {
struct pb_buffer base;
union {
struct {
void *cpu_ptr; /* for user_ptr and permanent maps */
int map_count;
bool is_user_ptr;
bool use_reusable_pool;
/* Whether buffer_get_handle or buffer_from_handle has been called,
* it can only transition from false to true. Protected by lock.
*/
bool is_shared;
} real;
struct {
struct pb_slab_entry entry;
struct zink_bo *real;
} slab;
struct {
uint32_t num_va_pages;
uint32_t num_backing_pages;
struct list_head backing;
/* Commitment information for each page of the virtual memory area. */
struct zink_sparse_commitment *commitments;
} sparse;
} u;
VkDeviceMemory mem;
uint64_t offset;
uint32_t unique_id;
simple_mtx_t lock;
struct zink_batch_usage *reads;
struct zink_batch_usage *writes;
struct pb_cache_entry cache_entry[];
};
static inline struct zink_bo *
zink_bo(struct pb_buffer *pbuf)
{
return (struct zink_bo*)pbuf;
}
static inline enum zink_alloc_flag
zink_alloc_flags_from_heap(enum zink_heap heap)
{
enum zink_alloc_flag flags = 0;
switch (heap) {
case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
flags |= ZINK_ALLOC_SPARSE;
break;
default:
break;
}
return flags;
}
static inline VkMemoryPropertyFlags
vk_domain_from_heap(enum zink_heap heap)
{
VkMemoryPropertyFlags domains = 0;
switch (heap) {
case ZINK_HEAP_DEVICE_LOCAL:
case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
domains = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
break;
case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
break;
case ZINK_HEAP_HOST_VISIBLE_ANY:
domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
break;
case ZINK_HEAP_HOST_VISIBLE_COHERENT:
domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
break;
case ZINK_HEAP_HOST_VISIBLE_CACHED:
domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
break;
default:
break;
}
return domains;
}
static inline enum zink_heap
zink_heap_from_domain_flags(VkMemoryPropertyFlags domains, enum zink_alloc_flag flags)
{
if (flags & ZINK_ALLOC_SPARSE)
return ZINK_HEAP_DEVICE_LOCAL_SPARSE;
if ((domains & VK_VIS_VRAM) == VK_VIS_VRAM)
return ZINK_HEAP_DEVICE_LOCAL_VISIBLE;
if (domains & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
return ZINK_HEAP_DEVICE_LOCAL;
if (domains & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)
return ZINK_HEAP_HOST_VISIBLE_COHERENT;
if (domains & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
return ZINK_HEAP_HOST_VISIBLE_CACHED;
return ZINK_HEAP_HOST_VISIBLE_ANY;
}
bool
zink_bo_init(struct zink_screen *screen);
void
zink_bo_deinit(struct zink_screen *screen);
struct pb_buffer *
zink_bo_create(struct zink_screen *screen, uint64_t size, unsigned alignment, enum zink_heap heap, enum zink_alloc_flag flags);
static inline uint64_t
zink_bo_get_offset(const struct zink_bo *bo)
{
return bo->offset;
}
static inline VkDeviceMemory
zink_bo_get_mem(const struct zink_bo *bo)
{
return bo->mem ? bo->mem : bo->u.slab.real->mem;
}
static inline VkDeviceSize
zink_bo_get_size(const struct zink_bo *bo)
{
return bo->mem ? bo->base.size : bo->u.slab.real->base.size;
}
void *
zink_bo_map(struct zink_screen *screen, struct zink_bo *bo);
void
zink_bo_unmap(struct zink_screen *screen, struct zink_bo *bo);
bool
zink_bo_commit(struct zink_screen *screen, struct zink_resource *res, uint32_t offset, uint32_t size, bool commit);
static inline bool
zink_bo_has_unflushed_usage(const struct zink_bo *bo)
{
return zink_batch_usage_is_unflushed(bo->reads) ||
zink_batch_usage_is_unflushed(bo->writes);
}
static inline bool
zink_bo_has_usage(const struct zink_bo *bo)
{
return zink_batch_usage_exists(bo->reads) ||
zink_batch_usage_exists(bo->writes);
}
static inline bool
zink_bo_usage_matches(const struct zink_bo *bo, const struct zink_batch_state *bs)
{
return zink_batch_usage_matches(bo->reads, bs) ||
zink_batch_usage_matches(bo->writes, bs);
}
static inline bool
zink_bo_usage_check_completion(struct zink_screen *screen, struct zink_bo *bo, enum zink_resource_access access)
{
if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, bo->reads))
return false;
if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, bo->writes))
return false;
return true;
}
static inline void
zink_bo_usage_wait(struct zink_context *ctx, struct zink_bo *bo, enum zink_resource_access access)
{
if (access & ZINK_RESOURCE_ACCESS_READ)
zink_batch_usage_wait(ctx, bo->reads);
if (access & ZINK_RESOURCE_ACCESS_WRITE)
zink_batch_usage_wait(ctx, bo->writes);
}
static inline void
zink_bo_usage_set(struct zink_bo *bo, struct zink_batch_state *bs, bool write)
{
if (write)
zink_batch_usage_set(&bo->writes, bs);
else
zink_batch_usage_set(&bo->reads, bs);
}
static inline void
zink_bo_usage_unset(struct zink_bo *bo, struct zink_batch_state *bs)
{
zink_batch_usage_unset(&bo->reads, bs);
zink_batch_usage_unset(&bo->writes, bs);
}
static inline void
zink_bo_unref(struct zink_screen *screen, struct zink_bo *bo)
{
struct pb_buffer *pbuf = &bo->base;
pb_reference_with_winsys(screen, &pbuf, NULL);
}
#endif

70
src/gallium/drivers/zink/zink_context.c
View file

@ -3201,29 +3201,6 @@ rebind_buffer(struct zink_context *ctx, struct zink_resource *res)
zink_batch_resource_usage_set(&ctx->batch, res, has_write);
}
static inline struct zink_screen **
get_screen_ptr_for_commit(uint8_t *mem)
{
return (struct zink_screen**)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind));
}
static bool
resource_commit(struct zink_screen *screen, VkBindSparseInfo *sparse)
{
VkQueue queue = screen->threaded ? screen->thread_queue : screen->queue;
VkResult ret = vkQueueBindSparse(queue, 1, sparse, VK_NULL_HANDLE);
return zink_screen_handle_vkresult(screen, ret);
}
static void
submit_resource_commit(void *data, void *gdata, int thread_index)
{
struct zink_screen **screen = get_screen_ptr_for_commit(data);
resource_commit(*screen, data);
free(data);
}
static bool
zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
{
@ -3235,50 +3212,11 @@ zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsi
if (zink_resource_has_unflushed_usage(res))
zink_flush_queue(ctx);
uint8_t *mem = malloc(sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind) + sizeof(void*));
if (!mem)
return false;
VkBindSparseInfo *sparse = (void*)mem;
sparse->sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
sparse->pNext = NULL;
sparse->waitSemaphoreCount = 0;
sparse->bufferBindCount = 1;
sparse->imageOpaqueBindCount = 0;
sparse->imageBindCount = 0;
sparse->signalSemaphoreCount = 0;
bool ret = zink_bo_commit(screen, res, box->x, box->width, commit);
if (!ret)
check_device_lost(ctx);
VkSparseBufferMemoryBindInfo *sparse_bind = (void*)(mem + sizeof(VkBindSparseInfo));
sparse_bind->buffer = res->obj->buffer;
sparse_bind->bindCount = 1;
sparse->pBufferBinds = sparse_bind;
VkSparseMemoryBind *mem_bind = (void*)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo));
mem_bind->resourceOffset = box->x;
mem_bind->size = box->width;
mem_bind->memory = commit ? res->obj->mem : VK_NULL_HANDLE;
/* currently sparse buffers allocate memory 1:1 for the max sparse size,
* but probably it should dynamically allocate the committed regions;
* if this ever changes, update the below line
*/
mem_bind->memoryOffset = box->x;
mem_bind->flags = 0;
sparse_bind->pBinds = mem_bind;
struct zink_screen **ptr = get_screen_ptr_for_commit(mem);
*ptr = screen;
if (screen->threaded) {
/* this doesn't need any kind of fencing because any access to this resource
* will be automagically synchronized by queue dispatch */
util_queue_add_job(&screen->flush_queue, mem, NULL, submit_resource_commit, NULL, 0);
} else {
bool ret = resource_commit(screen, sparse);
if (!ret)
check_device_lost(ctx);
free(sparse);
return ret;
}
return true;
return ret;
}
static void

168
src/gallium/drivers/zink/zink_resource.c
View file

@ -78,49 +78,6 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object
sprintf(buf, "zink_resource_object");
}
static uint32_t
mem_hash(const void *key)
{
const struct mem_key *mkey = key;
return _mesa_hash_data(&mkey->key, sizeof(mkey->key));
}
static bool
mem_equals(const void *a, const void *b)
{
const struct mem_key *ma = a;
const struct mem_key *mb = b;
return !memcmp(&ma->key, &mb->key, sizeof(ma->key));
}
static void
cache_or_free_mem(struct zink_screen *screen, struct zink_resource_object *obj)
{
if (obj->mkey.key.heap_index != UINT32_MAX) {
simple_mtx_lock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&screen->mem[obj->mkey.key.heap_index].resource_mem_cache, obj->mem_hash, &obj->mkey);
assert(he);
struct util_dynarray *array = he->data;
struct mem_key *mkey = (void*)he->key;
unsigned seen = mkey->seen_count;
mkey->seen_count--;
if (util_dynarray_num_elements(array, struct mem_cache_entry) < seen) {
struct mem_cache_entry mc = { obj->mem, obj->map };
screen->mem[obj->mkey.key.heap_index].mem_cache_size += obj->size;
if (sizeof(void*) == 4 && obj->map) {
vkUnmapMemory(screen->dev, obj->mem);
mc.map = NULL;
}
util_dynarray_append(array, struct mem_cache_entry, mc);
simple_mtx_unlock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
return;
}
simple_mtx_unlock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
}
vkFreeMemory(screen->dev, obj->mem, NULL);
}
void
zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_object *obj)
{
@ -134,7 +91,10 @@ zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_ob
util_dynarray_fini(&obj->tmp);
zink_descriptor_set_refs_clear(&obj->desc_set_refs, obj);
cache_or_free_mem(screen, obj);
if (obj->dedicated)
vkFreeMemory(screen->dev, obj->mem, NULL);
else
zink_bo_unref(screen, obj->bo);
FREE(obj);
}
@ -155,38 +115,6 @@ zink_resource_destroy(struct pipe_screen *pscreen,
FREE(res);
}
static uint32_t
get_memory_type_index(struct zink_screen *screen,
const VkMemoryRequirements *reqs,
VkMemoryPropertyFlags props)
{
int32_t idx = -1;
for (uint32_t i = 0u; i < VK_MAX_MEMORY_TYPES; i++) {
if (((reqs->memoryTypeBits >> i) & 1) == 1) {
if ((screen->info.mem_props.memoryTypes[i].propertyFlags & props) == props) {
if (!(props & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) &&
screen->info.mem_props.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
idx = i;
} else
return i;
}
}
}
if (idx >= 0)
return idx;
if (props & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
/* if no suitable cached memory can be found, fall back
* to non-cached memory instead.
*/
return get_memory_type_index(screen, reqs,
props & ~VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
}
unreachable("Unsupported memory-type");
return 0;
}
static VkImageAspectFlags
aspect_from_format(enum pipe_format fmt)
{
@ -636,17 +564,31 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
VkMemoryAllocateInfo mai = {0};
enum zink_alloc_flag aflags = templ->flags & PIPE_RESOURCE_FLAG_SPARSE ? ZINK_ALLOC_SPARSE : 0;
mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
mai.allocationSize = reqs.size;
mai.memoryTypeIndex = get_memory_type_index(screen, &reqs, flags);
enum zink_heap heap = zink_heap_from_domain_flags(flags, aflags);
mai.memoryTypeIndex = screen->heap_map[heap];
if (unlikely(!(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex)))) {
/* not valid based on reqs; demote to more compatible type */
switch (heap) {
case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
heap = ZINK_HEAP_DEVICE_LOCAL;
break;
case ZINK_HEAP_HOST_VISIBLE_CACHED:
heap = ZINK_HEAP_HOST_VISIBLE_ANY;
break;
default:
break;
}
mai.memoryTypeIndex = screen->heap_map[heap];
assert(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex));
}
VkMemoryType mem_type = screen->info.mem_props.memoryTypes[mai.memoryTypeIndex];
obj->coherent = mem_type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
obj->host_visible = mem_type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
if (templ->target == PIPE_BUFFER && !obj->coherent && obj->host_visible) {
mai.allocationSize = reqs.size = align(reqs.size, screen->info.props.limits.nonCoherentAtomSize);
}
VkMemoryDedicatedAllocateInfo ded_alloc_info = {
.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
@ -695,46 +637,33 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
mai.pNext = &memory_wsi_info;
}
if (!mai.pNext && !(templ->flags & (PIPE_RESOURCE_FLAG_MAP_COHERENT | PIPE_RESOURCE_FLAG_SPARSE))) {
obj->mkey.key.reqs = reqs;
obj->mkey.key.heap_index = mai.memoryTypeIndex;
obj->mem_hash = mem_hash(&obj->mkey);
simple_mtx_lock(&screen->mem[mai.memoryTypeIndex].mem_cache_mtx);
struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&screen->mem[mai.memoryTypeIndex].resource_mem_cache, obj->mem_hash, &obj->mkey);
struct mem_key *mkey;
if (he) {
struct util_dynarray *array = he->data;
mkey = (void*)he->key;
if (array && util_dynarray_num_elements(array, struct mem_cache_entry)) {
struct mem_cache_entry mc = util_dynarray_pop(array, struct mem_cache_entry);
obj->mem = mc.mem;
obj->map = mc.map;
screen->mem[mai.memoryTypeIndex].mem_cache_size -= reqs.size;
screen->mem[mai.memoryTypeIndex].mem_cache_count--;
}
if (!mai.pNext) {
unsigned alignment = MAX2(reqs.alignment, 256);
if (templ->usage == PIPE_USAGE_STAGING && obj->is_buffer)
alignment = MAX2(alignment, screen->info.props.limits.minMemoryMapAlignment);
obj->alignment = alignment;
obj->bo = zink_bo(zink_bo_create(screen, reqs.size, alignment, heap, 0));
if (!obj->bo)
goto fail2;
if (aflags == ZINK_ALLOC_SPARSE) {
obj->size = templ->width0;
} else {
mkey = ralloc(screen, struct mem_key);
memcpy(&mkey->key, &obj->mkey.key, sizeof(obj->mkey.key));
mkey->seen_count = 0;
struct util_dynarray *array = rzalloc(screen, struct util_dynarray);
util_dynarray_init(array, screen);
_mesa_hash_table_insert_pre_hashed(&screen->mem[mai.memoryTypeIndex].resource_mem_cache, obj->mem_hash, mkey, array);
obj->offset = zink_bo_get_offset(obj->bo);
obj->mem = zink_bo_get_mem(obj->bo);
obj->size = zink_bo_get_size(obj->bo);
}
mkey->seen_count++;
simple_mtx_unlock(&screen->mem[mai.memoryTypeIndex].mem_cache_mtx);
} else
obj->mkey.key.heap_index = UINT32_MAX;
} else {
obj->dedicated = true;
obj->offset = 0;
obj->size = reqs.size;
}
/* TODO: sparse buffers should probably allocate multiple regions of memory instead of giant blobs? */
if (!obj->mem && vkAllocateMemory(screen->dev, &mai, NULL, &obj->mem) != VK_SUCCESS) {
if (obj->dedicated && vkAllocateMemory(screen->dev, &mai, NULL, &obj->mem) != VK_SUCCESS) {
debug_printf("vkAllocateMemory failed\n");
goto fail2;
}
obj->offset = 0;
obj->size = reqs.size;
if (templ->target == PIPE_BUFFER) {
if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
if (vkBindBufferMemory(screen->dev, obj->buffer, obj->mem, obj->offset) != VK_SUCCESS)
@ -1109,6 +1038,8 @@ map_resource(struct zink_screen *screen, struct zink_resource *res)
if (res->obj->map)
return res->obj->map;
assert(res->obj->host_visible);
if (!res->obj->dedicated)
return zink_bo_map(screen, res->obj->bo);
result = vkMapMemory(screen->dev, res->obj->mem, res->obj->offset,
res->obj->size, 0, &res->obj->map);
if (zink_screen_handle_vkresult(screen, result))
@ -1120,7 +1051,10 @@ static void
unmap_resource(struct zink_screen *screen, struct zink_resource *res)
{
res->obj->map = NULL;
vkUnmapMemory(screen->dev, res->obj->mem);
if (!res->obj->dedicated)
zink_bo_unmap(screen, res->obj->bo);
else
vkUnmapMemory(screen->dev, res->obj->mem);
}
static void *
@ -1672,14 +1606,6 @@ zink_screen_resource_init(struct pipe_screen *pscreen)
pscreen->resource_from_handle = zink_resource_from_handle;
}
pscreen->resource_get_param = zink_resource_get_param;
screen->mem = rzalloc_array(screen, struct zink_mem_cache, screen->info.mem_props.memoryTypeCount);
if (!screen->mem)
return false;
for (uint32_t i = 0; i < screen->info.mem_props.memoryTypeCount; ++i) {
simple_mtx_init(&screen->mem[i].mem_cache_mtx, mtx_plain);
_mesa_hash_table_init(&screen->mem[i].resource_mem_cache, screen, mem_hash, mem_equals);
}
return true;
}

83
src/gallium/drivers/zink/zink_resource.h
View file

@ -28,7 +28,7 @@ struct pipe_screen;
struct sw_displaytarget;
struct zink_batch;
struct zink_context;
struct zink_bo;
#define ZINK_RESOURCE_USAGE_STREAMOUT (1 << 10) //much greater than ZINK_DESCRIPTOR_TYPES
#include "util/simple_mtx.h"
@ -44,12 +44,6 @@ struct zink_context;
#define ZINK_MAP_TEMPORARY (PIPE_MAP_DRV_PRV << 0)
enum zink_resource_access {
ZINK_RESOURCE_ACCESS_READ = 1,
ZINK_RESOURCE_ACCESS_WRITE = 32,
ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
};
struct mem_key {
unsigned seen_count;
struct {
@ -70,9 +64,9 @@ struct zink_resource_object {
bool transfer_dst;
VkImageAspectFlags modifier_aspect;
bool dedicated;
struct zink_bo *bo;
VkDeviceMemory mem;
uint32_t mem_hash;
struct mem_key mkey;
VkDeviceSize offset, size, alignment;
VkSampleLocationsInfoEXT zs_evaluate;
@ -187,75 +181,98 @@ bool
zink_resource_object_init_storage(struct zink_context *ctx, struct zink_resource *res);
#ifndef __cplusplus
#include "zink_bo.h"
static inline bool
zink_resource_usage_is_unflushed(const struct zink_resource *res)
{
return zink_batch_usage_is_unflushed(res->obj->reads) ||
zink_batch_usage_is_unflushed(res->obj->writes);
if (res->obj->dedicated)
return zink_batch_usage_is_unflushed(res->obj->reads) ||
zink_batch_usage_is_unflushed(res->obj->writes);
return zink_bo_has_unflushed_usage(res->obj->bo);
}
static inline bool
zink_resource_usage_is_unflushed_write(const struct zink_resource *res)
{
return zink_batch_usage_is_unflushed(res->obj->writes);
if (res->obj->dedicated)
return zink_batch_usage_is_unflushed(res->obj->writes);
return zink_batch_usage_is_unflushed(res->obj->bo->writes);
}
static inline bool
zink_resource_usage_matches(const struct zink_resource *res, const struct zink_batch_state *bs)
{
return zink_batch_usage_matches(res->obj->reads, bs) ||
zink_batch_usage_matches(res->obj->writes, bs);
if (res->obj->dedicated)
return zink_batch_usage_matches(res->obj->reads, bs) ||
zink_batch_usage_matches(res->obj->writes, bs);
return zink_bo_usage_matches(res->obj->bo, bs);
}
static inline bool
zink_resource_has_usage(const struct zink_resource *res)
{
return zink_batch_usage_exists(res->obj->reads) ||
zink_batch_usage_exists(res->obj->writes);
if (res->obj->dedicated)
return zink_batch_usage_exists(res->obj->reads) ||
zink_batch_usage_exists(res->obj->writes);
return zink_bo_has_usage(res->obj->bo);
}
static inline bool
zink_resource_has_unflushed_usage(const struct zink_resource *res)
{
return zink_batch_usage_is_unflushed(res->obj->reads) ||
zink_batch_usage_is_unflushed(res->obj->writes);
if (res->obj->dedicated)
return zink_batch_usage_is_unflushed(res->obj->reads) ||
zink_batch_usage_is_unflushed(res->obj->writes);
return zink_bo_has_unflushed_usage(res->obj->bo);
}
static inline bool
zink_resource_usage_check_completion(struct zink_screen *screen, struct zink_resource *res, enum zink_resource_access access)
{
if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, res->obj->reads))
return false;
if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, res->obj->writes))
return false;
return true;
if (res->obj->dedicated) {
if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, res->obj->reads))
return false;
if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, res->obj->writes))
return false;
return true;
}
return zink_bo_usage_check_completion(screen, res->obj->bo, access);
}
static inline void
zink_resource_usage_wait(struct zink_context *ctx, struct zink_resource *res, enum zink_resource_access access)
{
if (access & ZINK_RESOURCE_ACCESS_READ)
zink_batch_usage_wait(ctx, res->obj->reads);
if (access & ZINK_RESOURCE_ACCESS_WRITE)
zink_batch_usage_wait(ctx, res->obj->writes);
if (res->obj->dedicated) {
if (access & ZINK_RESOURCE_ACCESS_READ)
zink_batch_usage_wait(ctx, res->obj->reads);
if (access & ZINK_RESOURCE_ACCESS_WRITE)
zink_batch_usage_wait(ctx, res->obj->writes);
} else
zink_bo_usage_wait(ctx, res->obj->bo, access);
}
static inline void
zink_resource_usage_set(struct zink_resource *res, struct zink_batch_state *bs, bool write)
{
if (write)
zink_batch_usage_set(&res->obj->writes, bs);
else
zink_batch_usage_set(&res->obj->reads, bs);
if (res->obj->dedicated) {
if (write)
zink_batch_usage_set(&res->obj->writes, bs);
else
zink_batch_usage_set(&res->obj->reads, bs);
} else
zink_bo_usage_set(res->obj->bo, bs, write);
}
static inline void
zink_resource_object_usage_unset(struct zink_resource_object *obj, struct zink_batch_state *bs)
{
zink_batch_usage_unset(&obj->reads, bs);
zink_batch_usage_unset(&obj->writes, bs);
if (obj->dedicated) {
zink_batch_usage_unset(&obj->reads, bs);
zink_batch_usage_unset(&obj->writes, bs);
} else
zink_bo_usage_unset(obj->bo, bs);
}
#endif

44
src/gallium/drivers/zink/zink_screen.c
View file

@ -585,8 +585,8 @@ zink_get_param(struct pipe_screen *pscreen, enum pipe_cap param)
return screen->info.feats.features.shaderCullDistance;
case PIPE_CAP_SPARSE_BUFFER_PAGE_SIZE:
/* this is the spec minimum */
return screen->info.feats.features.sparseBinding ? 64 * 1024 : 0;
return screen->info.feats.features.sparseBinding ? ZINK_SPARSE_BUFFER_PAGE_SIZE : 0;
case PIPE_CAP_VIEWPORT_SUBPIXEL_BITS:
return screen->info.props.limits.viewportSubPixelBits;
@ -1037,16 +1037,6 @@ zink_is_format_supported(struct pipe_screen *pscreen,
return true;
}
static void
resource_cache_entry_destroy(struct zink_screen *screen, struct hash_entry *he)
{
struct util_dynarray *array = (void*)he->data;
util_dynarray_foreach(array, struct mem_cache_entry, mc) {
vkFreeMemory(screen->dev, mc->mem, NULL);
}
util_dynarray_fini(array);
}
static void
zink_destroy_screen(struct pipe_screen *pscreen)
{
@ -1087,15 +1077,7 @@ zink_destroy_screen(struct pipe_screen *pscreen)
}
#endif
disk_cache_destroy(screen->disk_cache);
for (uint32_t i = 0; i < screen->info.mem_props.memoryHeapCount; ++i) {
simple_mtx_lock(&screen->mem[i].mem_cache_mtx);
hash_table_foreach(&screen->mem[i].resource_mem_cache, he)
resource_cache_entry_destroy(screen, he);
simple_mtx_unlock(&screen->mem[i].mem_cache_mtx);
simple_mtx_destroy(&screen->mem[i].mem_cache_mtx);
}
zink_bo_deinit(screen);
util_live_shader_cache_deinit(&screen->shaders);
if (screen->sem)
@ -1892,6 +1874,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
if (!zink_screen_resource_init(&screen->base))
goto fail;
zink_bo_init(screen);
zink_screen_fence_init(&screen->base);
zink_screen_init_compiler(screen);
@ -1921,6 +1904,25 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
if (screen->info.have_KHR_timeline_semaphore)
zink_screen_init_semaphore(screen);
memset(&screen->heap_map, UINT8_MAX, sizeof(screen->heap_map));
for (enum zink_heap i = 0; i < ZINK_HEAP_MAX; i++) {
for (unsigned j = 0; j < screen->info.mem_props.memoryTypeCount; j++) {
VkMemoryPropertyFlags domains = vk_domain_from_heap(i);
if ((screen->info.mem_props.memoryTypes[j].propertyFlags & domains) == domains) {
assert(screen->heap_map[i] == UINT8_MAX);
screen->heap_map[i] = j;
break;
}
}
/* not found: use compatible heap */
if (screen->heap_map[i] == UINT8_MAX) {
/* only cached mem has a failure case for now */
assert(i == ZINK_HEAP_HOST_VISIBLE_CACHED);
screen->heap_map[i] = screen->heap_map[ZINK_HEAP_HOST_VISIBLE_ANY];
}
}
simple_mtx_init(&screen->surface_mtx, mtx_plain);
simple_mtx_init(&screen->bufferview_mtx, mtx_plain);
simple_mtx_init(&screen->framebuffer_mtx, mtx_plain);

25
src/gallium/drivers/zink/zink_screen.h
View file

@ -37,7 +37,8 @@
#include "util/simple_mtx.h"
#include "util/u_queue.h"
#include "util/u_live_shader_cache.h"
#include "pipebuffer/pb_cache.h"
#include "pipebuffer/pb_slab.h"
#include <vulkan/vulkan.h>
extern uint32_t zink_debug;
@ -50,11 +51,16 @@ struct zink_program;
struct zink_shader;
enum zink_descriptor_type;
/* this is the spec minimum */
#define ZINK_SPARSE_BUFFER_PAGE_SIZE (64 * 1024)
#define ZINK_DEBUG_NIR 0x1
#define ZINK_DEBUG_SPIRV 0x2
#define ZINK_DEBUG_TGSI 0x4
#define ZINK_DEBUG_VALIDATION 0x8
#define NUM_SLAB_ALLOCATORS 3
enum zink_descriptor_mode {
ZINK_DESCRIPTOR_MODE_AUTO,
ZINK_DESCRIPTOR_MODE_LAZY,
@ -66,13 +72,6 @@ struct zink_modifier_prop {
VkDrmFormatModifierPropertiesEXT* pDrmFormatModifierProperties;
};
struct zink_mem_cache {
simple_mtx_t mem_cache_mtx;
struct hash_table resource_mem_cache;
uint64_t mem_cache_size;
unsigned mem_cache_count;
};
struct zink_screen {
struct pipe_screen base;
bool threaded;
@ -99,7 +98,15 @@ struct zink_screen {
struct util_live_shader_cache shaders;
struct zink_mem_cache *mem;
struct {
struct pb_cache bo_cache;
struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
unsigned min_alloc_size;
struct hash_table *bo_export_table;
simple_mtx_t bo_export_table_lock;
uint32_t next_bo_unique_id;
} pb;
uint8_t heap_map[VK_MAX_MEMORY_TYPES];
uint64_t total_video_mem;
uint64_t clamp_video_mem;