fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2025-12-23 09:00:10 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

zink: queue v3.0

Browse source 
this uses a pointer to a batch state substruct for timeline tracking,
which provides a few nice benefits:
* explicit ability to detect unflushed batches (even on other contexts)
* the context doesn't need to have a "current" timeline id
* timeline (batch) ids can be distributed during submit, not when recording begins
* an abstracted api which can be more easily changed under the hood

Acked-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11437>
This commit is contained in:
Mike Blumenkrantz 2021-06-16 21:28:46 -04:00 • committed by Marge Bot
parent 28496f6ff2
commit d80d9e1c93
11 changed files with 95 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
src/gallium/drivers/zink/zink_batch.c
View file

@ -95,6 +95,7 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
zink_screen_update_last_finished(screen, bs->fence.batch_id);
bs->submit_count++;
bs->fence.batch_id = 0;
bs->usage.usage = 0;
bs->draw_count = bs->compute_count = 0;
}
@ -278,6 +279,8 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch)
{
zink_reset_batch(ctx, batch);
batch->state->usage.unflushed = true;
VkCommandBufferBeginInfo cbbi = {0};
cbbi.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
cbbi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
@ -290,7 +293,7 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch)
batch->state->fence.completed = false;
if (ctx->last_fence) {
struct zink_batch_state *last_state = zink_batch_state(ctx->last_fence);
batch->last_batch_id = last_state->fence.batch_id;
batch->last_batch_usage = &last_state->usage;
} else {
if (zink_screen(ctx->base.screen)->threaded)
util_queue_init(&batch->flush_queue, "zfq", 8, 1, UTIL_QUEUE_INIT_RESIZE_IF_FULL, NULL);
@ -315,7 +318,18 @@ static void
submit_queue(void *data, void *gdata, int thread_index)
{
struct zink_batch_state *bs = data;
struct zink_context *ctx = bs->ctx;
struct zink_screen *screen = zink_screen(ctx->base.screen);
VkSubmitInfo si = {0};
simple_mtx_lock(&ctx->batch_mtx);
while (!bs->fence.batch_id)
bs->fence.batch_id = p_atomic_inc_return(&screen->curr_batch);
_mesa_hash_table_insert_pre_hashed(&ctx->batch_states, bs->fence.batch_id, (void*)(uintptr_t)bs->fence.batch_id, bs);
bs->usage.usage = bs->fence.batch_id;
bs->usage.unflushed = false;
simple_mtx_unlock(&ctx->batch_mtx);
uint64_t batch_id = bs->fence.batch_id;
si.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
si.waitSemaphoreCount = 0;
@ -337,7 +351,7 @@ submit_queue(void *data, void *gdata, int thread_index)
tsi.signalSemaphoreValueCount = 1;
tsi.pSignalSemaphoreValues = &batch_id;
si.signalSemaphoreCount = 1;
si.pSignalSemaphores = &zink_screen(bs->ctx->base.screen)->sem;
si.pSignalSemaphores = &screen->sem;
}
struct wsi_memory_signal_submit_info mem_signal = {
@ -345,7 +359,7 @@ submit_queue(void *data, void *gdata, int thread_index)
.pNext = si.pNext,
};
if (bs->flush_res && zink_screen(bs->ctx->base.screen)->needs_mesa_flush_wsi) {
if (bs->flush_res && screen->needs_mesa_flush_wsi) {
mem_signal.memory = bs->flush_res->scanout_obj ? bs->flush_res->scanout_obj->mem : bs->flush_res->obj->mem;
si.pNext = &mem_signal;
}
@ -518,11 +532,8 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
vkFlushMappedMemoryRanges(screen->dev, 1, &range);
}
simple_mtx_lock(&ctx->batch_mtx);
ctx->last_fence = &batch->state->fence;
_mesa_hash_table_insert_pre_hashed(&ctx->batch_states, batch->state->fence.batch_id, (void*)(uintptr_t)batch->state->fence.batch_id, batch->state);
simple_mtx_unlock(&ctx->batch_mtx);
ctx->resource_size += batch->state->resource_size;
ctx->last_fence = &batch->state->fence;
if (screen->device_lost)
return;
@ -550,20 +561,20 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource
zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil);
/* if the resource already has usage of any sort set for this batch, we can skip hashing */
if (!zink_batch_usage_matches(&res->obj->reads, batch->state) &&
!zink_batch_usage_matches(&res->obj->writes, batch->state)) {
if (!zink_batch_usage_matches(res->obj->reads, batch->state) &&
!zink_batch_usage_matches(res->obj->writes, batch->state)) {
bool found = false;
_mesa_set_search_and_add(batch->state->fence.resources, res->obj, &found);
if (!found) {
pipe_reference(NULL, &res->obj->reference);
if (!batch->last_batch_id || res->obj->reads.usage != batch->last_batch_id)
if (!batch->last_batch_usage || res->obj->reads != batch->last_batch_usage)
/* only add resource usage if it's "new" usage, though this only checks the most recent usage
* and not all pending usages
*/
batch->state->resource_size += res->obj->size;
if (stencil) {
pipe_reference(NULL, &stencil->obj->reference);
if (!batch->last_batch_id || stencil->obj->reads.usage != batch->last_batch_id)
if (!batch->last_batch_usage || stencil->obj->reads != batch->last_batch_usage)
batch->state->resource_size += stencil->obj->size;
}
}
@ -586,10 +597,10 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource
}
bool
batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u)
batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage **u)
{
bool found = false;
if (u->usage == batch->state->fence.batch_id)
if (*u == &batch->state->usage)
return false;
_mesa_set_search_and_add(s, ptr, &found);
assert(!found);
@ -661,11 +672,13 @@ zink_batch_usage_check_completion(struct zink_context *ctx, const struct zink_ba
{
if (!zink_batch_usage_exists(u))
return true;
if (zink_batch_usage_is_unflushed(u))
return false;
return zink_check_batch_completion(ctx, u->usage);
}
void
zink_batch_usage_wait(struct zink_context *ctx, const struct zink_batch_usage *u)
zink_batch_usage_wait(struct zink_context *ctx, struct zink_batch_usage *u)
{
if (!zink_batch_usage_exists(u))
return;

33
src/gallium/drivers/zink/zink_batch.h
View file

@ -45,19 +45,20 @@ struct zink_sampler_view;
struct zink_surface;
struct zink_batch_usage {
/* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */
uint32_t usage;
bool unflushed;
};
/* not real api don't use */
bool
batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u);
batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage **u);
struct zink_batch_state {
struct zink_fence fence;
struct pipe_reference reference;
unsigned draw_count;
struct zink_batch_usage usage;
struct zink_context *ctx;
VkCommandPool cmdpool;
VkCommandBuffer cmdbuf;
@ -97,7 +98,7 @@ struct zink_batch_state {
struct zink_batch {
struct zink_batch_state *state;
uint32_t last_batch_id;
struct zink_batch_usage *last_batch_usage;
struct util_queue flush_queue; //TODO: move to wsi
bool has_work;
@ -175,34 +176,40 @@ zink_batch_state_reference(struct zink_screen *screen,
if (dst) *dst = src;
}
static inline void
zink_batch_usage_unset(struct zink_batch_usage *u, struct zink_batch_state *bs)
static inline bool
zink_batch_usage_is_unflushed(const struct zink_batch_usage *u)
{
p_atomic_cmpxchg(&u->usage, bs->fence.batch_id, 0);
return u && u->unflushed;
}
static inline void
zink_batch_usage_set(struct zink_batch_usage *u, struct zink_batch_state *bs)
zink_batch_usage_unset(struct zink_batch_usage **u, struct zink_batch_state *bs)
{
u->usage = bs->fence.batch_id;
(void)p_atomic_cmpxchg(u, &bs->usage, NULL);
}
static inline void
zink_batch_usage_set(struct zink_batch_usage **u, struct zink_batch_state *bs)
{
*u = &bs->usage;
}
static inline bool
zink_batch_usage_matches(const struct zink_batch_usage *u, const struct zink_batch_state *bs)
{
return u->usage == bs->fence.batch_id;
return u == &bs->usage;
}
static inline bool
zink_batch_usage_exists(struct zink_batch_usage *u)
zink_batch_usage_exists(const struct zink_batch_usage *u)
{
uint32_t usage = p_atomic_read(&u->usage);
return !!usage;
return u && (u->usage || u->unflushed);
}
bool
zink_batch_usage_check_completion(struct zink_context *ctx, const struct zink_batch_usage *u);
void
zink_batch_usage_wait(struct zink_context *ctx, const struct zink_batch_usage *u);
zink_batch_usage_wait(struct zink_context *ctx, struct zink_batch_usage *u);
#endif

75
src/gallium/drivers/zink/zink_context.c
View file

@ -56,15 +56,6 @@
#define XXH_INLINE_ALL
#include "util/xxhash.h"
static void
incr_curr_batch(struct zink_context *ctx)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
ctx->curr_batch = p_atomic_inc_return(&screen->curr_batch);
if (!ctx->curr_batch) //never use batchid 0
incr_curr_batch(ctx);
}
static void
calc_descriptor_hash_sampler_state(struct zink_sampler_state *sampler_state)
{
@ -1804,8 +1795,6 @@ flush_batch(struct zink_context *ctx, bool sync)
if (ctx->batch.state->is_device_lost) {
check_device_lost(ctx);
} else {
incr_curr_batch(ctx);
zink_start_batch(ctx, batch);
if (zink_screen(ctx->base.screen)->info.have_EXT_transform_feedback && ctx->num_so_targets)
ctx->dirty_so_targets = true;
@ -2539,10 +2528,8 @@ zink_wait_on_batch(struct zink_context *ctx, uint32_t batch_id)
bool
zink_check_batch_completion(struct zink_context *ctx, uint32_t batch_id)
{
assert(batch_id);
struct zink_batch_state *bs = ctx->batch.state;
assert(bs);
if (bs->fence.batch_id == batch_id)
assert(ctx->batch.state);
if (!batch_id)
/* not submitted yet */
return false;
@ -3161,8 +3148,8 @@ zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsi
struct zink_screen *screen = zink_screen(pctx->screen);
/* if any current usage exists, flush the queue */
if (res->obj->reads.usage == ctx->curr_batch ||
res->obj->writes.usage == ctx->curr_batch)
if (zink_batch_usage_is_unflushed(res->obj->reads) ||
zink_batch_usage_is_unflushed(res->obj->writes))
zink_flush_queue(ctx);
VkBindSparseInfo sparse;
@ -3264,45 +3251,38 @@ zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resou
zink_resource_rebind(zink_context(pctx), d);
}
ALWAYS_INLINE static bool
is_usage_completed(struct zink_screen *screen, const struct zink_batch_usage *u)
{
if (!zink_batch_usage_exists(u))
return true;
if (zink_batch_usage_is_unflushed(u))
return false;
/* check fastpath first */
if (zink_screen_check_last_finished(screen, u->usage))
return true;
/* if we have timelines, do a quick check */
if (screen->info.have_KHR_timeline_semaphore)
return zink_screen_timeline_wait(screen, u->usage, 0);
/* otherwise assume busy */
return false;
}
static bool
zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage)
{
struct zink_screen *screen = zink_screen(pscreen);
struct zink_resource *res = zink_resource(pres);
uint32_t reads = 0, writes = 0;
const struct zink_batch_usage *reads = NULL, *writes = NULL;
if (((usage & (PIPE_MAP_READ | PIPE_MAP_WRITE)) == (PIPE_MAP_READ | PIPE_MAP_WRITE)) ||
usage & PIPE_MAP_WRITE) {
reads = p_atomic_read(&res->obj->reads.usage);
writes = p_atomic_read(&res->obj->writes.usage);
reads = res->obj->reads;
writes = res->obj->writes;
} else if (usage & PIPE_MAP_READ)
writes = p_atomic_read(&res->obj->writes.usage);
writes = res->obj->writes;
/* get latest usage accounting for 32bit int rollover:
* a rollover is detected if there are reads and writes,
* but one of the values is over UINT32_MAX/2 while the other is under,
* as it is impossible for this many unflushed batch states to ever
* exist at any given time
*/
uint32_t last;
if (reads && writes)
last = abs((int)reads - (int)writes) > UINT32_MAX / 2 ? MIN2(reads, writes) : MAX2(reads, writes);
else
last = reads ? reads : writes;
if (!last)
return false;
/* check fastpath first */
if (zink_screen_check_last_finished(screen, last))
return false;
/* if we have timelines, do a quick check */
if (screen->info.have_KHR_timeline_semaphore)
return !zink_screen_timeline_wait(screen, last, 0);
/* otherwise assume busy */
return true;
return !is_usage_completed(screen, reads) || !is_usage_completed(screen, writes);
}
static void
@ -3484,7 +3464,6 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
ctx->have_timelines = screen->info.have_KHR_timeline_semaphore;
simple_mtx_init(&ctx->batch_mtx, mtx_plain);
incr_curr_batch(ctx);
zink_start_batch(ctx, &ctx->batch);
if (!ctx->batch.state)
goto fail;

4
src/gallium/drivers/zink/zink_context.h
View file

@ -69,7 +69,7 @@ struct zink_sampler_state {
VkSampler sampler;
uint32_t hash;
struct zink_descriptor_refs desc_set_refs;
struct zink_batch_usage batch_uses;
struct zink_batch_usage *batch_uses;
bool custom_border_color;
};
@ -78,7 +78,7 @@ struct zink_buffer_view {
VkBufferViewCreateInfo bvci;
VkBufferView buffer_view;
uint32_t hash;
struct zink_batch_usage batch_uses;
struct zink_batch_usage *batch_uses;
struct zink_descriptor_refs desc_set_refs;
};

8
src/gallium/drivers/zink/zink_descriptors.c
View file

@ -60,7 +60,7 @@ struct zink_descriptor_set {
bool punted;
bool recycled;
struct zink_descriptor_state_key key;
struct zink_batch_usage batch_uses;
struct zink_batch_usage *batch_uses;
#ifndef NDEBUG
/* for extra debug asserts */
unsigned num_resources;
@ -622,7 +622,7 @@ allocate_desc_set(struct zink_context *ctx, struct zink_program *pg, enum zink_d
pipe_reference_init(&zds->reference, 1);
zds->pool = pool;
zds->hash = 0;
zds->batch_uses.usage = 0;
zds->batch_uses = NULL;
zds->invalid = true;
zds->punted = zds->recycled = false;
#ifndef NDEBUG
@ -731,7 +731,7 @@ zink_descriptor_set_get(struct zink_context *ctx,
zds->recycled = false;
}
if (zds->invalid) {
if (zink_batch_usage_exists(&zds->batch_uses))
if (zink_batch_usage_exists(zds->batch_uses))
punt_invalid_set(zds, NULL);
else
/* this set is guaranteed to be in pool->alloc_desc_sets */
@ -746,7 +746,7 @@ zink_descriptor_set_get(struct zink_context *ctx,
bool recycled = false, punted = false;
if (he) {
zds = (void*)he->data;
if (zds->invalid && zink_batch_usage_exists(&zds->batch_uses)) {
if (zds->invalid && zink_batch_usage_exists(zds->batch_uses)) {
punt_invalid_set(zds, he);
zds = NULL;
punted = true;

3
src/gallium/drivers/zink/zink_fence.c
View file

@ -158,8 +158,9 @@ zink_vkfence_wait(struct zink_screen *screen, struct zink_fence *fence, uint64_t
if (success) {
p_atomic_set(&fence->completed, true);
zink_fence_clear_resources(screen, fence);
zink_batch_state(fence)->usage.usage = 0;
zink_screen_update_last_finished(screen, fence->batch_id);
zink_fence_clear_resources(screen, fence);
}
return success;
}

2
src/gallium/drivers/zink/zink_program.h
View file

@ -73,7 +73,7 @@ struct zink_shader_cache {
struct zink_program {
struct pipe_reference reference;
struct zink_batch_usage batch_uses;
struct zink_batch_usage *batch_uses;
bool is_compute;
struct zink_program_descriptor_data *dd;

4
src/gallium/drivers/zink/zink_query.c
View file

@ -46,7 +46,7 @@ struct zink_query {
bool have_gs[NUM_QUERIES]; /* geometry shaders use GEOMETRY_SHADER_PRIMITIVES_BIT */
bool have_xfb[NUM_QUERIES]; /* xfb was active during this query */
struct zink_batch_usage batch_id; //batch that the query was started in
struct zink_batch_usage *batch_id; //batch that the query was started in
struct list_head buffers;
struct zink_query_buffer *curr_qbo;
@ -780,7 +780,7 @@ zink_get_query_result(struct pipe_context *pctx,
if (query->needs_update)
update_qbo(ctx, query);
if (query->batch_id.usage == ctx->curr_batch) {
if (zink_batch_usage_is_unflushed(query->batch_id)) {
if (!threaded_query(q)->flushed)
pctx->flush(pctx, NULL, 0);
if (!wait)

14
src/gallium/drivers/zink/zink_resource.c
View file

@ -77,8 +77,8 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object
static uint32_t
get_resource_usage(struct zink_resource *res)
{
bool reads = zink_batch_usage_exists(&res->obj->reads);
bool writes = zink_batch_usage_exists(&res->obj->writes);
bool reads = zink_batch_usage_exists(res->obj->reads);
bool writes = zink_batch_usage_exists(res->obj->writes);
uint32_t batch_uses = 0;
if (reads)
batch_uses |= ZINK_RESOURCE_ACCESS_READ;
@ -867,8 +867,8 @@ buffer_transfer_map(struct zink_context *ctx, struct zink_resource *res, unsigne
*/
if (!res->obj->host_visible ||
!zink_batch_usage_check_completion(ctx, &res->obj->reads) ||
!zink_batch_usage_check_completion(ctx, &res->obj->writes)) {
!zink_batch_usage_check_completion(ctx, res->obj->reads) ||
!zink_batch_usage_check_completion(ctx, res->obj->writes)) {
/* Do a wait-free write-only transfer using a temporary buffer. */
unsigned offset;
@ -896,7 +896,7 @@ buffer_transfer_map(struct zink_context *ctx, struct zink_resource *res, unsigne
/* sparse/device-local will always need to wait since it has to copy */
if (!res->obj->host_visible)
return NULL;
if (!zink_batch_usage_check_completion(ctx, &res->obj->writes))
if (!zink_batch_usage_check_completion(ctx, res->obj->writes))
return NULL;
} else if (!res->obj->host_visible) {
zink_fence_wait(&ctx->base);
@ -909,7 +909,7 @@ buffer_transfer_map(struct zink_context *ctx, struct zink_resource *res, unsigne
res = staging_res;
zink_fence_wait(&ctx->base);
} else
zink_batch_usage_wait(ctx, &res->obj->writes);
zink_batch_usage_wait(ctx, res->obj->writes);
}
if (!ptr) {
@ -1036,7 +1036,7 @@ zink_transfer_map(struct pipe_context *pctx,
if (usage & PIPE_MAP_WRITE)
zink_fence_wait(pctx);
else
zink_batch_usage_wait(ctx, &res->obj->writes);
zink_batch_usage_wait(ctx, res->obj->writes);
}
VkImageSubresource isr = {
res->aspect,

4
src/gallium/drivers/zink/zink_resource.h
View file

@ -75,8 +75,8 @@ struct zink_resource_object {
unsigned persistent_maps; //if nonzero, requires vkFlushMappedMemoryRanges during batch use
struct zink_descriptor_refs desc_set_refs;
struct zink_batch_usage reads;
struct zink_batch_usage writes;
struct zink_batch_usage *reads;
struct zink_batch_usage *writes;
void *map;
bool is_buffer;
bool host_visible;

2
src/gallium/drivers/zink/zink_surface.h
View file

@ -37,7 +37,7 @@ struct zink_surface {
VkImageView simage_view;//old iview after storage replacement/rebind
void *obj; //backing resource object
uint32_t hash;
struct zink_batch_usage batch_uses;
struct zink_batch_usage *batch_uses;
struct util_dynarray framebuffer_refs;
struct zink_descriptor_refs desc_set_refs;
};