fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2026-02-20 23:10:30 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

zink: rewrite queue dispatch to use monotonic batch ids instead of hardcoded ones

Browse source 
historically zink has been bound to 4 gfx batches and then a separate compute batch
was added. this is not ideal for a number of reasons, the primary one being that if
an application performs 5 glFlush commands, the fifth one will force a gpu stall

this patch aims to do the following, all of which are necessarily done in the same patch
because they can't be added incrementally and still have the same function:
* rewrite batch tracking for resources/views/queries/descriptors/...
  |originally this was done with a single uint32_t as a bitmask, but that becomes cumbersome
   to track as batch counts increase, not to mention it becomes doubly-annoying
   when factoring in separate compute batches with their own ids. zink_batch_usage gives
   us separate tracking for gfx and compute batches along with a standardized api for
   managing usage
* flatten batch objects to a gfx batch and a compute batch
  |these are separate queues, so we can use an enum to choose between an array[2] of
   all batch-related objects
* switch to monotonic batch ids with batch "states"
  |with the flattened queues, we can just use monotonic uints to represent batch ids,
   thus freeing us from constantly using bitfield operations here and also enabling
   batch counts to scale dynamically by allocating/caching "states" that represent a batch
   for a given queue

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9547>
This commit is contained in:
Mike Blumenkrantz 2020-11-05 12:39:50 -05:00 • committed by Marge Bot
parent a06958278f
commit 60ea60ec4d
14 changed files with 373 additions and 263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

151
src/gallium/drivers/zink/zink_batch.c
View file

@ -16,12 +16,20 @@
#include "wsi_common.h"
static void
batch_usage_unset(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id)
{
p_atomic_cmpxchg(&u->usage[queue], batch_id, 0);
}
void
zink_batch_state_clear_resources(struct zink_screen *screen, struct zink_batch_state *bs)
{
/* unref all used resources */
set_foreach(bs->resources, entry) {
struct zink_resource_object *obj = (struct zink_resource_object *)entry->key;
batch_usage_unset(&obj->reads, !!bs->is_compute, bs->batch_id);
batch_usage_unset(&obj->writes, !!bs->is_compute, bs->batch_id);
zink_resource_object_reference(screen, &obj, NULL);
_mesa_set_remove(bs->resources, entry);
}
@ -31,8 +39,6 @@ void
zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
if (bs->fence->submitted)
zink_fence_finish(screen, &ctx->base, bs->fence, PIPE_TIMEOUT_INFINITE);
zink_batch_state_clear_resources(screen, bs);
@ -44,13 +50,13 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
set_foreach(bs->surfaces, entry) {
struct zink_surface *surf = (struct zink_surface *)entry->key;
surf->batch_uses &= ~BITFIELD64_BIT(bs->batch_id);
batch_usage_unset(&surf->batch_uses, !!bs->is_compute, bs->batch_id);
pipe_surface_reference((struct pipe_surface**)&surf, NULL);
_mesa_set_remove(bs->surfaces, entry);
}
set_foreach(bs->bufferviews, entry) {
struct zink_buffer_view *buffer_view = (struct zink_buffer_view *)entry->key;
buffer_view->batch_uses &= ~BITFIELD64_BIT(bs->batch_id);
batch_usage_unset(&buffer_view->batch_uses, !!bs->is_compute, bs->batch_id);
zink_buffer_view_reference(screen, &buffer_view, NULL);
_mesa_set_remove(bs->bufferviews, entry);
}
@ -63,7 +69,7 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
set_foreach(bs->desc_sets, entry) {
struct zink_descriptor_set *zds = (void*)entry->key;
zds->batch_uses &= ~BITFIELD_BIT(bs->batch_id);
batch_usage_unset(&zds->batch_uses, !!bs->is_compute, bs->batch_id);
/* reset descriptor pools when no bs is using this program to avoid
* having some inactive program hogging a billion descriptors
*/
@ -96,10 +102,21 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
bs->flush_res = NULL;
bs->descs_used = 0;
bs->fence->submitted = false;
ctx->resource_size[bs->is_compute] -= bs->resource_size;
bs->resource_size = 0;
}
void
zink_batch_reset_all(struct zink_context *ctx, enum zink_queue queue)
{
hash_table_foreach(&ctx->batch_states[queue], entry) {
struct zink_batch_state *bs = entry->data;
zink_reset_batch_state(ctx, bs);
_mesa_hash_table_remove(&ctx->batch_states[queue], entry);
util_dynarray_append(&ctx->free_batch_states[queue], struct zink_batch_state *, bs);
}
}
void
zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs)
{
@ -123,7 +140,7 @@ zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs
}
static struct zink_batch_state *
create_batch_state(struct zink_context *ctx, unsigned idx)
create_batch_state(struct zink_context *ctx, enum zink_queue queue)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_batch_state *bs = rzalloc(NULL, struct zink_batch_state);
@ -157,13 +174,12 @@ create_batch_state(struct zink_context *ctx, unsigned idx)
SET_CREATE_OR_FAIL(bs->active_queries);
util_dynarray_init(&bs->zombie_samplers, NULL);
util_dynarray_init(&bs->persistent_resources, NULL);
bs->batch_id = idx;
if (!zink_create_fence(screen, bs))
/* this destroys the batch state on failure */
return NULL;
bs->is_compute = idx == ZINK_COMPUTE_BATCH_ID;
bs->is_compute = queue == ZINK_QUEUE_COMPUTE;
return bs;
fail:
@ -171,10 +187,40 @@ fail:
return NULL;
}
static bool
find_unused_state(struct hash_entry *entry)
{
struct zink_fence *fence = entry->data;
/* we can't reset these from fence_finish because threads */
bool submitted = p_atomic_read(&fence->submitted);
return !submitted;
}
static void
init_batch_state(struct zink_context *ctx, struct zink_batch *batch)
{
struct zink_batch_state *bs = create_batch_state(ctx, batch->batch_id);
struct zink_batch_state *bs = NULL;
if (util_dynarray_num_elements(&ctx->free_batch_states[batch->queue], struct zink_batch_state*))
bs = util_dynarray_pop(&ctx->free_batch_states[batch->queue], struct zink_batch_state*);
if (!bs) {
struct hash_entry *he = _mesa_hash_table_random_entry(&ctx->batch_states[batch->queue], find_unused_state);
if (he) { //there may not be any entries available
bs = he->data;
_mesa_hash_table_remove(&ctx->batch_states[batch->queue], he);
zink_reset_batch_state(ctx, bs);
}
}
if (!bs) {
if (!batch->state) {
/* this is batch init, so create a few more states for later use */
for (int i = 0; i < 3; i++) {
struct zink_batch_state *state = create_batch_state(ctx, batch->queue);
util_dynarray_append(&ctx->free_batch_states[batch->queue], struct zink_batch_state *, state);
}
}
bs = create_batch_state(ctx, batch->queue);
}
batch->state = bs;
}
@ -205,6 +251,11 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch)
if (vkBeginCommandBuffer(batch->state->cmdbuf, &cbbi) != VK_SUCCESS)
debug_printf("vkBeginCommandBuffer failed\n");
batch->state->batch_id = ctx->curr_batch;
if (ctx->last_fence[batch->queue]) {
struct zink_batch_state *last_state = zink_batch_state(ctx->last_fence[batch->queue]);
batch->last_batch_id = last_state->batch_id;
}
if (!ctx->queries_disabled)
zink_resume_queries(ctx, batch);
}
@ -255,7 +306,7 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
si.pNext = &mem_signal;
}
if (vkQueueSubmit(ctx->queue, 1, &si, batch->state->fence->fence) != VK_SUCCESS) {
if (vkQueueSubmit(ctx->queue, 1, &si, batch->state->fence.fence) != VK_SUCCESS) {
debug_printf("ZINK: vkQueueSubmit() failed\n");
ctx->is_device_lost = true;
@ -263,7 +314,10 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET);
}
}
batch->state->fence->submitted = true;
ctx->last_fence[batch->queue] = &batch->state->fence;
_mesa_hash_table_insert_pre_hashed(&ctx->batch_states[batch->queue], batch->state->batch_id, (void*)(uintptr_t)batch->state->batch_id, batch->state);
ctx->resource_size[batch->queue] += batch->state->resource_size;
}
/* returns a queue based on whether a resource
@ -272,7 +326,6 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
enum zink_queue
zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource *res, bool write)
{
unsigned mask = write ? ZINK_RESOURCE_ACCESS_WRITE : ZINK_RESOURCE_ACCESS_READ;
enum zink_queue batch_to_flush = 0;
/* u_transfer_helper unrefs the stencil buffer when the depth buffer is unrefed,
@ -282,7 +335,7 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource
zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil);
if (batch->batch_id == ZINK_COMPUTE_BATCH_ID) {
if (batch->queue == ZINK_QUEUE_COMPUTE) {
if ((write && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_GFX)) ||
(!write && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_WRITE, ZINK_QUEUE_GFX)))
batch_to_flush = ZINK_QUEUE_GFX;
@ -293,43 +346,50 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource
}
/* if the resource already has usage of any sort set for this batch, we can skip hashing */
if (!zink_resource_has_usage_for_id(res, batch->state->batch_id)) {
if (!zink_batch_usage_matches(&res->obj->reads, batch->queue, batch->state->batch_id) &&
!zink_batch_usage_matches(&res->obj->writes, batch->queue, batch->state->batch_id)) {
bool found = false;
_mesa_set_search_and_add(batch->state->resources, res->obj, &found);
if (!found) {
pipe_reference(NULL, &res->obj->reference);
batch->state->resource_size += res->obj->size;
if (!batch->last_batch_id || !zink_batch_usage_matches(&res->obj->reads, batch->queue, batch->last_batch_id))
/* only add resource usage if it's "new" usage, though this only checks the most recent usage
* and not all pending usages
*/
batch->state->resource_size += res->obj->size;
if (stencil) {
pipe_reference(NULL, &stencil->obj->reference);
batch->state->resource_size += stencil->obj->size;
if (!batch->last_batch_id || !zink_batch_usage_matches(&stencil->obj->reads, batch->queue, batch->last_batch_id))
batch->state->resource_size += stencil->obj->size;
}
}
}
if (write) {
if (stencil)
zink_batch_usage_set(&stencil->obj->writes, batch->queue, batch->state->batch_id);
zink_batch_usage_set(&res->obj->writes, batch->queue, batch->state->batch_id);
} else {
if (stencil)
zink_batch_usage_set(&stencil->obj->reads, batch->queue, batch->state->batch_id);
zink_batch_usage_set(&res->obj->reads, batch->queue, batch->state->batch_id);
}
/* multiple array entries are fine */
if (res->obj->persistent_maps)
util_dynarray_append(&batch->state->persistent_resources, struct zink_resource*, res);
/* the batch_uses value for this batch is guaranteed to not be in use now because
* zink_reset_batch() waits on the fence and removes access before resetting
*/
res->obj->batch_uses[batch->batch_id] |= mask;
if (stencil)
stencil->obj->batch_uses[batch->batch_id] |= mask;
batch->has_work = true;
return batch_to_flush;
}
static bool
ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, uint32_t *u)
ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u)
{
bool found = false;
uint32_t bit = BITFIELD_BIT(batch->state->batch_id);
if ((*u) & bit)
if (zink_batch_usage_matches(u, batch->queue, batch->state->batch_id))
return false;
_mesa_set_search_and_add(s, ptr, &found);
assert(!found);
*u |= bit;
zink_batch_usage_set(u, batch->queue, batch->state->batch_id);
return true;
}
@ -394,3 +454,38 @@ zink_batch_reference_image_view(struct zink_batch *batch,
}
batch->has_work = true;
}
void
zink_batch_usage_set(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id)
{
if (queue == ZINK_QUEUE_ANY) {
p_atomic_set(&u->usage[ZINK_QUEUE_GFX], batch_id);
p_atomic_set(&u->usage[ZINK_QUEUE_COMPUTE], batch_id);
} else
p_atomic_set(&u->usage[queue], batch_id);
}
bool
zink_batch_usage_matches(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id)
{
if (queue < ZINK_QUEUE_ANY) {
uint32_t usage = p_atomic_read(&u->usage[queue]);
return usage == batch_id;
}
for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) {
uint32_t usage = p_atomic_read(&u->usage[queue]);
if (usage == batch_id)
return true;
}
return false;
}
bool
zink_batch_usage_exists(struct zink_batch_usage *u)
{
uint32_t usage = p_atomic_read(&u->usage[ZINK_QUEUE_GFX]);
if (usage)
return true;
usage = p_atomic_read(&u->usage[ZINK_QUEUE_COMPUTE]);
return !!usage;
}

31
src/gallium/drivers/zink/zink_batch.h
View file

@ -29,18 +29,18 @@
#include "util/list.h"
#include "util/u_dynarray.h"
#include "zink_fence.h"
struct pipe_reference;
struct zink_context;
struct zink_descriptor_set;
struct zink_fence;
struct zink_framebuffer;
struct zink_image_view;
struct zink_program;
struct zink_render_pass;
struct zink_resource;
struct zink_sampler_view;
struct zink_screen;
struct zink_surface;
enum zink_queue {
@ -49,15 +49,19 @@ enum zink_queue {
ZINK_QUEUE_ANY,
};
struct zink_batch_usage {
/* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */
uint32_t usage[2]; //gfx, compute
};
struct zink_batch_state {
unsigned batch_id : 3;
struct zink_fence fence;
VkCommandPool cmdpool;
VkCommandBuffer cmdbuf;
struct zink_resource *flush_res;
unsigned short descs_used; //number of descriptors currently allocated
struct zink_fence *fence;
struct set *fbs;
struct set *programs;
@ -74,22 +78,33 @@ struct zink_batch_state {
VkDeviceSize resource_size;
uint32_t batch_id;
bool is_compute;
};
struct zink_batch {
unsigned batch_id : 3;
struct zink_batch_state *state;
enum zink_queue queue;
uint32_t last_batch_id;
bool has_work;
bool in_rp; //renderpass is currently active
};
static inline struct zink_batch_state *
zink_batch_state(struct zink_fence *fence)
{
return (struct zink_batch_state *)fence;
}
void
zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs);
void
zink_batch_reset_all(struct zink_context *ctx, enum zink_queue queue);
void
zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs);
@ -127,4 +142,10 @@ zink_batch_reference_image_view(struct zink_batch *batch,
bool
zink_batch_add_desc_set(struct zink_batch *batch, struct zink_descriptor_set *zds);
void
zink_batch_usage_set(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id);
bool
zink_batch_usage_matches(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id);
bool
zink_batch_usage_exists(struct zink_batch_usage *u);
#endif

8
src/gallium/drivers/zink/zink_clear.c
View file

@ -192,7 +192,7 @@ zink_clear(struct pipe_context *pctx,
{
struct zink_context *ctx = zink_context(pctx);
struct pipe_framebuffer_state *fb = &ctx->fb_state;
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
bool needs_rp = false;
if (scissor_state) {
@ -363,7 +363,7 @@ zink_clear_texture(struct pipe_context *pctx,
struct pipe_screen *pscreen = pctx->screen;
struct u_rect region = zink_rect_from_box(box);
bool needs_rp = !zink_blit_region_fills(region, pres->width0, pres->height0) || ctx->render_condition_active;
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
struct pipe_surface *surf = NULL;
if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
@ -431,7 +431,7 @@ fb_clears_apply_internal(struct zink_context *ctx, struct pipe_resource *pres, i
if (!fb_clear->enabled)
return;
if (zink_resource(pres)->aspect == VK_IMAGE_ASPECT_COLOR_BIT) {
assert(!zink_curr_batch(ctx)->in_rp);
assert(!zink_batch_g(ctx)->in_rp);
if (zink_fb_clear_needs_explicit(fb_clear) || !check_3d_layers(ctx->fb_state.cbufs[i]))
/* this will automatically trigger all the clears */
zink_batch_rp(ctx);
@ -455,7 +455,7 @@ fb_clears_apply_internal(struct zink_context *ctx, struct pipe_resource *pres, i
zink_fb_clear_reset(&ctx->fb_clears[i]);
return;
} else {
assert(!zink_curr_batch(ctx)->in_rp);
assert(!zink_batch_g(ctx)->in_rp);
if (zink_fb_clear_needs_explicit(fb_clear) || !check_3d_layers(ctx->fb_state.zsbuf))
/* this will automatically trigger all the clears */
zink_batch_rp(ctx);

170
src/gallium/drivers/zink/zink_context.c
View file

@ -54,6 +54,14 @@
#define XXH_INLINE_ALL
#include "util/xxhash.h"
static void
incr_curr_batch(struct zink_context *ctx)
{
ctx->curr_batch++;
if (!ctx->curr_batch)
ctx->curr_batch = 1;
}
static struct zink_resource *
get_resource_for_descriptor(struct zink_context *ctx, enum zink_descriptor_type type, enum pipe_shader_type shader, int idx)
{
@ -281,7 +289,7 @@ zink_context_destroy(struct pipe_context *pctx)
struct zink_context *ctx = zink_context(pctx);
struct zink_screen *screen = zink_screen(pctx->screen);
if (vkQueueWaitIdle(ctx->queue) != VK_SUCCESS)
if (ctx->queue && vkQueueWaitIdle(ctx->queue) != VK_SUCCESS)
debug_printf("vkQueueWaitIdle failed\n");
util_blitter_destroy(ctx->blitter);
@ -293,12 +301,21 @@ zink_context_destroy(struct pipe_context *pctx)
for (unsigned i = 0; i < ARRAY_SIZE(ctx->null_buffers); i++)
pipe_resource_reference(&ctx->null_buffers[i], NULL);
for (int i = 0; i < ARRAY_SIZE(ctx->batches); ++i) {
for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) {
struct zink_fence *fence = zink_fence(&ctx->batches[i].state);
zink_reset_batch_state(ctx, ctx->batches[i].state);
zink_batch_state_destroy(screen, ctx->batches[i].state);
zink_fence_reference(zink_screen(pctx->screen), &fence, NULL);
hash_table_foreach(&ctx->batch_states[i], entry) {
fence = entry->data;
zink_reset_batch_state(ctx, entry->data);
zink_fence_reference(zink_screen(pctx->screen), &fence, NULL);
}
util_dynarray_foreach(&ctx->free_batch_states[i], struct zink_batch_state*, bs) {
fence = zink_fence(*bs);
zink_reset_batch_state(ctx, *bs);
zink_fence_reference(zink_screen(pctx->screen), &fence, NULL);
}
}
zink_reset_batch_state(ctx, ctx->compute_batch.state);
zink_batch_state_destroy(screen, ctx->compute_batch.state);
hash_table_foreach(ctx->render_pass_cache, he)
zink_destroy_render_pass(screen, he->data);
@ -495,7 +512,7 @@ zink_delete_sampler_state(struct pipe_context *pctx,
void *sampler_state)
{
struct zink_sampler_state *sampler = sampler_state;
struct zink_batch *batch = zink_curr_batch(zink_context(pctx));
struct zink_batch *batch = zink_batch_g(zink_context(pctx));
zink_descriptor_set_refs_clear(&sampler->desc_set_refs, sampler_state);
util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
sampler->sampler);
@ -1179,7 +1196,7 @@ setup_framebuffer(struct zink_context *ctx)
void
zink_begin_render_pass(struct zink_context *ctx, struct zink_batch *batch)
{
assert(batch == zink_curr_batch(ctx));
assert(batch == zink_batch_g(ctx));
setup_framebuffer(ctx);
assert(ctx->gfx_pipeline_state.render_pass);
@ -1268,23 +1285,22 @@ zink_end_render_pass(struct zink_context *ctx, struct zink_batch *batch)
}
static void
flush_batch(struct zink_context *ctx)
flush_batch(struct zink_context *ctx, enum zink_queue queue)
{
struct zink_batch *batch = zink_curr_batch(ctx);
zink_end_render_pass(ctx, batch);
struct zink_batch *batch = zink_batch_queue(ctx, queue);
if (queue == ZINK_QUEUE_GFX)
zink_end_render_pass(ctx, batch);
zink_end_batch(ctx, batch);
ctx->curr_batch++;
if (ctx->curr_batch == ARRAY_SIZE(ctx->batches))
ctx->curr_batch = 0;
incr_curr_batch(ctx);
zink_start_batch(ctx, zink_curr_batch(ctx));
zink_start_batch(ctx, batch);
}
struct zink_batch *
zink_batch_rp(struct zink_context *ctx)
{
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
if (!batch->in_rp) {
zink_begin_render_pass(ctx, batch);
assert(ctx->framebuffer && ctx->framebuffer->rp);
@ -1295,7 +1311,7 @@ zink_batch_rp(struct zink_context *ctx)
struct zink_batch *
zink_batch_no_rp(struct zink_context *ctx)
{
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
zink_end_render_pass(ctx, batch);
assert(!batch->in_rp);
return batch;
@ -1304,19 +1320,14 @@ zink_batch_no_rp(struct zink_context *ctx)
void
zink_flush_compute(struct zink_context *ctx)
{
zink_end_batch(ctx, &ctx->compute_batch);
zink_start_batch(ctx, &ctx->compute_batch);
flush_batch(ctx, ZINK_QUEUE_COMPUTE);
}
struct zink_batch *
zink_flush_batch(struct zink_context *ctx, struct zink_batch *batch)
{
if (batch && batch->batch_id >= ZINK_COMPUTE_BATCH_ID) {
zink_flush_compute(ctx);
return &ctx->compute_batch;
}
flush_batch(ctx);
return zink_curr_batch(ctx);
flush_batch(ctx, batch->queue);
return batch;
}
static void
@ -1526,7 +1537,7 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_batch *batch,
/* only barrier if we're changing layout or doing something besides read -> read */
if (!batch) {
if (pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)
batch = &ctx->compute_batch;
batch = zink_batch_c(ctx);
else
batch = zink_batch_no_rp(ctx);
}
@ -1624,7 +1635,7 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_batch *batch,
/* only barrier if we're changing layout or doing something besides read -> read */
if (!batch) {
if (pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)
batch = &ctx->compute_batch;
batch = zink_batch_c(ctx);
else
batch = zink_batch_no_rp(ctx);
}
@ -1704,10 +1715,11 @@ zink_flush(struct pipe_context *pctx,
{
struct zink_context *ctx = zink_context(pctx);
bool deferred = flags & PIPE_FLUSH_DEFERRED;
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
struct zink_fence *fence = &batch->state->fence;
if (deferred)
batch->state->fence->deferred_ctx = pctx;
batch->state->fence.deferred_ctx = pctx;
else if (batch->has_work) {
if (flags & PIPE_FLUSH_END_OF_FRAME) {
if (ctx->fb_state.nr_cbufs)
@ -1719,7 +1731,7 @@ zink_flush(struct pipe_context *pctx,
if (zink_screen(pctx->screen)->needs_mesa_flush_wsi && ctx->fb_state.cbufs[0])
batch->state->flush_res = zink_resource(ctx->fb_state.cbufs[0]->texture);
}
flush_batch(ctx);
flush_batch(ctx, ZINK_QUEUE_GFX);
if (zink_screen(pctx->screen)->info.have_EXT_transform_feedback && ctx->num_so_targets)
ctx->dirty_so_targets = true;
@ -1728,56 +1740,81 @@ zink_flush(struct pipe_context *pctx,
if (!pfence)
return;
if (deferred && !batch->has_work) {
batch = zink_prev_batch(ctx);
fence = ctx->last_fence[ZINK_QUEUE_GFX];
}
zink_fence_reference(zink_screen(pctx->screen),
(struct zink_fence **)pfence,
batch->state->fence);
fence);
if (flags & PIPE_FLUSH_END_OF_FRAME) {
/* if the first frame has not yet occurred, we need an explicit fence here
* in some cases in order to correctly draw the first frame, though it's
* unknown at this time why this is the case
*/
if (!ctx->first_frame_done)
zink_fence_finish(zink_screen(pctx->screen), pctx, batch->state->fence, PIPE_TIMEOUT_INFINITE);
zink_fence_finish(zink_screen(pctx->screen), pctx, fence, PIPE_TIMEOUT_INFINITE);
ctx->first_frame_done = true;
}
}
void
zink_maybe_flush_or_stall(struct zink_context *ctx, enum zink_queue queue)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
/* flush anytime our total batch memory usage is potentially >= 1/10 of total system memory */
if (zink_batch_queue(ctx, queue)->state->resource_size >= screen->total_mem / 10)
flush_batch(ctx, queue);
if (ctx->resource_size[queue] >= screen->total_mem / 10) {
zink_fence_finish(zink_screen(ctx->base.screen), &ctx->base, ctx->last_fence[queue], PIPE_TIMEOUT_INFINITE);
zink_batch_reset_all(ctx, queue);
}
}
void
zink_fence_wait(struct pipe_context *pctx)
{
struct pipe_fence_handle *fence = NULL;
pctx->flush(pctx, &fence, PIPE_FLUSH_HINT_FINISH);
if (fence) {
pctx->screen->fence_finish(pctx->screen, NULL, fence,
PIPE_TIMEOUT_INFINITE);
pctx->screen->fence_reference(pctx->screen, &fence, NULL);
}
struct zink_context *ctx = zink_context(pctx);
if (zink_batch_g(ctx)->has_work)
pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
if (ctx->last_fence[ZINK_QUEUE_GFX])
zink_fence_finish(zink_screen(pctx->screen), pctx, ctx->last_fence[ZINK_QUEUE_GFX], PIPE_TIMEOUT_INFINITE);
}
void
zink_wait_on_batch(struct zink_context *ctx, int batch_id)
zink_wait_on_batch(struct zink_context *ctx, enum zink_queue queue, uint32_t batch_id)
{
if (batch_id >= 0) {
struct zink_batch *batch = batch_id == ZINK_COMPUTE_BATCH_ID ? &ctx->compute_batch : &ctx->batches[batch_id];
if (batch != zink_curr_batch(ctx)) {
if (!batch->state->fence->submitted) { // this is the compute batch
zink_flush_compute(ctx);
} else
ctx->base.screen->fence_finish(ctx->base.screen, NULL, (struct pipe_fence_handle*)batch->state->fence,
PIPE_TIMEOUT_INFINITE);
return;
struct zink_batch_state *bs = zink_batch_queue(ctx, queue)->state;
assert(bs);
if (!batch_id || bs->batch_id == batch_id)
/* not submitted yet */
flush_batch(ctx, queue);
struct zink_fence *fence;
assert(batch_id || ctx->last_fence[queue]);
if (ctx->last_fence[queue] && (!batch_id || batch_id == zink_batch_state(ctx->last_fence[queue])->batch_id))
fence = ctx->last_fence[queue];
else {
struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&ctx->batch_states[queue], batch_id, (void*)(uintptr_t)batch_id);
if (!he) {
util_dynarray_foreach(&ctx->free_batch_states[queue], struct zink_batch_state*, bs) {
if ((*bs)->batch_id == batch_id)
return;
}
unreachable("should've found batch state");
}
fence = he->data;
}
zink_fence_wait(&ctx->base);
assert(fence);
ctx->base.screen->fence_finish(ctx->base.screen, &ctx->base, (struct pipe_fence_handle*)fence, PIPE_TIMEOUT_INFINITE);
}
static void
zink_texture_barrier(struct pipe_context *pctx, unsigned flags)
{
struct zink_context *ctx = zink_context(pctx);
if (zink_curr_batch(ctx)->has_work)
if (zink_batch_g(ctx)->has_work)
pctx->flush(pctx, NULL, 0);
zink_flush_compute(ctx);
}
@ -1895,15 +1932,15 @@ zink_memory_barrier(struct pipe_context *pctx, unsigned flags)
b.srcAccessMask = sflags;
b.dstAccessMask = dflags;
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
if (batch->has_work) {
zink_end_render_pass(ctx, batch);
/* this should be the only call needed */
vkCmdPipelineBarrier(batch->state->cmdbuf, src, dst, 0, 0, &b, 0, NULL, 0, NULL);
flush_batch(ctx);
flush_batch(ctx, ZINK_QUEUE_GFX);
}
batch = &ctx->compute_batch;
batch = zink_batch_c(ctx);
if (batch->has_work) {
/* this should be the only call needed */
vkCmdPipelineBarrier(batch->state->cmdbuf, src, dst, 0, 0, &b, 0, NULL, 0, NULL);
@ -2227,13 +2264,12 @@ zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res)
}
}
static bool
init_batch(struct zink_context *ctx, struct zink_batch *batch, unsigned idx)
static void
init_batch(struct zink_context *ctx, enum zink_queue queue)
{
batch->queue = idx == ZINK_COMPUTE_BATCH_ID ? ZINK_QUEUE_COMPUTE : ZINK_QUEUE_GFX;
batch->batch_id = idx;
struct zink_batch *batch = zink_batch_queue(ctx, queue);
batch->queue = queue;
zink_start_batch(ctx, batch);
return !!batch->state;
}
struct pipe_context *
@ -2303,6 +2339,11 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
zink_context_resource_init(&ctx->base);
zink_context_query_init(&ctx->base);
for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) {
util_dynarray_init(&ctx->free_batch_states[i], ctx);
_mesa_hash_table_init(&ctx->batch_states[i], ctx, NULL, _mesa_key_pointer_equal);
}
ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state;
slab_create_child(&ctx->transfer_pool, &screen->transfer_pool);
@ -2328,12 +2369,13 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
if (!ctx->blitter)
goto fail;
for (int i = 0; i < ARRAY_SIZE(ctx->batches); ++i) {
if (!init_batch(ctx, &ctx->batches[i], i))
goto fail;
}
incr_curr_batch(ctx);
init_batch(ctx, ZINK_QUEUE_GFX);
if (!zink_batch_g(ctx)->state)
goto fail;
if (!init_batch(ctx, &ctx->compute_batch, ZINK_COMPUTE_BATCH_ID))
init_batch(ctx, ZINK_QUEUE_COMPUTE);
if (!zink_batch_c(ctx)->state)
goto fail;
vkGetDeviceQueue(screen->dev, screen->gfx_queue, 0, &ctx->queue);

46
src/gallium/drivers/zink/zink_context.h
View file

@ -25,10 +25,6 @@
#define ZINK_CONTEXT_H
#define ZINK_SHADER_COUNT (PIPE_SHADER_TYPES - 1)
#define ZINK_NUM_GFX_BATCHES 4
#define ZINK_COMPUTE_BATCH_ID ZINK_NUM_GFX_BATCHES
#define ZINK_COMPUTE_BATCH_COUNT 1
#define ZINK_NUM_BATCHES (ZINK_NUM_GFX_BATCHES + 1)
#define ZINK_DEFAULT_MAX_DESCS 5000
@ -45,6 +41,7 @@
#include "util/slab.h"
#include "util/list.h"
#include "util/u_dynarray.h"
#include <vulkan/vulkan.h>
@ -70,7 +67,7 @@ struct zink_sampler_state {
VkSampler sampler;
uint32_t hash;
struct zink_descriptor_refs desc_set_refs;
uint32_t batch_uses;
struct zink_batch_usage batch_uses;
bool custom_border_color;
};
@ -79,7 +76,7 @@ struct zink_buffer_view {
VkBufferViewCreateInfo bvci;
VkBufferView buffer_view;
uint32_t hash;
uint32_t batch_uses;
struct zink_batch_usage batch_uses;
};
struct zink_sampler_view {
@ -140,13 +137,15 @@ struct zink_context {
struct pipe_device_reset_callback reset;
struct zink_batch batches[ZINK_NUM_GFX_BATCHES];
bool is_device_lost;
unsigned curr_batch;
uint32_t curr_batch; //the current batch id
struct zink_batch batches[2]; //gfx, compute
struct zink_fence *last_fence[2]; //gfx, compute; the last command buffer submitted
VkQueue queue;
struct zink_batch compute_batch;
struct hash_table batch_states[2]; //gfx, compute; submitted batch states
struct util_dynarray free_batch_states[2]; //gfx, compute; unused batch states
VkDeviceSize resource_size[2]; //gfx, compute; the accumulated size of resources in submitted buffers
struct pipe_constant_buffer ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS];
struct pipe_shader_buffer ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS];
@ -231,22 +230,22 @@ zink_context(struct pipe_context *context)
}
static inline struct zink_batch *
zink_curr_batch(struct zink_context *ctx)
zink_batch_queue(struct zink_context *ctx, enum zink_queue queue_type)
{
assert(ctx->curr_batch < ARRAY_SIZE(ctx->batches));
return ctx->batches + ctx->curr_batch;
assert(queue_type < ARRAY_SIZE(ctx->batches));
return &ctx->batches[queue_type];
}
static inline struct zink_batch *
zink_prev_batch(struct zink_context *ctx)
zink_batch_g(struct zink_context *ctx)
{
unsigned curr_batch = ctx->curr_batch;
if (!curr_batch)
curr_batch = ZINK_NUM_GFX_BATCHES - 1;
else
curr_batch--;
assert(curr_batch < ARRAY_SIZE(ctx->batches));
return ctx->batches + curr_batch;
return &ctx->batches[ZINK_QUEUE_GFX];
}
static inline struct zink_batch *
zink_batch_c(struct zink_context *ctx)
{
return &ctx->batches[ZINK_QUEUE_COMPUTE];
}
struct zink_batch *
@ -259,7 +258,7 @@ void
zink_fence_wait(struct pipe_context *ctx);
void
zink_wait_on_batch(struct zink_context *ctx, int batch_id);
zink_wait_on_batch(struct zink_context *ctx, enum zink_queue queue, uint32_t batch_id);
void
zink_flush_compute(struct zink_context *ctx);
@ -267,6 +266,9 @@ zink_flush_compute(struct zink_context *ctx);
struct zink_batch *
zink_flush_batch(struct zink_context *ctx, struct zink_batch *batch);
void
zink_maybe_flush_or_stall(struct zink_context *ctx, enum zink_queue queue);
bool
zink_resource_access_is_write(VkAccessFlags flags);

10
src/gallium/drivers/zink/zink_descriptors.c
View file

@ -232,7 +232,7 @@ allocate_desc_set(struct zink_screen *screen, struct zink_program *pg, enum zink
pipe_reference_init(&zds->reference, 1);
zds->pool = pool;
zds->hash = 0;
zds->batch_uses = 0;
zds->batch_uses.usage[0] = zds->batch_uses.usage[1] = 0;
zds->invalid = true;
zds->punted = zds->recycled = false;
if (num_resources) {
@ -296,7 +296,7 @@ zink_descriptor_set_get(struct zink_context *ctx,
struct zink_descriptor_set *zds;
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_program *pg = is_compute ? (struct zink_program *)ctx->curr_compute : (struct zink_program *)ctx->curr_program;
struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
struct zink_descriptor_pool *pool = pg->pool[type];
unsigned descs_used = 1;
assert(type < ZINK_DESCRIPTOR_TYPES);
@ -315,7 +315,7 @@ zink_descriptor_set_get(struct zink_context *ctx,
zds->recycled = false;
}
if (zds->invalid) {
if (zds->batch_uses)
if (zink_batch_usage_exists(&zds->batch_uses))
punt_invalid_set(zds, NULL);
else
/* this set is guaranteed to be in pool->alloc_desc_sets */
@ -332,7 +332,7 @@ zink_descriptor_set_get(struct zink_context *ctx,
bool recycled = false, punted = false;
if (he) {
zds = (void*)he->data;
if (zds->invalid && zds->batch_uses) {
if (zds->invalid && zink_batch_usage_exists(&zds->batch_uses)) {
punt_invalid_set(zds, he);
zds = NULL;
punted = true;
@ -375,7 +375,7 @@ skip_hash_tables:
}
if (pool->num_sets_allocated + pool->key.num_descriptors > ZINK_DEFAULT_MAX_DESCS) {
batch = zink_flush_batch(ctx, batch);
zink_fence_wait(&ctx->base);
zink_batch_reference_program(batch, pg);
return zink_descriptor_set_get(ctx, type, is_compute, cache_hit, need_resource_refs);
}

2
src/gallium/drivers/zink/zink_descriptors.h
View file

@ -96,7 +96,7 @@ struct zink_descriptor_set {
bool recycled;
struct zink_descriptor_state_key key;
struct util_dynarray barriers;
uint32_t batch_uses;
struct zink_batch_usage batch_uses;
#ifndef NDEBUG
/* for extra debug asserts */
unsigned num_resources;

28
src/gallium/drivers/zink/zink_draw.c
View file

@ -118,7 +118,7 @@ zink_emit_stream_output_targets(struct pipe_context *pctx)
{
struct zink_context *ctx = zink_context(pctx);
struct zink_screen *screen = zink_screen(pctx->screen);
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
VkBuffer buffers[PIPE_MAX_SO_OUTPUTS] = {};
VkDeviceSize buffer_offsets[PIPE_MAX_SO_OUTPUTS] = {};
VkDeviceSize buffer_sizes[PIPE_MAX_SO_OUTPUTS] = {};
@ -342,7 +342,7 @@ write_descriptors(struct zink_context *ctx, struct zink_descriptor_set *zds, uns
bool is_compute, bool cache_hit, bool need_resource_refs)
{
bool need_flush = false;
struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
struct zink_screen *screen = zink_screen(ctx->base.screen);
assert(zds->desc_set);
enum zink_queue check_flush_id = is_compute ? ZINK_QUEUE_GFX : ZINK_QUEUE_COMPUTE;
@ -644,12 +644,12 @@ update_sampler_descriptors(struct zink_context *ctx, struct zink_descriptor_set
desc_set_sampler_add(ctx, zds, sampler_view, sampler, num_resources++,
zink_shader_descriptor_is_buffer(shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j),
cache_hit);
struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
if (sampler_view)
zink_batch_reference_sampler_view(batch, sampler_view);
if (sampler)
/* this only tracks the most recent usage for now */
sampler->batch_uses = BITFIELD_BIT(batch->state->batch_id);
zink_batch_usage_set(&sampler->batch_uses, batch->queue, batch->state->batch_id);
}
assert(num_wds < num_descriptors);
@ -734,7 +734,7 @@ update_image_descriptors(struct zink_context *ctx, struct zink_descriptor_set *z
&num_buffer_info, &buffer_views[num_buffer_info],
NULL, imageview, bufferview, !k);
struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
if (res)
zink_batch_reference_image_view(batch, image_view);
}
@ -762,7 +762,7 @@ update_descriptors(struct zink_context *ctx, struct zink_screen *screen, bool is
else
zds[h] = NULL;
}
struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx);
struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx);
zink_batch_reference_program(batch, pg);
uint32_t dynamic_offsets[PIPE_MAX_CONSTANT_BUFFERS];
@ -863,11 +863,8 @@ zink_draw_vbo(struct pipe_context *pctx,
VkDeviceSize counter_buffer_offsets[PIPE_MAX_SO_OUTPUTS] = {};
bool need_index_buffer_unref = false;
/* flush anytime our total batch memory usage is potentially >= 1/10 of total gpu memory
* this should also eventually trigger a stall if the app is going nuts with gpu memory
*/
if (zink_curr_batch(ctx)->state->resource_size >= screen->total_mem / 10 / ZINK_NUM_BATCHES)
ctx->base.flush(&ctx->base, NULL, 0);
/* check memory usage and flush/stall as needed to avoid oom */
zink_maybe_flush_or_stall(ctx, ZINK_QUEUE_GFX);
if (dinfo->primitive_restart && !restart_supported(dinfo->mode)) {
util_draw_vbo_without_prim_restart(pctx, dinfo, dindirect, &draws[0]);
@ -1178,13 +1175,10 @@ zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info)
{
struct zink_context *ctx = zink_context(pctx);
struct zink_screen *screen = zink_screen(pctx->screen);
struct zink_batch *batch = &ctx->compute_batch;
struct zink_batch *batch = zink_batch_c(ctx);
/* flush anytime our total batch memory usage is potentially >= 1/10 of total gpu memory
* this should also eventually trigger a stall if the app is going nuts with gpu memory
*/
if (batch->state->resource_size >= screen->total_mem / 10 / ZINK_NUM_BATCHES)
zink_flush_compute(ctx);
/* check memory usage and flush/stall as needed to avoid oom */
zink_maybe_flush_or_stall(ctx, ZINK_QUEUE_COMPUTE);
struct zink_compute_program *comp_program = get_compute_program(ctx);
if (!comp_program)

53
src/gallium/drivers/zink/zink_fence.c
View file

@ -36,50 +36,34 @@ destroy_fence(struct zink_screen *screen, struct zink_fence *fence)
{
if (fence->fence)
vkDestroyFence(screen->dev, fence->fence, NULL);
util_dynarray_fini(&fence->resources);
FREE(fence);
zink_batch_state_destroy(screen, zink_batch_state(fence));
}
bool
zink_create_fence(struct zink_screen *screen, struct zink_batch_state *bs)
{
struct zink_fence *fence = zink_fence(bs);
VkFenceCreateInfo fci = {};
fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
struct zink_fence *ret = CALLOC_STRUCT(zink_fence);
if (!ret) {
debug_printf("CALLOC_STRUCT failed\n");
return false;
}
if (vkCreateFence(screen->dev, &fci, NULL, &ret->fence) != VK_SUCCESS) {
if (vkCreateFence(screen->dev, &fci, NULL, &fence->fence) != VK_SUCCESS) {
debug_printf("vkCreateFence failed\n");
goto fail;
}
ret->batch_id = bs->batch_id;
util_dynarray_init(&ret->resources, NULL);
pipe_reference_init(&ret->reference, 1);
bs->fence = ret;
pipe_reference_init(&fence->reference, 1);
return true;
fail:
destroy_fence(screen, ret);
destroy_fence(screen, fence);
return false;
}
void
zink_fence_init(struct zink_context *ctx, struct zink_batch *batch)
{
struct zink_fence *fence = batch->state->fence;
set_foreach(batch->state->resources, entry) {
/* the fence needs its own reference to ensure it can safely access lifetime-dependent
* resource members
*/
struct zink_resource_object *obj = (struct zink_resource_object *)entry->key;
pipe_reference(NULL, &obj->reference);
util_dynarray_append(&fence->resources, struct zink_resource_object*, obj);
}
struct zink_fence *fence = zink_fence(batch->state);
vkResetFences(zink_screen(ctx->base.screen)->dev, 1, &fence->fence);
fence->deferred_ctx = NULL;
fence->submitted = true;
@ -105,18 +89,12 @@ fence_reference(struct pipe_screen *pscreen,
zink_fence(pfence));
}
static inline void
fence_remove_resource_access(struct zink_fence *fence, struct zink_resource_object *obj)
{
p_atomic_set(&obj->batch_uses[fence->batch_id], 0);
}
bool
zink_fence_finish(struct zink_screen *screen, struct pipe_context *pctx, struct zink_fence *fence,
uint64_t timeout_ns)
{
if (pctx && fence->deferred_ctx == pctx) {
zink_curr_batch(zink_context(pctx))->has_work = true;
zink_batch_g(zink_context(pctx))->has_work = true;
/* this must be the current batch */
pctx->flush(pctx, NULL, 0);
}
@ -131,14 +109,9 @@ zink_fence_finish(struct zink_screen *screen, struct pipe_context *pctx, struct
success = vkGetFenceStatus(screen->dev, fence->fence) == VK_SUCCESS;
if (success) {
/* unref all used resources */
util_dynarray_foreach(&fence->resources, struct zink_resource_object*, obj) {
fence_remove_resource_access(fence, *obj);
zink_resource_object_reference(screen, obj, NULL);
}
util_dynarray_clear(&fence->resources);
fence->submitted = false;
struct zink_batch_state *bs = zink_batch_state(fence);
zink_batch_state_clear_resources(screen, bs);
p_atomic_set(&fence->submitted, false);
}
return success;
}
@ -160,7 +133,7 @@ zink_fence_server_sync(struct pipe_context *pctx, struct pipe_fence_handle *pfen
return;
if (fence->deferred_ctx) {
zink_curr_batch(zink_context(pctx))->has_work = true;
zink_batch_g(zink_context(pctx))->has_work = true;
/* this must be the current batch */
pctx->flush(pctx, NULL, 0);
}

6
src/gallium/drivers/zink/zink_fence.h
View file

@ -25,27 +25,25 @@
#define ZINK_FENCE_H
#include "util/u_inlines.h"
#include "util/u_dynarray.h"
#include <vulkan/vulkan.h>
struct pipe_context;
struct pipe_screen;
struct zink_batch;
struct zink_batch_state;
struct zink_context;
struct zink_screen;
struct zink_fence {
struct pipe_reference reference;
unsigned batch_id : 3;
VkFence fence;
struct util_dynarray resources;
struct pipe_context *deferred_ctx;
bool submitted;
};
static inline struct zink_fence *
zink_fence(struct pipe_fence_handle *pfence)
zink_fence(void *pfence)
{
return (struct zink_fence *)pfence;
}

25
src/gallium/drivers/zink/zink_query.c
View file

@ -37,7 +37,7 @@ struct zink_query {
bool have_gs[NUM_QUERIES]; /* geometry shaders use GEOMETRY_SHADER_PRIMITIVES_BIT */
bool have_xfb[NUM_QUERIES]; /* xfb was active during this query */
unsigned batch_id : 3; //batch that the query was started in
struct zink_batch_usage batch_id; //batch that the query was started in
union pipe_query_result accumulated_result;
};
@ -137,10 +137,10 @@ static struct zink_batch *
get_batch_for_query(struct zink_context *ctx, struct zink_query *query, bool no_rp)
{
if (query && is_cs_query(query))
return &ctx->compute_batch;
return zink_batch_c(ctx);
if (no_rp)
return zink_batch_no_rp(ctx);
return zink_curr_batch(ctx);
return zink_batch_g(ctx);
}
static struct pipe_query *
@ -406,7 +406,7 @@ force_cpu_read(struct zink_context *ctx, struct pipe_query *pquery, bool wait, e
unsigned result_size = result_type <= PIPE_QUERY_TYPE_U32 ? sizeof(uint32_t) : sizeof(uint64_t);
struct zink_query *query = (struct zink_query*)pquery;
union pipe_query_result result;
if (zink_curr_batch(ctx)->state->batch_id == query->batch_id)
if (zink_batch_usage_matches(&query->batch_id, ZINK_QUEUE_GFX, zink_batch_g(ctx)->state->batch_id))
pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
else if (is_cs_query(query))
zink_flush_compute(ctx);
@ -535,7 +535,7 @@ begin_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_quer
if (needs_stats_list(q))
list_addtail(&q->stats_list, &ctx->primitives_generated_queries);
p_atomic_inc(&q->fences);
q->batch_id = batch->state->batch_id;
zink_batch_usage_set(&q->batch_id, batch->queue, batch->state->batch_id);
_mesa_set_add(batch->state->active_queries, q);
}
@ -565,7 +565,7 @@ end_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query
if (is_time_query(q)) {
vkCmdWriteTimestamp(batch->state->cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
q->query_pool, q->curr_query);
q->batch_id = batch->state->batch_id;
zink_batch_usage_set(&q->batch_id, batch->queue, batch->state->batch_id);
} else if (q->type == PIPE_QUERY_PRIMITIVES_EMITTED ||
q->type == PIPE_QUERY_PRIMITIVES_GENERATED ||
q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE)
@ -613,9 +613,10 @@ zink_get_query_result(struct pipe_context *pctx,
struct zink_query *query = (void*)q;
struct zink_context *ctx = zink_context(pctx);
if (is_cs_query(query)) {
if (wait)
zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID);
else {
if (wait) {
uint32_t batch_id = p_atomic_read(&query->batch_id.usage[ZINK_QUEUE_COMPUTE]);
zink_wait_on_batch(ctx, ZINK_QUEUE_COMPUTE, batch_id);
} else {
zink_flush_compute(ctx);
}
} else {
@ -671,7 +672,7 @@ zink_set_active_query_state(struct pipe_context *pctx, bool enable)
struct zink_context *ctx = zink_context(pctx);
ctx->queries_disabled = !enable;
struct zink_batch *batch = zink_curr_batch(ctx);
struct zink_batch *batch = zink_batch_g(ctx);
if (ctx->queries_disabled)
zink_suspend_queries(ctx, batch);
else
@ -721,11 +722,11 @@ zink_render_condition(struct pipe_context *pctx,
if (query->type != PIPE_QUERY_PRIMITIVES_GENERATED &&
!is_so_overflow_query(query)) {
copy_results_to_buffer(ctx, query, res, 0, num_results, flags);
batch = zink_curr_batch(ctx);
batch = zink_batch_g(ctx);
} else {
/* these need special handling */
force_cpu_read(ctx, pquery, true, PIPE_QUERY_TYPE_U32, pres, 0);
batch = zink_curr_batch(ctx);
batch = zink_batch_g(ctx);
zink_batch_reference_resource_rw(batch, res, false);
}

94
src/gallium/drivers/zink/zink_resource.c
View file

@ -54,52 +54,48 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object
}
static uint32_t
get_resource_usage(struct zink_resource *res)
get_resource_usage(struct zink_resource *res, enum zink_queue queue)
{
assert(queue < 2);
uint32_t reads = p_atomic_read(&res->obj->reads.usage[queue]);
uint32_t writes = p_atomic_read(&res->obj->writes.usage[queue]);
uint32_t batch_uses = 0;
if (reads)
batch_uses |= ZINK_RESOURCE_ACCESS_READ << queue;
if (writes)
batch_uses |= ZINK_RESOURCE_ACCESS_WRITE << queue;
return batch_uses;
}
static uint32_t
get_all_resource_usage(struct zink_resource *res)
{
uint32_t batch_uses = 0;
for (unsigned i = 0; i < ARRAY_SIZE(res->obj->batch_uses); i++)
batch_uses |= p_atomic_read(&res->obj->batch_uses[i]) << i;
for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++)
batch_uses |= get_resource_usage(res, i);
return batch_uses;
}
static void
resource_sync_reads_from_compute(struct zink_context *ctx, struct zink_resource *res)
{
uint32_t reads = p_atomic_read(&res->obj->reads.usage[ZINK_QUEUE_COMPUTE]);
assert(reads);
zink_wait_on_batch(ctx, ZINK_QUEUE_COMPUTE, reads);
}
static void
resource_sync_writes_from_batch_usage(struct zink_context *ctx, struct zink_resource *res)
{
uint32_t batch_uses = get_resource_usage(res);
batch_uses &= ~(ZINK_RESOURCE_ACCESS_READ << ZINK_COMPUTE_BATCH_ID);
uint32_t writes[2];
for (int i = 0; i < ZINK_QUEUE_ANY; i++)
writes[i] = p_atomic_read(&res->obj->writes.usage[i]);
uint32_t write_mask = 0;
for (int i = 0; i < ZINK_NUM_GFX_BATCHES + ZINK_COMPUTE_BATCH_COUNT; i++)
write_mask |= ZINK_RESOURCE_ACCESS_WRITE << i;
while (batch_uses & write_mask) {
int batch_id = zink_get_resource_latest_batch_usage(ctx, batch_uses);
if (batch_id == -1)
break;
zink_wait_on_batch(ctx, batch_id);
batch_uses &= ~((ZINK_RESOURCE_ACCESS_RW) << batch_id);
}
}
int
zink_get_resource_latest_batch_usage(struct zink_context *ctx, uint32_t batch_uses)
{
unsigned cur_batch = zink_curr_batch(ctx)->batch_id;
if (batch_uses & ZINK_RESOURCE_ACCESS_WRITE << ZINK_COMPUTE_BATCH_ID)
return ZINK_COMPUTE_BATCH_ID;
batch_uses &= ~(ZINK_RESOURCE_ACCESS_WRITE << ZINK_COMPUTE_BATCH_ID);
if (!batch_uses)
return -1;
for (unsigned i = 0; i < ZINK_NUM_BATCHES + 1; i++) {
/* loop backwards and sync with highest batch id that has writes */
if (batch_uses & (ZINK_RESOURCE_ACCESS_WRITE << cur_batch)) {
return cur_batch;
}
cur_batch--;
if (cur_batch > ZINK_COMPUTE_BATCH_ID - 1) // underflowed past max batch id
cur_batch = ZINK_COMPUTE_BATCH_ID - 1;
}
return -1;
enum zink_queue queue = writes[0] < writes[1] ? ZINK_QUEUE_COMPUTE : ZINK_QUEUE_GFX;
/* sync lower id first */
if (writes[!queue])
zink_wait_on_batch(ctx, !queue, writes[!queue]);
zink_wait_on_batch(ctx, queue, writes[queue]);
}
static uint32_t
@ -603,7 +599,7 @@ zink_resource_invalidate(struct pipe_context *pctx, struct pipe_resource *pres)
res->bind_history &= ~ZINK_RESOURCE_USAGE_STREAMOUT;
util_range_set_empty(&res->valid_buffer_range);
if (!get_resource_usage(res))
if (!get_all_resource_usage(res))
return;
struct zink_resource_object *old_obj = res->obj;
@ -640,20 +636,17 @@ zink_transfer_copy_bufimage(struct zink_context *ctx,
box.y, box.z, trans->base.level, &box, trans->base.usage);
}
#define ALL_GFX_USAGE(batch_uses, usage) (batch_uses & ((usage << ZINK_NUM_GFX_BATCHES) - ((usage & ZINK_RESOURCE_ACCESS_RW))))
#define ALL_COMPUTE_USAGE(batch_uses, usage) (batch_uses & (usage << ZINK_COMPUTE_BATCH_ID))
bool
zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usage, enum zink_queue queue)
{
uint32_t batch_uses = get_resource_usage(res);
uint32_t batch_uses = get_all_resource_usage(res);
switch (queue) {
case ZINK_QUEUE_COMPUTE:
return ALL_COMPUTE_USAGE(batch_uses, usage);
return batch_uses & (usage << ZINK_QUEUE_COMPUTE);
case ZINK_QUEUE_GFX:
return ALL_GFX_USAGE(batch_uses, usage);
return batch_uses & (usage << ZINK_QUEUE_GFX);
case ZINK_QUEUE_ANY:
return ALL_GFX_USAGE(batch_uses, usage) || ALL_COMPUTE_USAGE(batch_uses, usage);
return batch_uses & ((usage << ZINK_QUEUE_GFX) | (usage << ZINK_QUEUE_COMPUTE));
default:
break;
}
@ -661,13 +654,6 @@ zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usa
return false;
}
bool
zink_resource_has_usage_for_id(struct zink_resource *res, uint32_t id)
{
uint32_t batch_uses = get_resource_usage(res);
return batch_uses & (ZINK_RESOURCE_ACCESS_RW) << id;
}
static void *
zink_transfer_map(struct pipe_context *pctx,
struct pipe_resource *pres,
@ -706,7 +692,7 @@ zink_transfer_map(struct pipe_context *pctx,
if (util_ranges_intersect(&res->valid_buffer_range, box->x, box->x + box->width)) {
/* special case compute reads since they aren't handled by zink_fence_wait() */
if (usage & PIPE_MAP_WRITE && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_READ, ZINK_QUEUE_COMPUTE))
zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID);
resource_sync_reads_from_compute(ctx, res);
if (usage & PIPE_MAP_READ && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_WRITE, ZINK_QUEUE_ANY))
resource_sync_writes_from_batch_usage(ctx, res);
else if (usage & PIPE_MAP_WRITE && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_ANY)) {
@ -806,7 +792,7 @@ zink_transfer_map(struct pipe_context *pctx,
assert(!res->optimal_tiling);
/* special case compute reads since they aren't handled by zink_fence_wait() */
if (zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_READ, ZINK_QUEUE_COMPUTE))
zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID);
resource_sync_reads_from_compute(ctx, res);
if (zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_ANY)) {
if (usage & PIPE_MAP_READ)
resource_sync_writes_from_batch_usage(ctx, res);

8
src/gallium/drivers/zink/zink_resource.h
View file

@ -35,6 +35,7 @@ struct zink_context;
#include "util/u_range.h"
#include "util/u_dynarray.h"
#include "zink_batch.h"
#include "zink_descriptors.h"
#include <vulkan/vulkan.h>
@ -64,8 +65,8 @@ struct zink_resource_object {
unsigned persistent_maps; //if nonzero, requires vkFlushMappedMemoryRanges during batch use
struct zink_descriptor_refs desc_set_refs;
/* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */
uint8_t batch_uses[5]; //ZINK_NUM_BATCHES
struct zink_batch_usage reads;
struct zink_batch_usage writes;
bool is_buffer;
bool host_visible;
};
@ -121,9 +122,6 @@ zink_get_depth_stencil_resources(struct pipe_resource *res,
void
zink_resource_setup_transfer_layouts(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst);
int
zink_get_resource_latest_batch_usage(struct zink_context *ctx, uint32_t batch_uses);
bool
zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usage, enum zink_queue queue);

4
src/gallium/drivers/zink/zink_surface.h
View file

@ -25,7 +25,7 @@
#define ZINK_SURFACE_H
#include "pipe/p_state.h"
#include "zink_batch.h"
#include <vulkan/vulkan.h>
struct pipe_context;
@ -35,7 +35,7 @@ struct zink_surface {
VkImageViewCreateInfo ivci;
VkImageView image_view;
uint32_t hash;
uint32_t batch_uses;
struct zink_batch_usage batch_uses;
};
static inline struct zink_surface *