diff --git a/src/gallium/drivers/zink/zink_batch.c b/src/gallium/drivers/zink/zink_batch.c index 38c64d93673..b76b5f03b90 100644 --- a/src/gallium/drivers/zink/zink_batch.c +++ b/src/gallium/drivers/zink/zink_batch.c @@ -16,12 +16,20 @@ #include "wsi_common.h" +static void +batch_usage_unset(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id) +{ + p_atomic_cmpxchg(&u->usage[queue], batch_id, 0); +} + void zink_batch_state_clear_resources(struct zink_screen *screen, struct zink_batch_state *bs) { /* unref all used resources */ set_foreach(bs->resources, entry) { struct zink_resource_object *obj = (struct zink_resource_object *)entry->key; + batch_usage_unset(&obj->reads, !!bs->is_compute, bs->batch_id); + batch_usage_unset(&obj->writes, !!bs->is_compute, bs->batch_id); zink_resource_object_reference(screen, &obj, NULL); _mesa_set_remove(bs->resources, entry); } @@ -31,8 +39,6 @@ void zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) { struct zink_screen *screen = zink_screen(ctx->base.screen); - if (bs->fence->submitted) - zink_fence_finish(screen, &ctx->base, bs->fence, PIPE_TIMEOUT_INFINITE); zink_batch_state_clear_resources(screen, bs); @@ -44,13 +50,13 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) set_foreach(bs->surfaces, entry) { struct zink_surface *surf = (struct zink_surface *)entry->key; - surf->batch_uses &= ~BITFIELD64_BIT(bs->batch_id); + batch_usage_unset(&surf->batch_uses, !!bs->is_compute, bs->batch_id); pipe_surface_reference((struct pipe_surface**)&surf, NULL); _mesa_set_remove(bs->surfaces, entry); } set_foreach(bs->bufferviews, entry) { struct zink_buffer_view *buffer_view = (struct zink_buffer_view *)entry->key; - buffer_view->batch_uses &= ~BITFIELD64_BIT(bs->batch_id); + batch_usage_unset(&buffer_view->batch_uses, !!bs->is_compute, bs->batch_id); zink_buffer_view_reference(screen, &buffer_view, NULL); _mesa_set_remove(bs->bufferviews, entry); } @@ -63,7 +69,7 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) set_foreach(bs->desc_sets, entry) { struct zink_descriptor_set *zds = (void*)entry->key; - zds->batch_uses &= ~BITFIELD_BIT(bs->batch_id); + batch_usage_unset(&zds->batch_uses, !!bs->is_compute, bs->batch_id); /* reset descriptor pools when no bs is using this program to avoid * having some inactive program hogging a billion descriptors */ @@ -96,10 +102,21 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) bs->flush_res = NULL; bs->descs_used = 0; - bs->fence->submitted = false; + ctx->resource_size[bs->is_compute] -= bs->resource_size; bs->resource_size = 0; } +void +zink_batch_reset_all(struct zink_context *ctx, enum zink_queue queue) +{ + hash_table_foreach(&ctx->batch_states[queue], entry) { + struct zink_batch_state *bs = entry->data; + zink_reset_batch_state(ctx, bs); + _mesa_hash_table_remove(&ctx->batch_states[queue], entry); + util_dynarray_append(&ctx->free_batch_states[queue], struct zink_batch_state *, bs); + } +} + void zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs) { @@ -123,7 +140,7 @@ zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs } static struct zink_batch_state * -create_batch_state(struct zink_context *ctx, unsigned idx) +create_batch_state(struct zink_context *ctx, enum zink_queue queue) { struct zink_screen *screen = zink_screen(ctx->base.screen); struct zink_batch_state *bs = rzalloc(NULL, struct zink_batch_state); @@ -157,13 +174,12 @@ create_batch_state(struct zink_context *ctx, unsigned idx) SET_CREATE_OR_FAIL(bs->active_queries); util_dynarray_init(&bs->zombie_samplers, NULL); util_dynarray_init(&bs->persistent_resources, NULL); - bs->batch_id = idx; if (!zink_create_fence(screen, bs)) /* this destroys the batch state on failure */ return NULL; - bs->is_compute = idx == ZINK_COMPUTE_BATCH_ID; + bs->is_compute = queue == ZINK_QUEUE_COMPUTE; return bs; fail: @@ -171,10 +187,40 @@ fail: return NULL; } +static bool +find_unused_state(struct hash_entry *entry) +{ + struct zink_fence *fence = entry->data; + /* we can't reset these from fence_finish because threads */ + bool submitted = p_atomic_read(&fence->submitted); + return !submitted; +} + static void init_batch_state(struct zink_context *ctx, struct zink_batch *batch) { - struct zink_batch_state *bs = create_batch_state(ctx, batch->batch_id); + struct zink_batch_state *bs = NULL; + + if (util_dynarray_num_elements(&ctx->free_batch_states[batch->queue], struct zink_batch_state*)) + bs = util_dynarray_pop(&ctx->free_batch_states[batch->queue], struct zink_batch_state*); + if (!bs) { + struct hash_entry *he = _mesa_hash_table_random_entry(&ctx->batch_states[batch->queue], find_unused_state); + if (he) { //there may not be any entries available + bs = he->data; + _mesa_hash_table_remove(&ctx->batch_states[batch->queue], he); + zink_reset_batch_state(ctx, bs); + } + } + if (!bs) { + if (!batch->state) { + /* this is batch init, so create a few more states for later use */ + for (int i = 0; i < 3; i++) { + struct zink_batch_state *state = create_batch_state(ctx, batch->queue); + util_dynarray_append(&ctx->free_batch_states[batch->queue], struct zink_batch_state *, state); + } + } + bs = create_batch_state(ctx, batch->queue); + } batch->state = bs; } @@ -205,6 +251,11 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch) if (vkBeginCommandBuffer(batch->state->cmdbuf, &cbbi) != VK_SUCCESS) debug_printf("vkBeginCommandBuffer failed\n"); + batch->state->batch_id = ctx->curr_batch; + if (ctx->last_fence[batch->queue]) { + struct zink_batch_state *last_state = zink_batch_state(ctx->last_fence[batch->queue]); + batch->last_batch_id = last_state->batch_id; + } if (!ctx->queries_disabled) zink_resume_queries(ctx, batch); } @@ -255,7 +306,7 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch) si.pNext = &mem_signal; } - if (vkQueueSubmit(ctx->queue, 1, &si, batch->state->fence->fence) != VK_SUCCESS) { + if (vkQueueSubmit(ctx->queue, 1, &si, batch->state->fence.fence) != VK_SUCCESS) { debug_printf("ZINK: vkQueueSubmit() failed\n"); ctx->is_device_lost = true; @@ -263,7 +314,10 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch) ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET); } } - batch->state->fence->submitted = true; + + ctx->last_fence[batch->queue] = &batch->state->fence; + _mesa_hash_table_insert_pre_hashed(&ctx->batch_states[batch->queue], batch->state->batch_id, (void*)(uintptr_t)batch->state->batch_id, batch->state); + ctx->resource_size[batch->queue] += batch->state->resource_size; } /* returns a queue based on whether a resource @@ -272,7 +326,6 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch) enum zink_queue zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource *res, bool write) { - unsigned mask = write ? ZINK_RESOURCE_ACCESS_WRITE : ZINK_RESOURCE_ACCESS_READ; enum zink_queue batch_to_flush = 0; /* u_transfer_helper unrefs the stencil buffer when the depth buffer is unrefed, @@ -282,7 +335,7 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil); - if (batch->batch_id == ZINK_COMPUTE_BATCH_ID) { + if (batch->queue == ZINK_QUEUE_COMPUTE) { if ((write && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_GFX)) || (!write && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_WRITE, ZINK_QUEUE_GFX))) batch_to_flush = ZINK_QUEUE_GFX; @@ -293,43 +346,50 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource } /* if the resource already has usage of any sort set for this batch, we can skip hashing */ - if (!zink_resource_has_usage_for_id(res, batch->state->batch_id)) { + if (!zink_batch_usage_matches(&res->obj->reads, batch->queue, batch->state->batch_id) && + !zink_batch_usage_matches(&res->obj->writes, batch->queue, batch->state->batch_id)) { bool found = false; _mesa_set_search_and_add(batch->state->resources, res->obj, &found); if (!found) { pipe_reference(NULL, &res->obj->reference); - batch->state->resource_size += res->obj->size; + if (!batch->last_batch_id || !zink_batch_usage_matches(&res->obj->reads, batch->queue, batch->last_batch_id)) + /* only add resource usage if it's "new" usage, though this only checks the most recent usage + * and not all pending usages + */ + batch->state->resource_size += res->obj->size; if (stencil) { pipe_reference(NULL, &stencil->obj->reference); - batch->state->resource_size += stencil->obj->size; + if (!batch->last_batch_id || !zink_batch_usage_matches(&stencil->obj->reads, batch->queue, batch->last_batch_id)) + batch->state->resource_size += stencil->obj->size; } } + } + if (write) { + if (stencil) + zink_batch_usage_set(&stencil->obj->writes, batch->queue, batch->state->batch_id); + zink_batch_usage_set(&res->obj->writes, batch->queue, batch->state->batch_id); + } else { + if (stencil) + zink_batch_usage_set(&stencil->obj->reads, batch->queue, batch->state->batch_id); + zink_batch_usage_set(&res->obj->reads, batch->queue, batch->state->batch_id); } /* multiple array entries are fine */ if (res->obj->persistent_maps) util_dynarray_append(&batch->state->persistent_resources, struct zink_resource*, res); - /* the batch_uses value for this batch is guaranteed to not be in use now because - * zink_reset_batch() waits on the fence and removes access before resetting - */ - res->obj->batch_uses[batch->batch_id] |= mask; - - if (stencil) - stencil->obj->batch_uses[batch->batch_id] |= mask; batch->has_work = true; return batch_to_flush; } static bool -ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, uint32_t *u) +ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u) { bool found = false; - uint32_t bit = BITFIELD_BIT(batch->state->batch_id); - if ((*u) & bit) + if (zink_batch_usage_matches(u, batch->queue, batch->state->batch_id)) return false; _mesa_set_search_and_add(s, ptr, &found); assert(!found); - *u |= bit; + zink_batch_usage_set(u, batch->queue, batch->state->batch_id); return true; } @@ -394,3 +454,38 @@ zink_batch_reference_image_view(struct zink_batch *batch, } batch->has_work = true; } + +void +zink_batch_usage_set(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id) +{ + if (queue == ZINK_QUEUE_ANY) { + p_atomic_set(&u->usage[ZINK_QUEUE_GFX], batch_id); + p_atomic_set(&u->usage[ZINK_QUEUE_COMPUTE], batch_id); + } else + p_atomic_set(&u->usage[queue], batch_id); +} + +bool +zink_batch_usage_matches(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id) +{ + if (queue < ZINK_QUEUE_ANY) { + uint32_t usage = p_atomic_read(&u->usage[queue]); + return usage == batch_id; + } + for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) { + uint32_t usage = p_atomic_read(&u->usage[queue]); + if (usage == batch_id) + return true; + } + return false; +} + +bool +zink_batch_usage_exists(struct zink_batch_usage *u) +{ + uint32_t usage = p_atomic_read(&u->usage[ZINK_QUEUE_GFX]); + if (usage) + return true; + usage = p_atomic_read(&u->usage[ZINK_QUEUE_COMPUTE]); + return !!usage; +} diff --git a/src/gallium/drivers/zink/zink_batch.h b/src/gallium/drivers/zink/zink_batch.h index b99fe06348c..73c739453e3 100644 --- a/src/gallium/drivers/zink/zink_batch.h +++ b/src/gallium/drivers/zink/zink_batch.h @@ -29,18 +29,18 @@ #include "util/list.h" #include "util/u_dynarray.h" +#include "zink_fence.h" + struct pipe_reference; struct zink_context; struct zink_descriptor_set; -struct zink_fence; struct zink_framebuffer; struct zink_image_view; struct zink_program; struct zink_render_pass; struct zink_resource; struct zink_sampler_view; -struct zink_screen; struct zink_surface; enum zink_queue { @@ -49,15 +49,19 @@ enum zink_queue { ZINK_QUEUE_ANY, }; +struct zink_batch_usage { + /* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */ + uint32_t usage[2]; //gfx, compute +}; + struct zink_batch_state { - unsigned batch_id : 3; + struct zink_fence fence; VkCommandPool cmdpool; VkCommandBuffer cmdbuf; struct zink_resource *flush_res; unsigned short descs_used; //number of descriptors currently allocated - struct zink_fence *fence; struct set *fbs; struct set *programs; @@ -74,22 +78,33 @@ struct zink_batch_state { VkDeviceSize resource_size; + uint32_t batch_id; bool is_compute; }; struct zink_batch { - unsigned batch_id : 3; struct zink_batch_state *state; enum zink_queue queue; + uint32_t last_batch_id; + bool has_work; bool in_rp; //renderpass is currently active }; +static inline struct zink_batch_state * +zink_batch_state(struct zink_fence *fence) +{ + return (struct zink_batch_state *)fence; +} + void zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs); +void +zink_batch_reset_all(struct zink_context *ctx, enum zink_queue queue); + void zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs); @@ -127,4 +142,10 @@ zink_batch_reference_image_view(struct zink_batch *batch, bool zink_batch_add_desc_set(struct zink_batch *batch, struct zink_descriptor_set *zds); +void +zink_batch_usage_set(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id); +bool +zink_batch_usage_matches(struct zink_batch_usage *u, enum zink_queue queue, uint32_t batch_id); +bool +zink_batch_usage_exists(struct zink_batch_usage *u); #endif diff --git a/src/gallium/drivers/zink/zink_clear.c b/src/gallium/drivers/zink/zink_clear.c index f8dceb1a578..730da29915e 100644 --- a/src/gallium/drivers/zink/zink_clear.c +++ b/src/gallium/drivers/zink/zink_clear.c @@ -192,7 +192,7 @@ zink_clear(struct pipe_context *pctx, { struct zink_context *ctx = zink_context(pctx); struct pipe_framebuffer_state *fb = &ctx->fb_state; - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); bool needs_rp = false; if (scissor_state) { @@ -363,7 +363,7 @@ zink_clear_texture(struct pipe_context *pctx, struct pipe_screen *pscreen = pctx->screen; struct u_rect region = zink_rect_from_box(box); bool needs_rp = !zink_blit_region_fills(region, pres->width0, pres->height0) || ctx->render_condition_active; - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); struct pipe_surface *surf = NULL; if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) { @@ -431,7 +431,7 @@ fb_clears_apply_internal(struct zink_context *ctx, struct pipe_resource *pres, i if (!fb_clear->enabled) return; if (zink_resource(pres)->aspect == VK_IMAGE_ASPECT_COLOR_BIT) { - assert(!zink_curr_batch(ctx)->in_rp); + assert(!zink_batch_g(ctx)->in_rp); if (zink_fb_clear_needs_explicit(fb_clear) || !check_3d_layers(ctx->fb_state.cbufs[i])) /* this will automatically trigger all the clears */ zink_batch_rp(ctx); @@ -455,7 +455,7 @@ fb_clears_apply_internal(struct zink_context *ctx, struct pipe_resource *pres, i zink_fb_clear_reset(&ctx->fb_clears[i]); return; } else { - assert(!zink_curr_batch(ctx)->in_rp); + assert(!zink_batch_g(ctx)->in_rp); if (zink_fb_clear_needs_explicit(fb_clear) || !check_3d_layers(ctx->fb_state.zsbuf)) /* this will automatically trigger all the clears */ zink_batch_rp(ctx); diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 233565f20b8..afc25a5c4df 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -54,6 +54,14 @@ #define XXH_INLINE_ALL #include "util/xxhash.h" +static void +incr_curr_batch(struct zink_context *ctx) +{ + ctx->curr_batch++; + if (!ctx->curr_batch) + ctx->curr_batch = 1; +} + static struct zink_resource * get_resource_for_descriptor(struct zink_context *ctx, enum zink_descriptor_type type, enum pipe_shader_type shader, int idx) { @@ -281,7 +289,7 @@ zink_context_destroy(struct pipe_context *pctx) struct zink_context *ctx = zink_context(pctx); struct zink_screen *screen = zink_screen(pctx->screen); - if (vkQueueWaitIdle(ctx->queue) != VK_SUCCESS) + if (ctx->queue && vkQueueWaitIdle(ctx->queue) != VK_SUCCESS) debug_printf("vkQueueWaitIdle failed\n"); util_blitter_destroy(ctx->blitter); @@ -293,12 +301,21 @@ zink_context_destroy(struct pipe_context *pctx) for (unsigned i = 0; i < ARRAY_SIZE(ctx->null_buffers); i++) pipe_resource_reference(&ctx->null_buffers[i], NULL); - for (int i = 0; i < ARRAY_SIZE(ctx->batches); ++i) { + for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) { + struct zink_fence *fence = zink_fence(&ctx->batches[i].state); zink_reset_batch_state(ctx, ctx->batches[i].state); - zink_batch_state_destroy(screen, ctx->batches[i].state); + zink_fence_reference(zink_screen(pctx->screen), &fence, NULL); + hash_table_foreach(&ctx->batch_states[i], entry) { + fence = entry->data; + zink_reset_batch_state(ctx, entry->data); + zink_fence_reference(zink_screen(pctx->screen), &fence, NULL); + } + util_dynarray_foreach(&ctx->free_batch_states[i], struct zink_batch_state*, bs) { + fence = zink_fence(*bs); + zink_reset_batch_state(ctx, *bs); + zink_fence_reference(zink_screen(pctx->screen), &fence, NULL); + } } - zink_reset_batch_state(ctx, ctx->compute_batch.state); - zink_batch_state_destroy(screen, ctx->compute_batch.state); hash_table_foreach(ctx->render_pass_cache, he) zink_destroy_render_pass(screen, he->data); @@ -495,7 +512,7 @@ zink_delete_sampler_state(struct pipe_context *pctx, void *sampler_state) { struct zink_sampler_state *sampler = sampler_state; - struct zink_batch *batch = zink_curr_batch(zink_context(pctx)); + struct zink_batch *batch = zink_batch_g(zink_context(pctx)); zink_descriptor_set_refs_clear(&sampler->desc_set_refs, sampler_state); util_dynarray_append(&batch->state->zombie_samplers, VkSampler, sampler->sampler); @@ -1179,7 +1196,7 @@ setup_framebuffer(struct zink_context *ctx) void zink_begin_render_pass(struct zink_context *ctx, struct zink_batch *batch) { - assert(batch == zink_curr_batch(ctx)); + assert(batch == zink_batch_g(ctx)); setup_framebuffer(ctx); assert(ctx->gfx_pipeline_state.render_pass); @@ -1268,23 +1285,22 @@ zink_end_render_pass(struct zink_context *ctx, struct zink_batch *batch) } static void -flush_batch(struct zink_context *ctx) +flush_batch(struct zink_context *ctx, enum zink_queue queue) { - struct zink_batch *batch = zink_curr_batch(ctx); - zink_end_render_pass(ctx, batch); + struct zink_batch *batch = zink_batch_queue(ctx, queue); + if (queue == ZINK_QUEUE_GFX) + zink_end_render_pass(ctx, batch); zink_end_batch(ctx, batch); - ctx->curr_batch++; - if (ctx->curr_batch == ARRAY_SIZE(ctx->batches)) - ctx->curr_batch = 0; + incr_curr_batch(ctx); - zink_start_batch(ctx, zink_curr_batch(ctx)); + zink_start_batch(ctx, batch); } struct zink_batch * zink_batch_rp(struct zink_context *ctx) { - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); if (!batch->in_rp) { zink_begin_render_pass(ctx, batch); assert(ctx->framebuffer && ctx->framebuffer->rp); @@ -1295,7 +1311,7 @@ zink_batch_rp(struct zink_context *ctx) struct zink_batch * zink_batch_no_rp(struct zink_context *ctx) { - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); zink_end_render_pass(ctx, batch); assert(!batch->in_rp); return batch; @@ -1304,19 +1320,14 @@ zink_batch_no_rp(struct zink_context *ctx) void zink_flush_compute(struct zink_context *ctx) { - zink_end_batch(ctx, &ctx->compute_batch); - zink_start_batch(ctx, &ctx->compute_batch); + flush_batch(ctx, ZINK_QUEUE_COMPUTE); } struct zink_batch * zink_flush_batch(struct zink_context *ctx, struct zink_batch *batch) { - if (batch && batch->batch_id >= ZINK_COMPUTE_BATCH_ID) { - zink_flush_compute(ctx); - return &ctx->compute_batch; - } - flush_batch(ctx); - return zink_curr_batch(ctx); + flush_batch(ctx, batch->queue); + return batch; } static void @@ -1526,7 +1537,7 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_batch *batch, /* only barrier if we're changing layout or doing something besides read -> read */ if (!batch) { if (pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT) - batch = &ctx->compute_batch; + batch = zink_batch_c(ctx); else batch = zink_batch_no_rp(ctx); } @@ -1624,7 +1635,7 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_batch *batch, /* only barrier if we're changing layout or doing something besides read -> read */ if (!batch) { if (pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT) - batch = &ctx->compute_batch; + batch = zink_batch_c(ctx); else batch = zink_batch_no_rp(ctx); } @@ -1704,10 +1715,11 @@ zink_flush(struct pipe_context *pctx, { struct zink_context *ctx = zink_context(pctx); bool deferred = flags & PIPE_FLUSH_DEFERRED; - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); + struct zink_fence *fence = &batch->state->fence; if (deferred) - batch->state->fence->deferred_ctx = pctx; + batch->state->fence.deferred_ctx = pctx; else if (batch->has_work) { if (flags & PIPE_FLUSH_END_OF_FRAME) { if (ctx->fb_state.nr_cbufs) @@ -1719,7 +1731,7 @@ zink_flush(struct pipe_context *pctx, if (zink_screen(pctx->screen)->needs_mesa_flush_wsi && ctx->fb_state.cbufs[0]) batch->state->flush_res = zink_resource(ctx->fb_state.cbufs[0]->texture); } - flush_batch(ctx); + flush_batch(ctx, ZINK_QUEUE_GFX); if (zink_screen(pctx->screen)->info.have_EXT_transform_feedback && ctx->num_so_targets) ctx->dirty_so_targets = true; @@ -1728,56 +1740,81 @@ zink_flush(struct pipe_context *pctx, if (!pfence) return; if (deferred && !batch->has_work) { - batch = zink_prev_batch(ctx); + fence = ctx->last_fence[ZINK_QUEUE_GFX]; } zink_fence_reference(zink_screen(pctx->screen), (struct zink_fence **)pfence, - batch->state->fence); + fence); if (flags & PIPE_FLUSH_END_OF_FRAME) { /* if the first frame has not yet occurred, we need an explicit fence here * in some cases in order to correctly draw the first frame, though it's * unknown at this time why this is the case */ if (!ctx->first_frame_done) - zink_fence_finish(zink_screen(pctx->screen), pctx, batch->state->fence, PIPE_TIMEOUT_INFINITE); + zink_fence_finish(zink_screen(pctx->screen), pctx, fence, PIPE_TIMEOUT_INFINITE); ctx->first_frame_done = true; } } +void +zink_maybe_flush_or_stall(struct zink_context *ctx, enum zink_queue queue) +{ + struct zink_screen *screen = zink_screen(ctx->base.screen); + /* flush anytime our total batch memory usage is potentially >= 1/10 of total system memory */ + if (zink_batch_queue(ctx, queue)->state->resource_size >= screen->total_mem / 10) + flush_batch(ctx, queue); + + if (ctx->resource_size[queue] >= screen->total_mem / 10) { + zink_fence_finish(zink_screen(ctx->base.screen), &ctx->base, ctx->last_fence[queue], PIPE_TIMEOUT_INFINITE); + zink_batch_reset_all(ctx, queue); + } +} + void zink_fence_wait(struct pipe_context *pctx) { - struct pipe_fence_handle *fence = NULL; - pctx->flush(pctx, &fence, PIPE_FLUSH_HINT_FINISH); - if (fence) { - pctx->screen->fence_finish(pctx->screen, NULL, fence, - PIPE_TIMEOUT_INFINITE); - pctx->screen->fence_reference(pctx->screen, &fence, NULL); - } + struct zink_context *ctx = zink_context(pctx); + + if (zink_batch_g(ctx)->has_work) + pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH); + if (ctx->last_fence[ZINK_QUEUE_GFX]) + zink_fence_finish(zink_screen(pctx->screen), pctx, ctx->last_fence[ZINK_QUEUE_GFX], PIPE_TIMEOUT_INFINITE); } void -zink_wait_on_batch(struct zink_context *ctx, int batch_id) +zink_wait_on_batch(struct zink_context *ctx, enum zink_queue queue, uint32_t batch_id) { - if (batch_id >= 0) { - struct zink_batch *batch = batch_id == ZINK_COMPUTE_BATCH_ID ? &ctx->compute_batch : &ctx->batches[batch_id]; - if (batch != zink_curr_batch(ctx)) { - if (!batch->state->fence->submitted) { // this is the compute batch - zink_flush_compute(ctx); - } else - ctx->base.screen->fence_finish(ctx->base.screen, NULL, (struct pipe_fence_handle*)batch->state->fence, - PIPE_TIMEOUT_INFINITE); - return; + struct zink_batch_state *bs = zink_batch_queue(ctx, queue)->state; + assert(bs); + if (!batch_id || bs->batch_id == batch_id) + /* not submitted yet */ + flush_batch(ctx, queue); + + struct zink_fence *fence; + + assert(batch_id || ctx->last_fence[queue]); + if (ctx->last_fence[queue] && (!batch_id || batch_id == zink_batch_state(ctx->last_fence[queue])->batch_id)) + fence = ctx->last_fence[queue]; + else { + struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&ctx->batch_states[queue], batch_id, (void*)(uintptr_t)batch_id); + if (!he) { + util_dynarray_foreach(&ctx->free_batch_states[queue], struct zink_batch_state*, bs) { + if ((*bs)->batch_id == batch_id) + return; + } + unreachable("should've found batch state"); } + fence = he->data; } - zink_fence_wait(&ctx->base); + assert(fence); + ctx->base.screen->fence_finish(ctx->base.screen, &ctx->base, (struct pipe_fence_handle*)fence, PIPE_TIMEOUT_INFINITE); } static void zink_texture_barrier(struct pipe_context *pctx, unsigned flags) { struct zink_context *ctx = zink_context(pctx); - if (zink_curr_batch(ctx)->has_work) + if (zink_batch_g(ctx)->has_work) pctx->flush(pctx, NULL, 0); zink_flush_compute(ctx); } @@ -1895,15 +1932,15 @@ zink_memory_barrier(struct pipe_context *pctx, unsigned flags) b.srcAccessMask = sflags; b.dstAccessMask = dflags; - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); if (batch->has_work) { zink_end_render_pass(ctx, batch); /* this should be the only call needed */ vkCmdPipelineBarrier(batch->state->cmdbuf, src, dst, 0, 0, &b, 0, NULL, 0, NULL); - flush_batch(ctx); + flush_batch(ctx, ZINK_QUEUE_GFX); } - batch = &ctx->compute_batch; + batch = zink_batch_c(ctx); if (batch->has_work) { /* this should be the only call needed */ vkCmdPipelineBarrier(batch->state->cmdbuf, src, dst, 0, 0, &b, 0, NULL, 0, NULL); @@ -2227,13 +2264,12 @@ zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res) } } -static bool -init_batch(struct zink_context *ctx, struct zink_batch *batch, unsigned idx) +static void +init_batch(struct zink_context *ctx, enum zink_queue queue) { - batch->queue = idx == ZINK_COMPUTE_BATCH_ID ? ZINK_QUEUE_COMPUTE : ZINK_QUEUE_GFX; - batch->batch_id = idx; + struct zink_batch *batch = zink_batch_queue(ctx, queue); + batch->queue = queue; zink_start_batch(ctx, batch); - return !!batch->state; } struct pipe_context * @@ -2303,6 +2339,11 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) zink_context_resource_init(&ctx->base); zink_context_query_init(&ctx->base); + for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) { + util_dynarray_init(&ctx->free_batch_states[i], ctx); + _mesa_hash_table_init(&ctx->batch_states[i], ctx, NULL, _mesa_key_pointer_equal); + } + ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state; slab_create_child(&ctx->transfer_pool, &screen->transfer_pool); @@ -2328,12 +2369,13 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) if (!ctx->blitter) goto fail; - for (int i = 0; i < ARRAY_SIZE(ctx->batches); ++i) { - if (!init_batch(ctx, &ctx->batches[i], i)) - goto fail; - } + incr_curr_batch(ctx); + init_batch(ctx, ZINK_QUEUE_GFX); + if (!zink_batch_g(ctx)->state) + goto fail; - if (!init_batch(ctx, &ctx->compute_batch, ZINK_COMPUTE_BATCH_ID)) + init_batch(ctx, ZINK_QUEUE_COMPUTE); + if (!zink_batch_c(ctx)->state) goto fail; vkGetDeviceQueue(screen->dev, screen->gfx_queue, 0, &ctx->queue); diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index c5da1d158af..51eb4d3c14f 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -25,10 +25,6 @@ #define ZINK_CONTEXT_H #define ZINK_SHADER_COUNT (PIPE_SHADER_TYPES - 1) -#define ZINK_NUM_GFX_BATCHES 4 -#define ZINK_COMPUTE_BATCH_ID ZINK_NUM_GFX_BATCHES -#define ZINK_COMPUTE_BATCH_COUNT 1 -#define ZINK_NUM_BATCHES (ZINK_NUM_GFX_BATCHES + 1) #define ZINK_DEFAULT_MAX_DESCS 5000 @@ -45,6 +41,7 @@ #include "util/slab.h" #include "util/list.h" +#include "util/u_dynarray.h" #include @@ -70,7 +67,7 @@ struct zink_sampler_state { VkSampler sampler; uint32_t hash; struct zink_descriptor_refs desc_set_refs; - uint32_t batch_uses; + struct zink_batch_usage batch_uses; bool custom_border_color; }; @@ -79,7 +76,7 @@ struct zink_buffer_view { VkBufferViewCreateInfo bvci; VkBufferView buffer_view; uint32_t hash; - uint32_t batch_uses; + struct zink_batch_usage batch_uses; }; struct zink_sampler_view { @@ -140,13 +137,15 @@ struct zink_context { struct pipe_device_reset_callback reset; - struct zink_batch batches[ZINK_NUM_GFX_BATCHES]; bool is_device_lost; - unsigned curr_batch; + uint32_t curr_batch; //the current batch id + struct zink_batch batches[2]; //gfx, compute + struct zink_fence *last_fence[2]; //gfx, compute; the last command buffer submitted VkQueue queue; - - struct zink_batch compute_batch; + struct hash_table batch_states[2]; //gfx, compute; submitted batch states + struct util_dynarray free_batch_states[2]; //gfx, compute; unused batch states + VkDeviceSize resource_size[2]; //gfx, compute; the accumulated size of resources in submitted buffers struct pipe_constant_buffer ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS]; struct pipe_shader_buffer ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS]; @@ -231,22 +230,22 @@ zink_context(struct pipe_context *context) } static inline struct zink_batch * -zink_curr_batch(struct zink_context *ctx) +zink_batch_queue(struct zink_context *ctx, enum zink_queue queue_type) { - assert(ctx->curr_batch < ARRAY_SIZE(ctx->batches)); - return ctx->batches + ctx->curr_batch; + assert(queue_type < ARRAY_SIZE(ctx->batches)); + return &ctx->batches[queue_type]; } static inline struct zink_batch * -zink_prev_batch(struct zink_context *ctx) +zink_batch_g(struct zink_context *ctx) { - unsigned curr_batch = ctx->curr_batch; - if (!curr_batch) - curr_batch = ZINK_NUM_GFX_BATCHES - 1; - else - curr_batch--; - assert(curr_batch < ARRAY_SIZE(ctx->batches)); - return ctx->batches + curr_batch; + return &ctx->batches[ZINK_QUEUE_GFX]; +} + +static inline struct zink_batch * +zink_batch_c(struct zink_context *ctx) +{ + return &ctx->batches[ZINK_QUEUE_COMPUTE]; } struct zink_batch * @@ -259,7 +258,7 @@ void zink_fence_wait(struct pipe_context *ctx); void -zink_wait_on_batch(struct zink_context *ctx, int batch_id); +zink_wait_on_batch(struct zink_context *ctx, enum zink_queue queue, uint32_t batch_id); void zink_flush_compute(struct zink_context *ctx); @@ -267,6 +266,9 @@ zink_flush_compute(struct zink_context *ctx); struct zink_batch * zink_flush_batch(struct zink_context *ctx, struct zink_batch *batch); +void +zink_maybe_flush_or_stall(struct zink_context *ctx, enum zink_queue queue); + bool zink_resource_access_is_write(VkAccessFlags flags); diff --git a/src/gallium/drivers/zink/zink_descriptors.c b/src/gallium/drivers/zink/zink_descriptors.c index 91023c5f9f8..8d73589b735 100644 --- a/src/gallium/drivers/zink/zink_descriptors.c +++ b/src/gallium/drivers/zink/zink_descriptors.c @@ -232,7 +232,7 @@ allocate_desc_set(struct zink_screen *screen, struct zink_program *pg, enum zink pipe_reference_init(&zds->reference, 1); zds->pool = pool; zds->hash = 0; - zds->batch_uses = 0; + zds->batch_uses.usage[0] = zds->batch_uses.usage[1] = 0; zds->invalid = true; zds->punted = zds->recycled = false; if (num_resources) { @@ -296,7 +296,7 @@ zink_descriptor_set_get(struct zink_context *ctx, struct zink_descriptor_set *zds; struct zink_screen *screen = zink_screen(ctx->base.screen); struct zink_program *pg = is_compute ? (struct zink_program *)ctx->curr_compute : (struct zink_program *)ctx->curr_program; - struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx); + struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx); struct zink_descriptor_pool *pool = pg->pool[type]; unsigned descs_used = 1; assert(type < ZINK_DESCRIPTOR_TYPES); @@ -315,7 +315,7 @@ zink_descriptor_set_get(struct zink_context *ctx, zds->recycled = false; } if (zds->invalid) { - if (zds->batch_uses) + if (zink_batch_usage_exists(&zds->batch_uses)) punt_invalid_set(zds, NULL); else /* this set is guaranteed to be in pool->alloc_desc_sets */ @@ -332,7 +332,7 @@ zink_descriptor_set_get(struct zink_context *ctx, bool recycled = false, punted = false; if (he) { zds = (void*)he->data; - if (zds->invalid && zds->batch_uses) { + if (zds->invalid && zink_batch_usage_exists(&zds->batch_uses)) { punt_invalid_set(zds, he); zds = NULL; punted = true; @@ -375,7 +375,7 @@ skip_hash_tables: } if (pool->num_sets_allocated + pool->key.num_descriptors > ZINK_DEFAULT_MAX_DESCS) { - batch = zink_flush_batch(ctx, batch); + zink_fence_wait(&ctx->base); zink_batch_reference_program(batch, pg); return zink_descriptor_set_get(ctx, type, is_compute, cache_hit, need_resource_refs); } diff --git a/src/gallium/drivers/zink/zink_descriptors.h b/src/gallium/drivers/zink/zink_descriptors.h index db09b42d586..374de1ad752 100644 --- a/src/gallium/drivers/zink/zink_descriptors.h +++ b/src/gallium/drivers/zink/zink_descriptors.h @@ -96,7 +96,7 @@ struct zink_descriptor_set { bool recycled; struct zink_descriptor_state_key key; struct util_dynarray barriers; - uint32_t batch_uses; + struct zink_batch_usage batch_uses; #ifndef NDEBUG /* for extra debug asserts */ unsigned num_resources; diff --git a/src/gallium/drivers/zink/zink_draw.c b/src/gallium/drivers/zink/zink_draw.c index e8541cf188f..53d4116da94 100644 --- a/src/gallium/drivers/zink/zink_draw.c +++ b/src/gallium/drivers/zink/zink_draw.c @@ -118,7 +118,7 @@ zink_emit_stream_output_targets(struct pipe_context *pctx) { struct zink_context *ctx = zink_context(pctx); struct zink_screen *screen = zink_screen(pctx->screen); - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); VkBuffer buffers[PIPE_MAX_SO_OUTPUTS] = {}; VkDeviceSize buffer_offsets[PIPE_MAX_SO_OUTPUTS] = {}; VkDeviceSize buffer_sizes[PIPE_MAX_SO_OUTPUTS] = {}; @@ -342,7 +342,7 @@ write_descriptors(struct zink_context *ctx, struct zink_descriptor_set *zds, uns bool is_compute, bool cache_hit, bool need_resource_refs) { bool need_flush = false; - struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx); + struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx); struct zink_screen *screen = zink_screen(ctx->base.screen); assert(zds->desc_set); enum zink_queue check_flush_id = is_compute ? ZINK_QUEUE_GFX : ZINK_QUEUE_COMPUTE; @@ -644,12 +644,12 @@ update_sampler_descriptors(struct zink_context *ctx, struct zink_descriptor_set desc_set_sampler_add(ctx, zds, sampler_view, sampler, num_resources++, zink_shader_descriptor_is_buffer(shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j), cache_hit); - struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx); + struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx); if (sampler_view) zink_batch_reference_sampler_view(batch, sampler_view); if (sampler) /* this only tracks the most recent usage for now */ - sampler->batch_uses = BITFIELD_BIT(batch->state->batch_id); + zink_batch_usage_set(&sampler->batch_uses, batch->queue, batch->state->batch_id); } assert(num_wds < num_descriptors); @@ -734,7 +734,7 @@ update_image_descriptors(struct zink_context *ctx, struct zink_descriptor_set *z &num_buffer_info, &buffer_views[num_buffer_info], NULL, imageview, bufferview, !k); - struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx); + struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx); if (res) zink_batch_reference_image_view(batch, image_view); } @@ -762,7 +762,7 @@ update_descriptors(struct zink_context *ctx, struct zink_screen *screen, bool is else zds[h] = NULL; } - struct zink_batch *batch = is_compute ? &ctx->compute_batch : zink_curr_batch(ctx); + struct zink_batch *batch = is_compute ? zink_batch_c(ctx) : zink_batch_g(ctx); zink_batch_reference_program(batch, pg); uint32_t dynamic_offsets[PIPE_MAX_CONSTANT_BUFFERS]; @@ -863,11 +863,8 @@ zink_draw_vbo(struct pipe_context *pctx, VkDeviceSize counter_buffer_offsets[PIPE_MAX_SO_OUTPUTS] = {}; bool need_index_buffer_unref = false; - /* flush anytime our total batch memory usage is potentially >= 1/10 of total gpu memory - * this should also eventually trigger a stall if the app is going nuts with gpu memory - */ - if (zink_curr_batch(ctx)->state->resource_size >= screen->total_mem / 10 / ZINK_NUM_BATCHES) - ctx->base.flush(&ctx->base, NULL, 0); + /* check memory usage and flush/stall as needed to avoid oom */ + zink_maybe_flush_or_stall(ctx, ZINK_QUEUE_GFX); if (dinfo->primitive_restart && !restart_supported(dinfo->mode)) { util_draw_vbo_without_prim_restart(pctx, dinfo, dindirect, &draws[0]); @@ -1178,13 +1175,10 @@ zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info) { struct zink_context *ctx = zink_context(pctx); struct zink_screen *screen = zink_screen(pctx->screen); - struct zink_batch *batch = &ctx->compute_batch; + struct zink_batch *batch = zink_batch_c(ctx); - /* flush anytime our total batch memory usage is potentially >= 1/10 of total gpu memory - * this should also eventually trigger a stall if the app is going nuts with gpu memory - */ - if (batch->state->resource_size >= screen->total_mem / 10 / ZINK_NUM_BATCHES) - zink_flush_compute(ctx); + /* check memory usage and flush/stall as needed to avoid oom */ + zink_maybe_flush_or_stall(ctx, ZINK_QUEUE_COMPUTE); struct zink_compute_program *comp_program = get_compute_program(ctx); if (!comp_program) diff --git a/src/gallium/drivers/zink/zink_fence.c b/src/gallium/drivers/zink/zink_fence.c index 485d6108c8a..8569baf9a7c 100644 --- a/src/gallium/drivers/zink/zink_fence.c +++ b/src/gallium/drivers/zink/zink_fence.c @@ -36,50 +36,34 @@ destroy_fence(struct zink_screen *screen, struct zink_fence *fence) { if (fence->fence) vkDestroyFence(screen->dev, fence->fence, NULL); - util_dynarray_fini(&fence->resources); - FREE(fence); + zink_batch_state_destroy(screen, zink_batch_state(fence)); } bool zink_create_fence(struct zink_screen *screen, struct zink_batch_state *bs) { + struct zink_fence *fence = zink_fence(bs); + VkFenceCreateInfo fci = {}; fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; - struct zink_fence *ret = CALLOC_STRUCT(zink_fence); - if (!ret) { - debug_printf("CALLOC_STRUCT failed\n"); - return false; - } - - if (vkCreateFence(screen->dev, &fci, NULL, &ret->fence) != VK_SUCCESS) { + if (vkCreateFence(screen->dev, &fci, NULL, &fence->fence) != VK_SUCCESS) { debug_printf("vkCreateFence failed\n"); goto fail; } - ret->batch_id = bs->batch_id; - util_dynarray_init(&ret->resources, NULL); - pipe_reference_init(&ret->reference, 1); - bs->fence = ret; + pipe_reference_init(&fence->reference, 1); return true; - fail: - destroy_fence(screen, ret); + destroy_fence(screen, fence); return false; } void zink_fence_init(struct zink_context *ctx, struct zink_batch *batch) { - struct zink_fence *fence = batch->state->fence; - set_foreach(batch->state->resources, entry) { - /* the fence needs its own reference to ensure it can safely access lifetime-dependent - * resource members - */ - struct zink_resource_object *obj = (struct zink_resource_object *)entry->key; - pipe_reference(NULL, &obj->reference); - util_dynarray_append(&fence->resources, struct zink_resource_object*, obj); - } + struct zink_fence *fence = zink_fence(batch->state); + vkResetFences(zink_screen(ctx->base.screen)->dev, 1, &fence->fence); fence->deferred_ctx = NULL; fence->submitted = true; @@ -105,18 +89,12 @@ fence_reference(struct pipe_screen *pscreen, zink_fence(pfence)); } -static inline void -fence_remove_resource_access(struct zink_fence *fence, struct zink_resource_object *obj) -{ - p_atomic_set(&obj->batch_uses[fence->batch_id], 0); -} - bool zink_fence_finish(struct zink_screen *screen, struct pipe_context *pctx, struct zink_fence *fence, uint64_t timeout_ns) { if (pctx && fence->deferred_ctx == pctx) { - zink_curr_batch(zink_context(pctx))->has_work = true; + zink_batch_g(zink_context(pctx))->has_work = true; /* this must be the current batch */ pctx->flush(pctx, NULL, 0); } @@ -131,14 +109,9 @@ zink_fence_finish(struct zink_screen *screen, struct pipe_context *pctx, struct success = vkGetFenceStatus(screen->dev, fence->fence) == VK_SUCCESS; if (success) { - /* unref all used resources */ - util_dynarray_foreach(&fence->resources, struct zink_resource_object*, obj) { - fence_remove_resource_access(fence, *obj); - - zink_resource_object_reference(screen, obj, NULL); - } - util_dynarray_clear(&fence->resources); - fence->submitted = false; + struct zink_batch_state *bs = zink_batch_state(fence); + zink_batch_state_clear_resources(screen, bs); + p_atomic_set(&fence->submitted, false); } return success; } @@ -160,7 +133,7 @@ zink_fence_server_sync(struct pipe_context *pctx, struct pipe_fence_handle *pfen return; if (fence->deferred_ctx) { - zink_curr_batch(zink_context(pctx))->has_work = true; + zink_batch_g(zink_context(pctx))->has_work = true; /* this must be the current batch */ pctx->flush(pctx, NULL, 0); } diff --git a/src/gallium/drivers/zink/zink_fence.h b/src/gallium/drivers/zink/zink_fence.h index 4c12a231745..5968acfda4f 100644 --- a/src/gallium/drivers/zink/zink_fence.h +++ b/src/gallium/drivers/zink/zink_fence.h @@ -25,27 +25,25 @@ #define ZINK_FENCE_H #include "util/u_inlines.h" -#include "util/u_dynarray.h" #include struct pipe_context; struct pipe_screen; +struct zink_batch; struct zink_batch_state; struct zink_context; struct zink_screen; struct zink_fence { struct pipe_reference reference; - unsigned batch_id : 3; VkFence fence; - struct util_dynarray resources; struct pipe_context *deferred_ctx; bool submitted; }; static inline struct zink_fence * -zink_fence(struct pipe_fence_handle *pfence) +zink_fence(void *pfence) { return (struct zink_fence *)pfence; } diff --git a/src/gallium/drivers/zink/zink_query.c b/src/gallium/drivers/zink/zink_query.c index ea912190ca7..c67651fdf24 100644 --- a/src/gallium/drivers/zink/zink_query.c +++ b/src/gallium/drivers/zink/zink_query.c @@ -37,7 +37,7 @@ struct zink_query { bool have_gs[NUM_QUERIES]; /* geometry shaders use GEOMETRY_SHADER_PRIMITIVES_BIT */ bool have_xfb[NUM_QUERIES]; /* xfb was active during this query */ - unsigned batch_id : 3; //batch that the query was started in + struct zink_batch_usage batch_id; //batch that the query was started in union pipe_query_result accumulated_result; }; @@ -137,10 +137,10 @@ static struct zink_batch * get_batch_for_query(struct zink_context *ctx, struct zink_query *query, bool no_rp) { if (query && is_cs_query(query)) - return &ctx->compute_batch; + return zink_batch_c(ctx); if (no_rp) return zink_batch_no_rp(ctx); - return zink_curr_batch(ctx); + return zink_batch_g(ctx); } static struct pipe_query * @@ -406,7 +406,7 @@ force_cpu_read(struct zink_context *ctx, struct pipe_query *pquery, bool wait, e unsigned result_size = result_type <= PIPE_QUERY_TYPE_U32 ? sizeof(uint32_t) : sizeof(uint64_t); struct zink_query *query = (struct zink_query*)pquery; union pipe_query_result result; - if (zink_curr_batch(ctx)->state->batch_id == query->batch_id) + if (zink_batch_usage_matches(&query->batch_id, ZINK_QUEUE_GFX, zink_batch_g(ctx)->state->batch_id)) pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH); else if (is_cs_query(query)) zink_flush_compute(ctx); @@ -535,7 +535,7 @@ begin_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_quer if (needs_stats_list(q)) list_addtail(&q->stats_list, &ctx->primitives_generated_queries); p_atomic_inc(&q->fences); - q->batch_id = batch->state->batch_id; + zink_batch_usage_set(&q->batch_id, batch->queue, batch->state->batch_id); _mesa_set_add(batch->state->active_queries, q); } @@ -565,7 +565,7 @@ end_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query if (is_time_query(q)) { vkCmdWriteTimestamp(batch->state->cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, q->query_pool, q->curr_query); - q->batch_id = batch->state->batch_id; + zink_batch_usage_set(&q->batch_id, batch->queue, batch->state->batch_id); } else if (q->type == PIPE_QUERY_PRIMITIVES_EMITTED || q->type == PIPE_QUERY_PRIMITIVES_GENERATED || q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) @@ -613,9 +613,10 @@ zink_get_query_result(struct pipe_context *pctx, struct zink_query *query = (void*)q; struct zink_context *ctx = zink_context(pctx); if (is_cs_query(query)) { - if (wait) - zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID); - else { + if (wait) { + uint32_t batch_id = p_atomic_read(&query->batch_id.usage[ZINK_QUEUE_COMPUTE]); + zink_wait_on_batch(ctx, ZINK_QUEUE_COMPUTE, batch_id); + } else { zink_flush_compute(ctx); } } else { @@ -671,7 +672,7 @@ zink_set_active_query_state(struct pipe_context *pctx, bool enable) struct zink_context *ctx = zink_context(pctx); ctx->queries_disabled = !enable; - struct zink_batch *batch = zink_curr_batch(ctx); + struct zink_batch *batch = zink_batch_g(ctx); if (ctx->queries_disabled) zink_suspend_queries(ctx, batch); else @@ -721,11 +722,11 @@ zink_render_condition(struct pipe_context *pctx, if (query->type != PIPE_QUERY_PRIMITIVES_GENERATED && !is_so_overflow_query(query)) { copy_results_to_buffer(ctx, query, res, 0, num_results, flags); - batch = zink_curr_batch(ctx); + batch = zink_batch_g(ctx); } else { /* these need special handling */ force_cpu_read(ctx, pquery, true, PIPE_QUERY_TYPE_U32, pres, 0); - batch = zink_curr_batch(ctx); + batch = zink_batch_g(ctx); zink_batch_reference_resource_rw(batch, res, false); } diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index b96f9e7a2ce..a15c8ff014e 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -54,52 +54,48 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object } static uint32_t -get_resource_usage(struct zink_resource *res) +get_resource_usage(struct zink_resource *res, enum zink_queue queue) +{ + assert(queue < 2); + uint32_t reads = p_atomic_read(&res->obj->reads.usage[queue]); + uint32_t writes = p_atomic_read(&res->obj->writes.usage[queue]); + uint32_t batch_uses = 0; + if (reads) + batch_uses |= ZINK_RESOURCE_ACCESS_READ << queue; + if (writes) + batch_uses |= ZINK_RESOURCE_ACCESS_WRITE << queue; + return batch_uses; +} + +static uint32_t +get_all_resource_usage(struct zink_resource *res) { uint32_t batch_uses = 0; - for (unsigned i = 0; i < ARRAY_SIZE(res->obj->batch_uses); i++) - batch_uses |= p_atomic_read(&res->obj->batch_uses[i]) << i; + for (unsigned i = 0; i < ZINK_QUEUE_ANY; i++) + batch_uses |= get_resource_usage(res, i); return batch_uses; } +static void +resource_sync_reads_from_compute(struct zink_context *ctx, struct zink_resource *res) +{ + uint32_t reads = p_atomic_read(&res->obj->reads.usage[ZINK_QUEUE_COMPUTE]); + assert(reads); + zink_wait_on_batch(ctx, ZINK_QUEUE_COMPUTE, reads); +} + static void resource_sync_writes_from_batch_usage(struct zink_context *ctx, struct zink_resource *res) { - uint32_t batch_uses = get_resource_usage(res); - batch_uses &= ~(ZINK_RESOURCE_ACCESS_READ << ZINK_COMPUTE_BATCH_ID); + uint32_t writes[2]; + for (int i = 0; i < ZINK_QUEUE_ANY; i++) + writes[i] = p_atomic_read(&res->obj->writes.usage[i]); - uint32_t write_mask = 0; - for (int i = 0; i < ZINK_NUM_GFX_BATCHES + ZINK_COMPUTE_BATCH_COUNT; i++) - write_mask |= ZINK_RESOURCE_ACCESS_WRITE << i; - while (batch_uses & write_mask) { - int batch_id = zink_get_resource_latest_batch_usage(ctx, batch_uses); - if (batch_id == -1) - break; - zink_wait_on_batch(ctx, batch_id); - batch_uses &= ~((ZINK_RESOURCE_ACCESS_RW) << batch_id); - } -} - -int -zink_get_resource_latest_batch_usage(struct zink_context *ctx, uint32_t batch_uses) -{ - unsigned cur_batch = zink_curr_batch(ctx)->batch_id; - - if (batch_uses & ZINK_RESOURCE_ACCESS_WRITE << ZINK_COMPUTE_BATCH_ID) - return ZINK_COMPUTE_BATCH_ID; - batch_uses &= ~(ZINK_RESOURCE_ACCESS_WRITE << ZINK_COMPUTE_BATCH_ID); - if (!batch_uses) - return -1; - for (unsigned i = 0; i < ZINK_NUM_BATCHES + 1; i++) { - /* loop backwards and sync with highest batch id that has writes */ - if (batch_uses & (ZINK_RESOURCE_ACCESS_WRITE << cur_batch)) { - return cur_batch; - } - cur_batch--; - if (cur_batch > ZINK_COMPUTE_BATCH_ID - 1) // underflowed past max batch id - cur_batch = ZINK_COMPUTE_BATCH_ID - 1; - } - return -1; + enum zink_queue queue = writes[0] < writes[1] ? ZINK_QUEUE_COMPUTE : ZINK_QUEUE_GFX; + /* sync lower id first */ + if (writes[!queue]) + zink_wait_on_batch(ctx, !queue, writes[!queue]); + zink_wait_on_batch(ctx, queue, writes[queue]); } static uint32_t @@ -603,7 +599,7 @@ zink_resource_invalidate(struct pipe_context *pctx, struct pipe_resource *pres) res->bind_history &= ~ZINK_RESOURCE_USAGE_STREAMOUT; util_range_set_empty(&res->valid_buffer_range); - if (!get_resource_usage(res)) + if (!get_all_resource_usage(res)) return; struct zink_resource_object *old_obj = res->obj; @@ -640,20 +636,17 @@ zink_transfer_copy_bufimage(struct zink_context *ctx, box.y, box.z, trans->base.level, &box, trans->base.usage); } -#define ALL_GFX_USAGE(batch_uses, usage) (batch_uses & ((usage << ZINK_NUM_GFX_BATCHES) - ((usage & ZINK_RESOURCE_ACCESS_RW)))) -#define ALL_COMPUTE_USAGE(batch_uses, usage) (batch_uses & (usage << ZINK_COMPUTE_BATCH_ID)) - bool zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usage, enum zink_queue queue) { - uint32_t batch_uses = get_resource_usage(res); + uint32_t batch_uses = get_all_resource_usage(res); switch (queue) { case ZINK_QUEUE_COMPUTE: - return ALL_COMPUTE_USAGE(batch_uses, usage); + return batch_uses & (usage << ZINK_QUEUE_COMPUTE); case ZINK_QUEUE_GFX: - return ALL_GFX_USAGE(batch_uses, usage); + return batch_uses & (usage << ZINK_QUEUE_GFX); case ZINK_QUEUE_ANY: - return ALL_GFX_USAGE(batch_uses, usage) || ALL_COMPUTE_USAGE(batch_uses, usage); + return batch_uses & ((usage << ZINK_QUEUE_GFX) | (usage << ZINK_QUEUE_COMPUTE)); default: break; } @@ -661,13 +654,6 @@ zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usa return false; } -bool -zink_resource_has_usage_for_id(struct zink_resource *res, uint32_t id) -{ - uint32_t batch_uses = get_resource_usage(res); - return batch_uses & (ZINK_RESOURCE_ACCESS_RW) << id; -} - static void * zink_transfer_map(struct pipe_context *pctx, struct pipe_resource *pres, @@ -706,7 +692,7 @@ zink_transfer_map(struct pipe_context *pctx, if (util_ranges_intersect(&res->valid_buffer_range, box->x, box->x + box->width)) { /* special case compute reads since they aren't handled by zink_fence_wait() */ if (usage & PIPE_MAP_WRITE && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_READ, ZINK_QUEUE_COMPUTE)) - zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID); + resource_sync_reads_from_compute(ctx, res); if (usage & PIPE_MAP_READ && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_WRITE, ZINK_QUEUE_ANY)) resource_sync_writes_from_batch_usage(ctx, res); else if (usage & PIPE_MAP_WRITE && zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_ANY)) { @@ -806,7 +792,7 @@ zink_transfer_map(struct pipe_context *pctx, assert(!res->optimal_tiling); /* special case compute reads since they aren't handled by zink_fence_wait() */ if (zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_READ, ZINK_QUEUE_COMPUTE)) - zink_wait_on_batch(ctx, ZINK_COMPUTE_BATCH_ID); + resource_sync_reads_from_compute(ctx, res); if (zink_resource_has_usage(res, ZINK_RESOURCE_ACCESS_RW, ZINK_QUEUE_ANY)) { if (usage & PIPE_MAP_READ) resource_sync_writes_from_batch_usage(ctx, res); diff --git a/src/gallium/drivers/zink/zink_resource.h b/src/gallium/drivers/zink/zink_resource.h index cbe870cf5e3..4b6ac608994 100644 --- a/src/gallium/drivers/zink/zink_resource.h +++ b/src/gallium/drivers/zink/zink_resource.h @@ -35,6 +35,7 @@ struct zink_context; #include "util/u_range.h" #include "util/u_dynarray.h" +#include "zink_batch.h" #include "zink_descriptors.h" #include @@ -64,8 +65,8 @@ struct zink_resource_object { unsigned persistent_maps; //if nonzero, requires vkFlushMappedMemoryRanges during batch use struct zink_descriptor_refs desc_set_refs; - /* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */ - uint8_t batch_uses[5]; //ZINK_NUM_BATCHES + struct zink_batch_usage reads; + struct zink_batch_usage writes; bool is_buffer; bool host_visible; }; @@ -121,9 +122,6 @@ zink_get_depth_stencil_resources(struct pipe_resource *res, void zink_resource_setup_transfer_layouts(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst); -int -zink_get_resource_latest_batch_usage(struct zink_context *ctx, uint32_t batch_uses); - bool zink_resource_has_usage(struct zink_resource *res, enum zink_resource_access usage, enum zink_queue queue); diff --git a/src/gallium/drivers/zink/zink_surface.h b/src/gallium/drivers/zink/zink_surface.h index 5610ca08e5b..afed99d8143 100644 --- a/src/gallium/drivers/zink/zink_surface.h +++ b/src/gallium/drivers/zink/zink_surface.h @@ -25,7 +25,7 @@ #define ZINK_SURFACE_H #include "pipe/p_state.h" - +#include "zink_batch.h" #include struct pipe_context; @@ -35,7 +35,7 @@ struct zink_surface { VkImageViewCreateInfo ivci; VkImageView image_view; uint32_t hash; - uint32_t batch_uses; + struct zink_batch_usage batch_uses; }; static inline struct zink_surface *