diff --git a/src/gallium/drivers/zink/zink_batch.c b/src/gallium/drivers/zink/zink_batch.c index 3afe9610a5c..705cd53be4f 100644 --- a/src/gallium/drivers/zink/zink_batch.c +++ b/src/gallium/drivers/zink/zink_batch.c @@ -95,6 +95,7 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) zink_screen_update_last_finished(screen, bs->fence.batch_id); bs->submit_count++; bs->fence.batch_id = 0; + bs->usage.usage = 0; bs->draw_count = bs->compute_count = 0; } @@ -278,6 +279,8 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch) { zink_reset_batch(ctx, batch); + batch->state->usage.unflushed = true; + VkCommandBufferBeginInfo cbbi = {0}; cbbi.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; cbbi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; @@ -290,7 +293,7 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch) batch->state->fence.completed = false; if (ctx->last_fence) { struct zink_batch_state *last_state = zink_batch_state(ctx->last_fence); - batch->last_batch_id = last_state->fence.batch_id; + batch->last_batch_usage = &last_state->usage; } else { if (zink_screen(ctx->base.screen)->threaded) util_queue_init(&batch->flush_queue, "zfq", 8, 1, UTIL_QUEUE_INIT_RESIZE_IF_FULL, NULL); @@ -315,7 +318,18 @@ static void submit_queue(void *data, void *gdata, int thread_index) { struct zink_batch_state *bs = data; + struct zink_context *ctx = bs->ctx; + struct zink_screen *screen = zink_screen(ctx->base.screen); VkSubmitInfo si = {0}; + + simple_mtx_lock(&ctx->batch_mtx); + while (!bs->fence.batch_id) + bs->fence.batch_id = p_atomic_inc_return(&screen->curr_batch); + _mesa_hash_table_insert_pre_hashed(&ctx->batch_states, bs->fence.batch_id, (void*)(uintptr_t)bs->fence.batch_id, bs); + bs->usage.usage = bs->fence.batch_id; + bs->usage.unflushed = false; + simple_mtx_unlock(&ctx->batch_mtx); + uint64_t batch_id = bs->fence.batch_id; si.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; si.waitSemaphoreCount = 0; @@ -337,7 +351,7 @@ submit_queue(void *data, void *gdata, int thread_index) tsi.signalSemaphoreValueCount = 1; tsi.pSignalSemaphoreValues = &batch_id; si.signalSemaphoreCount = 1; - si.pSignalSemaphores = &zink_screen(bs->ctx->base.screen)->sem; + si.pSignalSemaphores = &screen->sem; } struct wsi_memory_signal_submit_info mem_signal = { @@ -345,7 +359,7 @@ submit_queue(void *data, void *gdata, int thread_index) .pNext = si.pNext, }; - if (bs->flush_res && zink_screen(bs->ctx->base.screen)->needs_mesa_flush_wsi) { + if (bs->flush_res && screen->needs_mesa_flush_wsi) { mem_signal.memory = bs->flush_res->scanout_obj ? bs->flush_res->scanout_obj->mem : bs->flush_res->obj->mem; si.pNext = &mem_signal; } @@ -518,11 +532,8 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch) vkFlushMappedMemoryRanges(screen->dev, 1, &range); } - simple_mtx_lock(&ctx->batch_mtx); - ctx->last_fence = &batch->state->fence; - _mesa_hash_table_insert_pre_hashed(&ctx->batch_states, batch->state->fence.batch_id, (void*)(uintptr_t)batch->state->fence.batch_id, batch->state); - simple_mtx_unlock(&ctx->batch_mtx); ctx->resource_size += batch->state->resource_size; + ctx->last_fence = &batch->state->fence; if (screen->device_lost) return; @@ -550,20 +561,20 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil); /* if the resource already has usage of any sort set for this batch, we can skip hashing */ - if (!zink_batch_usage_matches(&res->obj->reads, batch->state) && - !zink_batch_usage_matches(&res->obj->writes, batch->state)) { + if (!zink_batch_usage_matches(res->obj->reads, batch->state) && + !zink_batch_usage_matches(res->obj->writes, batch->state)) { bool found = false; _mesa_set_search_and_add(batch->state->fence.resources, res->obj, &found); if (!found) { pipe_reference(NULL, &res->obj->reference); - if (!batch->last_batch_id || res->obj->reads.usage != batch->last_batch_id) + if (!batch->last_batch_usage || res->obj->reads != batch->last_batch_usage) /* only add resource usage if it's "new" usage, though this only checks the most recent usage * and not all pending usages */ batch->state->resource_size += res->obj->size; if (stencil) { pipe_reference(NULL, &stencil->obj->reference); - if (!batch->last_batch_id || stencil->obj->reads.usage != batch->last_batch_id) + if (!batch->last_batch_usage || stencil->obj->reads != batch->last_batch_usage) batch->state->resource_size += stencil->obj->size; } } @@ -586,10 +597,10 @@ zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource } bool -batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u) +batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage **u) { bool found = false; - if (u->usage == batch->state->fence.batch_id) + if (*u == &batch->state->usage) return false; _mesa_set_search_and_add(s, ptr, &found); assert(!found); @@ -661,11 +672,13 @@ zink_batch_usage_check_completion(struct zink_context *ctx, const struct zink_ba { if (!zink_batch_usage_exists(u)) return true; + if (zink_batch_usage_is_unflushed(u)) + return false; return zink_check_batch_completion(ctx, u->usage); } void -zink_batch_usage_wait(struct zink_context *ctx, const struct zink_batch_usage *u) +zink_batch_usage_wait(struct zink_context *ctx, struct zink_batch_usage *u) { if (!zink_batch_usage_exists(u)) return; diff --git a/src/gallium/drivers/zink/zink_batch.h b/src/gallium/drivers/zink/zink_batch.h index a15747d21f0..8df4385fa38 100644 --- a/src/gallium/drivers/zink/zink_batch.h +++ b/src/gallium/drivers/zink/zink_batch.h @@ -45,19 +45,20 @@ struct zink_sampler_view; struct zink_surface; struct zink_batch_usage { - /* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */ uint32_t usage; + bool unflushed; }; /* not real api don't use */ bool -batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage *u); +batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr, struct zink_batch_usage **u); struct zink_batch_state { struct zink_fence fence; struct pipe_reference reference; unsigned draw_count; + struct zink_batch_usage usage; struct zink_context *ctx; VkCommandPool cmdpool; VkCommandBuffer cmdbuf; @@ -97,7 +98,7 @@ struct zink_batch_state { struct zink_batch { struct zink_batch_state *state; - uint32_t last_batch_id; + struct zink_batch_usage *last_batch_usage; struct util_queue flush_queue; //TODO: move to wsi bool has_work; @@ -175,34 +176,40 @@ zink_batch_state_reference(struct zink_screen *screen, if (dst) *dst = src; } -static inline void -zink_batch_usage_unset(struct zink_batch_usage *u, struct zink_batch_state *bs) +static inline bool +zink_batch_usage_is_unflushed(const struct zink_batch_usage *u) { - p_atomic_cmpxchg(&u->usage, bs->fence.batch_id, 0); + return u && u->unflushed; } static inline void -zink_batch_usage_set(struct zink_batch_usage *u, struct zink_batch_state *bs) +zink_batch_usage_unset(struct zink_batch_usage **u, struct zink_batch_state *bs) { - u->usage = bs->fence.batch_id; + (void)p_atomic_cmpxchg(u, &bs->usage, NULL); +} + +static inline void +zink_batch_usage_set(struct zink_batch_usage **u, struct zink_batch_state *bs) +{ + *u = &bs->usage; } static inline bool zink_batch_usage_matches(const struct zink_batch_usage *u, const struct zink_batch_state *bs) { - return u->usage == bs->fence.batch_id; + return u == &bs->usage; } static inline bool -zink_batch_usage_exists(struct zink_batch_usage *u) +zink_batch_usage_exists(const struct zink_batch_usage *u) { - uint32_t usage = p_atomic_read(&u->usage); - return !!usage; + return u && (u->usage || u->unflushed); } bool zink_batch_usage_check_completion(struct zink_context *ctx, const struct zink_batch_usage *u); void -zink_batch_usage_wait(struct zink_context *ctx, const struct zink_batch_usage *u); +zink_batch_usage_wait(struct zink_context *ctx, struct zink_batch_usage *u); + #endif diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 377cfd0cc14..ca912f1f566 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -56,15 +56,6 @@ #define XXH_INLINE_ALL #include "util/xxhash.h" -static void -incr_curr_batch(struct zink_context *ctx) -{ - struct zink_screen *screen = zink_screen(ctx->base.screen); - ctx->curr_batch = p_atomic_inc_return(&screen->curr_batch); - if (!ctx->curr_batch) //never use batchid 0 - incr_curr_batch(ctx); -} - static void calc_descriptor_hash_sampler_state(struct zink_sampler_state *sampler_state) { @@ -1804,8 +1795,6 @@ flush_batch(struct zink_context *ctx, bool sync) if (ctx->batch.state->is_device_lost) { check_device_lost(ctx); } else { - incr_curr_batch(ctx); - zink_start_batch(ctx, batch); if (zink_screen(ctx->base.screen)->info.have_EXT_transform_feedback && ctx->num_so_targets) ctx->dirty_so_targets = true; @@ -2539,10 +2528,8 @@ zink_wait_on_batch(struct zink_context *ctx, uint32_t batch_id) bool zink_check_batch_completion(struct zink_context *ctx, uint32_t batch_id) { - assert(batch_id); - struct zink_batch_state *bs = ctx->batch.state; - assert(bs); - if (bs->fence.batch_id == batch_id) + assert(ctx->batch.state); + if (!batch_id) /* not submitted yet */ return false; @@ -3161,8 +3148,8 @@ zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsi struct zink_screen *screen = zink_screen(pctx->screen); /* if any current usage exists, flush the queue */ - if (res->obj->reads.usage == ctx->curr_batch || - res->obj->writes.usage == ctx->curr_batch) + if (zink_batch_usage_is_unflushed(res->obj->reads) || + zink_batch_usage_is_unflushed(res->obj->writes)) zink_flush_queue(ctx); VkBindSparseInfo sparse; @@ -3264,45 +3251,38 @@ zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resou zink_resource_rebind(zink_context(pctx), d); } +ALWAYS_INLINE static bool +is_usage_completed(struct zink_screen *screen, const struct zink_batch_usage *u) +{ + if (!zink_batch_usage_exists(u)) + return true; + if (zink_batch_usage_is_unflushed(u)) + return false; + /* check fastpath first */ + if (zink_screen_check_last_finished(screen, u->usage)) + return true; + /* if we have timelines, do a quick check */ + if (screen->info.have_KHR_timeline_semaphore) + return zink_screen_timeline_wait(screen, u->usage, 0); + + /* otherwise assume busy */ + return false; +} + static bool zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage) { struct zink_screen *screen = zink_screen(pscreen); struct zink_resource *res = zink_resource(pres); - uint32_t reads = 0, writes = 0; + const struct zink_batch_usage *reads = NULL, *writes = NULL; if (((usage & (PIPE_MAP_READ | PIPE_MAP_WRITE)) == (PIPE_MAP_READ | PIPE_MAP_WRITE)) || usage & PIPE_MAP_WRITE) { - reads = p_atomic_read(&res->obj->reads.usage); - writes = p_atomic_read(&res->obj->writes.usage); + reads = res->obj->reads; + writes = res->obj->writes; } else if (usage & PIPE_MAP_READ) - writes = p_atomic_read(&res->obj->writes.usage); + writes = res->obj->writes; - /* get latest usage accounting for 32bit int rollover: - * a rollover is detected if there are reads and writes, - * but one of the values is over UINT32_MAX/2 while the other is under, - * as it is impossible for this many unflushed batch states to ever - * exist at any given time - */ - uint32_t last; - - if (reads && writes) - last = abs((int)reads - (int)writes) > UINT32_MAX / 2 ? MIN2(reads, writes) : MAX2(reads, writes); - else - last = reads ? reads : writes; - - if (!last) - return false; - - /* check fastpath first */ - if (zink_screen_check_last_finished(screen, last)) - return false; - - /* if we have timelines, do a quick check */ - if (screen->info.have_KHR_timeline_semaphore) - return !zink_screen_timeline_wait(screen, last, 0); - - /* otherwise assume busy */ - return true; + return !is_usage_completed(screen, reads) || !is_usage_completed(screen, writes); } static void @@ -3484,7 +3464,6 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) ctx->have_timelines = screen->info.have_KHR_timeline_semaphore; simple_mtx_init(&ctx->batch_mtx, mtx_plain); - incr_curr_batch(ctx); zink_start_batch(ctx, &ctx->batch); if (!ctx->batch.state) goto fail; diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index fb5e08faac8..7ef543f8c43 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -69,7 +69,7 @@ struct zink_sampler_state { VkSampler sampler; uint32_t hash; struct zink_descriptor_refs desc_set_refs; - struct zink_batch_usage batch_uses; + struct zink_batch_usage *batch_uses; bool custom_border_color; }; @@ -78,7 +78,7 @@ struct zink_buffer_view { VkBufferViewCreateInfo bvci; VkBufferView buffer_view; uint32_t hash; - struct zink_batch_usage batch_uses; + struct zink_batch_usage *batch_uses; struct zink_descriptor_refs desc_set_refs; }; diff --git a/src/gallium/drivers/zink/zink_descriptors.c b/src/gallium/drivers/zink/zink_descriptors.c index 67d6db6ea1d..90e0ceb2900 100644 --- a/src/gallium/drivers/zink/zink_descriptors.c +++ b/src/gallium/drivers/zink/zink_descriptors.c @@ -60,7 +60,7 @@ struct zink_descriptor_set { bool punted; bool recycled; struct zink_descriptor_state_key key; - struct zink_batch_usage batch_uses; + struct zink_batch_usage *batch_uses; #ifndef NDEBUG /* for extra debug asserts */ unsigned num_resources; @@ -622,7 +622,7 @@ allocate_desc_set(struct zink_context *ctx, struct zink_program *pg, enum zink_d pipe_reference_init(&zds->reference, 1); zds->pool = pool; zds->hash = 0; - zds->batch_uses.usage = 0; + zds->batch_uses = NULL; zds->invalid = true; zds->punted = zds->recycled = false; #ifndef NDEBUG @@ -731,7 +731,7 @@ zink_descriptor_set_get(struct zink_context *ctx, zds->recycled = false; } if (zds->invalid) { - if (zink_batch_usage_exists(&zds->batch_uses)) + if (zink_batch_usage_exists(zds->batch_uses)) punt_invalid_set(zds, NULL); else /* this set is guaranteed to be in pool->alloc_desc_sets */ @@ -746,7 +746,7 @@ zink_descriptor_set_get(struct zink_context *ctx, bool recycled = false, punted = false; if (he) { zds = (void*)he->data; - if (zds->invalid && zink_batch_usage_exists(&zds->batch_uses)) { + if (zds->invalid && zink_batch_usage_exists(zds->batch_uses)) { punt_invalid_set(zds, he); zds = NULL; punted = true; diff --git a/src/gallium/drivers/zink/zink_fence.c b/src/gallium/drivers/zink/zink_fence.c index 338ffcdf217..8e634be024c 100644 --- a/src/gallium/drivers/zink/zink_fence.c +++ b/src/gallium/drivers/zink/zink_fence.c @@ -158,8 +158,9 @@ zink_vkfence_wait(struct zink_screen *screen, struct zink_fence *fence, uint64_t if (success) { p_atomic_set(&fence->completed, true); - zink_fence_clear_resources(screen, fence); + zink_batch_state(fence)->usage.usage = 0; zink_screen_update_last_finished(screen, fence->batch_id); + zink_fence_clear_resources(screen, fence); } return success; } diff --git a/src/gallium/drivers/zink/zink_program.h b/src/gallium/drivers/zink/zink_program.h index 56feaa9a344..7f7ec6263c5 100644 --- a/src/gallium/drivers/zink/zink_program.h +++ b/src/gallium/drivers/zink/zink_program.h @@ -73,7 +73,7 @@ struct zink_shader_cache { struct zink_program { struct pipe_reference reference; - struct zink_batch_usage batch_uses; + struct zink_batch_usage *batch_uses; bool is_compute; struct zink_program_descriptor_data *dd; diff --git a/src/gallium/drivers/zink/zink_query.c b/src/gallium/drivers/zink/zink_query.c index 64fa30be246..9b51ecd2bc8 100644 --- a/src/gallium/drivers/zink/zink_query.c +++ b/src/gallium/drivers/zink/zink_query.c @@ -46,7 +46,7 @@ struct zink_query { bool have_gs[NUM_QUERIES]; /* geometry shaders use GEOMETRY_SHADER_PRIMITIVES_BIT */ bool have_xfb[NUM_QUERIES]; /* xfb was active during this query */ - struct zink_batch_usage batch_id; //batch that the query was started in + struct zink_batch_usage *batch_id; //batch that the query was started in struct list_head buffers; struct zink_query_buffer *curr_qbo; @@ -780,7 +780,7 @@ zink_get_query_result(struct pipe_context *pctx, if (query->needs_update) update_qbo(ctx, query); - if (query->batch_id.usage == ctx->curr_batch) { + if (zink_batch_usage_is_unflushed(query->batch_id)) { if (!threaded_query(q)->flushed) pctx->flush(pctx, NULL, 0); if (!wait) diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index ce2c6b37c80..7e20a79c1f0 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -77,8 +77,8 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object static uint32_t get_resource_usage(struct zink_resource *res) { - bool reads = zink_batch_usage_exists(&res->obj->reads); - bool writes = zink_batch_usage_exists(&res->obj->writes); + bool reads = zink_batch_usage_exists(res->obj->reads); + bool writes = zink_batch_usage_exists(res->obj->writes); uint32_t batch_uses = 0; if (reads) batch_uses |= ZINK_RESOURCE_ACCESS_READ; @@ -867,8 +867,8 @@ buffer_transfer_map(struct zink_context *ctx, struct zink_resource *res, unsigne */ if (!res->obj->host_visible || - !zink_batch_usage_check_completion(ctx, &res->obj->reads) || - !zink_batch_usage_check_completion(ctx, &res->obj->writes)) { + !zink_batch_usage_check_completion(ctx, res->obj->reads) || + !zink_batch_usage_check_completion(ctx, res->obj->writes)) { /* Do a wait-free write-only transfer using a temporary buffer. */ unsigned offset; @@ -896,7 +896,7 @@ buffer_transfer_map(struct zink_context *ctx, struct zink_resource *res, unsigne /* sparse/device-local will always need to wait since it has to copy */ if (!res->obj->host_visible) return NULL; - if (!zink_batch_usage_check_completion(ctx, &res->obj->writes)) + if (!zink_batch_usage_check_completion(ctx, res->obj->writes)) return NULL; } else if (!res->obj->host_visible) { zink_fence_wait(&ctx->base); @@ -909,7 +909,7 @@ buffer_transfer_map(struct zink_context *ctx, struct zink_resource *res, unsigne res = staging_res; zink_fence_wait(&ctx->base); } else - zink_batch_usage_wait(ctx, &res->obj->writes); + zink_batch_usage_wait(ctx, res->obj->writes); } if (!ptr) { @@ -1036,7 +1036,7 @@ zink_transfer_map(struct pipe_context *pctx, if (usage & PIPE_MAP_WRITE) zink_fence_wait(pctx); else - zink_batch_usage_wait(ctx, &res->obj->writes); + zink_batch_usage_wait(ctx, res->obj->writes); } VkImageSubresource isr = { res->aspect, diff --git a/src/gallium/drivers/zink/zink_resource.h b/src/gallium/drivers/zink/zink_resource.h index c64e5d59ea5..219bcaa5b6d 100644 --- a/src/gallium/drivers/zink/zink_resource.h +++ b/src/gallium/drivers/zink/zink_resource.h @@ -75,8 +75,8 @@ struct zink_resource_object { unsigned persistent_maps; //if nonzero, requires vkFlushMappedMemoryRanges during batch use struct zink_descriptor_refs desc_set_refs; - struct zink_batch_usage reads; - struct zink_batch_usage writes; + struct zink_batch_usage *reads; + struct zink_batch_usage *writes; void *map; bool is_buffer; bool host_visible; diff --git a/src/gallium/drivers/zink/zink_surface.h b/src/gallium/drivers/zink/zink_surface.h index e9004255220..8b682c4de3a 100644 --- a/src/gallium/drivers/zink/zink_surface.h +++ b/src/gallium/drivers/zink/zink_surface.h @@ -37,7 +37,7 @@ struct zink_surface { VkImageView simage_view;//old iview after storage replacement/rebind void *obj; //backing resource object uint32_t hash; - struct zink_batch_usage batch_uses; + struct zink_batch_usage *batch_uses; struct util_dynarray framebuffer_refs; struct zink_descriptor_refs desc_set_refs; };