diff --git a/src/gallium/drivers/zink/zink_program.c b/src/gallium/drivers/zink/zink_program.c index 1f3796757e5..13c1448729c 100644 --- a/src/gallium/drivers/zink/zink_program.c +++ b/src/gallium/drivers/zink/zink_program.c @@ -567,6 +567,10 @@ zink_create_gfx_program(struct zink_context *ctx, if (!prog->base.desc_sets) goto fail; + prog->base.free_desc_sets = _mesa_hash_table_create(NULL, NULL, _mesa_key_pointer_equal); + if (!prog->base.free_desc_sets) + goto fail; + util_dynarray_init(&prog->base.alloc_desc_sets, NULL); return prog; @@ -672,6 +676,10 @@ zink_create_compute_program(struct zink_context *ctx, struct zink_shader *shader if (!comp->base.desc_sets) goto fail; + comp->base.free_desc_sets = _mesa_hash_table_create(NULL, NULL, _mesa_key_pointer_equal); + if (!comp->base.free_desc_sets) + goto fail; + util_dynarray_init(&comp->base.alloc_desc_sets, NULL); return comp; @@ -690,6 +698,15 @@ desc_set_invalidate_resources(struct zink_program *pg, struct zink_descriptor_se zds->valid = false; } +static bool +get_invalidated_desc_set(struct zink_descriptor_set *zds) +{ + if (zds->valid) + return false; + return p_atomic_read(&zds->reference.count) == 1; +} + + static struct zink_descriptor_set * allocate_desc_set(struct zink_screen *screen, struct zink_program *pg, unsigned descs_used) { @@ -741,6 +758,7 @@ zink_program_allocate_desc_set(struct zink_context *ctx, struct zink_descriptor_set *zds; struct zink_screen *screen = zink_screen(ctx->base.screen); struct hash_entry *he = _mesa_hash_table_search_pre_hashed(pg->desc_sets, hash, (void*)(uintptr_t)hash); + bool recycled = false; if (he) { zds = (void*)he->data; /* this shouldn't happen, but if we somehow get a cache hit on an invalidated, active desc set then @@ -748,9 +766,18 @@ zink_program_allocate_desc_set(struct zink_context *ctx, */ assert(zds->valid); } + if (!he) { + he = _mesa_hash_table_search_pre_hashed(pg->free_desc_sets, hash, (void*)(uintptr_t)hash); + recycled = true; + } if (he) { zds = (void*)he->data; *cache_hit = zds->valid; + if (recycled) { + /* need to migrate this entry back to the in-use hash */ + _mesa_hash_table_remove(pg->free_desc_sets, he); + goto out; + } if (zink_batch_add_desc_set(batch, pg, hash, zds)) batch->descs_used += pg->num_descriptors; return zds; @@ -759,11 +786,26 @@ zink_program_allocate_desc_set(struct zink_context *ctx, if (util_dynarray_num_elements(&pg->alloc_desc_sets, struct zink_descriptor_set *)) { /* grab one off the allocated array */ zds = util_dynarray_pop(&pg->alloc_desc_sets, struct zink_descriptor_set *); - //printf("POP%u %p %u\n", batch->batch_id, pg, hash); goto out; } - unsigned descs_used = _mesa_hash_table_num_entries(pg->desc_sets); + if (_mesa_hash_table_num_entries(pg->free_desc_sets)) { + /* try for an invalidated set first */ + unsigned count = 0; + hash_table_foreach(pg->free_desc_sets, he) { + struct zink_descriptor_set *tmp = he->data; + if ((count >= 100 && tmp->reference.count == 1) || get_invalidated_desc_set(he->data)) { + zds = tmp; + assert(p_atomic_read(&zds->reference.count) == 1); + desc_set_invalidate_resources(pg, zds); + _mesa_hash_table_remove(pg->free_desc_sets, he); + goto out; + } else + count++; + } + } + + unsigned descs_used = _mesa_hash_table_num_entries(pg->desc_sets) + _mesa_hash_table_num_entries(pg->free_desc_sets); if (descs_used + pg->num_descriptors > ZINK_DEFAULT_MAX_DESCS) { batch = zink_flush_batch(ctx, batch); zink_batch_reference_program(batch, pg); @@ -771,9 +813,6 @@ zink_program_allocate_desc_set(struct zink_context *ctx, } zds = allocate_desc_set(screen, pg, descs_used); - //printf("NEW%u %p %u (%u total - %u / %u)\n", batch->batch_id, pg, hash, - //_mesa_hash_table_num_entries(pg->desc_sets), - //_mesa_hash_table_num_entries(pg->desc_sets)); out: zds->valid = true; _mesa_hash_table_insert_pre_hashed(pg->desc_sets, hash, (void*)(uintptr_t)hash, zds); @@ -795,13 +834,15 @@ zink_program_recycle_desc_set(struct zink_program *pg, uint32_t hash, struct zin /* desc sets can be used multiple times in the same batch */ return; _mesa_hash_table_remove(pg->desc_sets, he); - util_dynarray_append(&pg->alloc_desc_sets, struct zink_descriptor_set *, zds); + if (zds->valid) + _mesa_hash_table_insert_pre_hashed(pg->free_desc_sets, hash, (void*)(uintptr_t)hash, zds); + else + util_dynarray_append(&pg->alloc_desc_sets, struct zink_descriptor_set *, zds); } static void zink_program_clear_desc_sets(struct zink_program *pg, struct hash_table *ht) { - //printf("CLEAR %p\n", pg); hash_table_foreach(ht, entry) { struct zink_descriptor_set *zds = entry->data; desc_set_invalidate_resources(pg, zds); @@ -849,6 +890,8 @@ zink_destroy_gfx_program(struct zink_screen *screen, zink_program_clear_desc_sets((struct zink_program*)prog, prog->base.desc_sets); _mesa_hash_table_destroy(prog->base.desc_sets, NULL); + zink_program_clear_desc_sets((struct zink_program*)prog, prog->base.free_desc_sets); + _mesa_hash_table_destroy(prog->base.free_desc_sets, NULL); if (prog->base.descpool) vkDestroyDescriptorPool(screen->dev, prog->base.descpool, NULL); @@ -883,6 +926,8 @@ zink_destroy_compute_program(struct zink_screen *screen, zink_program_clear_desc_sets((struct zink_program*)comp, comp->base.desc_sets); _mesa_hash_table_destroy(comp->base.desc_sets, NULL); + zink_program_clear_desc_sets((struct zink_program*)comp, comp->base.free_desc_sets); + _mesa_hash_table_destroy(comp->base.free_desc_sets, NULL); if (comp->base.descpool) vkDestroyDescriptorPool(screen->dev, comp->base.descpool, NULL); diff --git a/src/gallium/drivers/zink/zink_program.h b/src/gallium/drivers/zink/zink_program.h index ce3fa6f3fa4..0babd01eb66 100644 --- a/src/gallium/drivers/zink/zink_program.h +++ b/src/gallium/drivers/zink/zink_program.h @@ -74,6 +74,7 @@ struct zink_program { struct pipe_reference reference; struct hash_table *desc_sets; + struct hash_table *free_desc_sets; struct util_dynarray alloc_desc_sets; VkDescriptorPool descpool; VkDescriptorSetLayout dsl;