diff --git a/src/gallium/drivers/zink/zink_pipeline.c b/src/gallium/drivers/zink/zink_pipeline.c index 1ea1413e738..f0896581059 100644 --- a/src/gallium/drivers/zink/zink_pipeline.c +++ b/src/gallium/drivers/zink/zink_pipeline.c @@ -233,11 +233,12 @@ zink_create_gfx_pipeline(struct zink_screen *screen, pci.stageCount = num_stages; VkPipeline pipeline; - if (vkCreateGraphicsPipelines(screen->dev, screen->pipeline_cache, 1, &pci, + if (vkCreateGraphicsPipelines(screen->dev, prog->base.pipeline_cache, 1, &pci, NULL, &pipeline) != VK_SUCCESS) { debug_printf("vkCreateGraphicsPipelines failed\n"); return VK_NULL_HANDLE; } + zink_screen_update_pipeline_cache(screen, &prog->base); return pipeline; } @@ -274,11 +275,12 @@ zink_create_compute_pipeline(struct zink_screen *screen, struct zink_compute_pro pci.stage = stage; VkPipeline pipeline; - if (vkCreateComputePipelines(screen->dev, screen->pipeline_cache, 1, &pci, + if (vkCreateComputePipelines(screen->dev, comp->base.pipeline_cache, 1, &pci, NULL, &pipeline) != VK_SUCCESS) { debug_printf("vkCreateComputePipelines failed\n"); return VK_NULL_HANDLE; } + zink_screen_update_pipeline_cache(screen, &comp->base); return pipeline; } diff --git a/src/gallium/drivers/zink/zink_program.c b/src/gallium/drivers/zink/zink_program.c index 0c3709c6177..c75193263c5 100644 --- a/src/gallium/drivers/zink/zink_program.c +++ b/src/gallium/drivers/zink/zink_program.c @@ -552,17 +552,22 @@ zink_create_gfx_program(struct zink_context *ctx, goto fail; } + struct mesa_sha1 sctx; + _mesa_sha1_init(&sctx); for (int i = 0; i < ZINK_SHADER_COUNT; ++i) { if (prog->modules[i]) { _mesa_set_add(stages[i]->programs, prog); zink_gfx_program_reference(screen, NULL, prog); + _mesa_sha1_update(&sctx, prog->shaders[i]->base.sha1, sizeof(prog->shaders[i]->base.sha1)); } } + _mesa_sha1_final(&sctx, prog->base.sha1); p_atomic_dec(&prog->base.reference.count); if (!screen->descriptor_program_init(ctx, &prog->base)) goto fail; + zink_screen_get_pipeline_cache(screen, &prog->base); return prog; fail: @@ -653,10 +658,12 @@ zink_create_compute_program(struct zink_context *ctx, struct zink_shader *shader _mesa_set_add(shader->programs, comp); comp->shader = shader; + memcpy(comp->base.sha1, shader->base.sha1, sizeof(shader->base.sha1)); if (!screen->descriptor_program_init(ctx, &comp->base)) goto fail; + zink_screen_get_pipeline_cache(screen, &comp->base); return comp; fail: @@ -792,6 +799,8 @@ zink_destroy_gfx_program(struct zink_screen *screen, _mesa_hash_table_destroy(prog->pipelines[i], NULL); } zink_shader_cache_reference(screen, &prog->shader_cache, NULL); + if (prog->base.pipeline_cache) + vkDestroyPipelineCache(screen->dev, prog->base.pipeline_cache, NULL); screen->descriptor_program_deinit(screen, &prog->base); ralloc_free(prog); @@ -817,6 +826,8 @@ zink_destroy_compute_program(struct zink_screen *screen, } _mesa_hash_table_destroy(comp->pipelines, NULL); zink_shader_cache_reference(screen, &comp->shader_cache, NULL); + if (comp->base.pipeline_cache) + vkDestroyPipelineCache(screen->dev, comp->base.pipeline_cache, NULL); screen->descriptor_program_deinit(screen, &comp->base); ralloc_free(comp); @@ -908,6 +919,7 @@ zink_get_gfx_pipeline(struct zink_context *ctx, entry = _mesa_hash_table_search_pre_hashed(prog->pipelines[vkmode], state->final_hash, state); if (!entry) { + util_queue_fence_wait(&prog->base.cache_fence); VkPipeline pipeline = zink_create_gfx_pipeline(screen, prog, state, vkmode); if (pipeline == VK_NULL_HANDLE) @@ -946,6 +958,7 @@ zink_get_compute_pipeline(struct zink_screen *screen, entry = _mesa_hash_table_search_pre_hashed(comp->pipelines, state->hash, state); if (!entry) { + util_queue_fence_wait(&comp->base.cache_fence); VkPipeline pipeline = zink_create_compute_pipeline(screen, comp, state); if (pipeline == VK_NULL_HANDLE) diff --git a/src/gallium/drivers/zink/zink_program.h b/src/gallium/drivers/zink/zink_program.h index 7f7ec6263c5..6015d7c3b5a 100644 --- a/src/gallium/drivers/zink/zink_program.h +++ b/src/gallium/drivers/zink/zink_program.h @@ -73,6 +73,10 @@ struct zink_shader_cache { struct zink_program { struct pipe_reference reference; + unsigned char sha1[20]; + struct util_queue_fence cache_fence; + VkPipelineCache pipeline_cache; + size_t pipeline_cache_size; struct zink_batch_usage *batch_uses; bool is_compute; diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index 6e117506d95..72bfd783406 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -151,30 +151,75 @@ disk_cache_init(struct zink_screen *screen) snprintf(buf, sizeof(buf), "zink_%x04x", screen->info.props.vendorID); screen->disk_cache = disk_cache_create(buf, screen->info.props.deviceName, 0); - if (screen->disk_cache) - disk_cache_compute_key(screen->disk_cache, buf, strlen(buf), screen->disk_cache_key); + if (screen->disk_cache) { + util_queue_init(&screen->cache_put_thread, "zcq", 8, 1, UTIL_QUEUE_INIT_RESIZE_IF_FULL, screen); + util_queue_init(&screen->cache_get_thread, "zcfq", 8, 4, UTIL_QUEUE_INIT_RESIZE_IF_FULL, screen); + } #endif } -void -zink_screen_update_pipeline_cache(struct zink_screen *screen) -{ - size_t size = 0; +static void +cache_put_job(void *data, void *gdata, int thread_index) +{ + struct zink_program *pg = data; + struct zink_screen *screen = gdata; + size_t size = 0; + if (vkGetPipelineCacheData(screen->dev, pg->pipeline_cache, &size, NULL) != VK_SUCCESS) + return; + if (pg->pipeline_cache_size == size) + return; + void *pipeline_data = malloc(size); + if (!pipeline_data) + return; + if (vkGetPipelineCacheData(screen->dev, pg->pipeline_cache, &size, pipeline_data) == VK_SUCCESS) { + pg->pipeline_cache_size = size; + + cache_key key; + disk_cache_compute_key(screen->disk_cache, pg->sha1, sizeof(pg->sha1), key); + disk_cache_put_nocopy(screen->disk_cache, key, pipeline_data, size, NULL); + } +} + +void +zink_screen_update_pipeline_cache(struct zink_screen *screen, struct zink_program *pg) +{ + util_queue_fence_init(&pg->cache_fence); if (!screen->disk_cache) return; - if (vkGetPipelineCacheData(screen->dev, screen->pipeline_cache, &size, NULL) != VK_SUCCESS) + + util_queue_add_job(&screen->cache_put_thread, pg, NULL, cache_put_job, NULL, 0); +} + +static void +cache_get_job(void *data, void *gdata, int thread_index) +{ + struct zink_program *pg = data; + struct zink_screen *screen = gdata; + + VkPipelineCacheCreateInfo pcci; + pcci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; + pcci.pNext = NULL; + pcci.flags = screen->info.have_EXT_pipeline_creation_cache_control ? VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT : 0; + pcci.initialDataSize = 0; + pcci.pInitialData = NULL; + + cache_key key; + disk_cache_compute_key(screen->disk_cache, pg->sha1, sizeof(pg->sha1), key); + pcci.pInitialData = disk_cache_get(screen->disk_cache, key, &pg->pipeline_cache_size); + pcci.initialDataSize = pg->pipeline_cache_size; + vkCreatePipelineCache(screen->dev, &pcci, NULL, &pg->pipeline_cache); + free((void*)pcci.pInitialData); +} + +void +zink_screen_get_pipeline_cache(struct zink_screen *screen, struct zink_program *pg) +{ + util_queue_fence_init(&pg->cache_fence); + if (!screen->disk_cache) return; - if (screen->pipeline_cache_size == size) - return; - void *data = malloc(size); - if (!data) - return; - if (vkGetPipelineCacheData(screen->dev, screen->pipeline_cache, &size, data) == VK_SUCCESS) { - screen->pipeline_cache_size = size; - disk_cache_put(screen->disk_cache, screen->disk_cache_key, data, size, NULL); - } - free(data); + + util_queue_add_job(&screen->cache_get_thread, pg, &pg->cache_fence, cache_get_job, NULL, 0); } static int @@ -1029,10 +1074,14 @@ zink_destroy_screen(struct pipe_screen *pscreen) simple_mtx_destroy(&screen->framebuffer_mtx); u_transfer_helper_destroy(pscreen->transfer_helper); - zink_screen_update_pipeline_cache(screen); #ifdef ENABLE_SHADER_CACHE - if (screen->disk_cache) + if (screen->disk_cache) { + util_queue_finish(&screen->cache_put_thread); + util_queue_finish(&screen->cache_get_thread); disk_cache_wait_for_idle(screen->disk_cache); + util_queue_destroy(&screen->cache_put_thread); + util_queue_destroy(&screen->cache_get_thread); + } #endif disk_cache_destroy(screen->disk_cache); simple_mtx_lock(&screen->mem_cache_mtx); @@ -1041,7 +1090,6 @@ zink_destroy_screen(struct pipe_screen *pscreen) _mesa_hash_table_destroy(screen->resource_mem_cache, NULL); simple_mtx_unlock(&screen->mem_cache_mtx); simple_mtx_destroy(&screen->mem_cache_mtx); - vkDestroyPipelineCache(screen->dev, screen->pipeline_cache, NULL); util_live_shader_cache_deinit(&screen->shaders); @@ -1737,20 +1785,6 @@ zink_internal_create_screen(const struct pipe_screen_config *config) populate_format_props(screen); pre_hash_descriptor_states(screen); - VkPipelineCacheCreateInfo pcci; - pcci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; - pcci.pNext = NULL; - /* we're single-threaded now, so we don't need synchronization */ - pcci.flags = screen->info.have_EXT_pipeline_creation_cache_control ? VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT : 0; - pcci.initialDataSize = 0; - pcci.pInitialData = NULL; - if (screen->disk_cache) { - pcci.pInitialData = disk_cache_get(screen->disk_cache, screen->disk_cache_key, &screen->pipeline_cache_size); - pcci.initialDataSize = screen->pipeline_cache_size; - } - vkCreatePipelineCache(screen->dev, &pcci, NULL, &screen->pipeline_cache); - free((void*)pcci.pInitialData); - slab_create_parent(&screen->transfer_pool, sizeof(struct zink_transfer), 16); #if WITH_XMLCONFIG diff --git a/src/gallium/drivers/zink/zink_screen.h b/src/gallium/drivers/zink/zink_screen.h index 6b10c32215d..7edeabb7869 100644 --- a/src/gallium/drivers/zink/zink_screen.h +++ b/src/gallium/drivers/zink/zink_screen.h @@ -81,10 +81,9 @@ struct zink_screen { simple_mtx_t bufferview_mtx; struct slab_parent_pool transfer_pool; - VkPipelineCache pipeline_cache; - size_t pipeline_cache_size; struct disk_cache *disk_cache; - cache_key disk_cache_key; + struct util_queue cache_put_thread; + struct util_queue cache_get_thread; struct util_live_shader_cache shaders; @@ -235,7 +234,10 @@ zink_is_depth_format_supported(struct zink_screen *screen, VkFormat format); #define GET_PROC_ADDR_INSTANCE_LOCAL(instance, x) PFN_vk##x vk_##x = (PFN_vk##x)vkGetInstanceProcAddr(instance, "vk"#x) void -zink_screen_update_pipeline_cache(struct zink_screen *screen); +zink_screen_update_pipeline_cache(struct zink_screen *screen, struct zink_program *pg); + +void +zink_screen_get_pipeline_cache(struct zink_screen *screen, struct zink_program *pg); void zink_screen_init_descriptor_funcs(struct zink_screen *screen, bool fallback);