diff --git a/src/gallium/drivers/v3d/meson.build b/src/gallium/drivers/v3d/meson.build index b760ca5c025..d716c3b629c 100644 --- a/src/gallium/drivers/v3d/meson.build +++ b/src/gallium/drivers/v3d/meson.build @@ -26,6 +26,7 @@ files_libv3d = files( 'v3d_cl.h', 'v3d_context.c', 'v3d_context.h', + 'v3d_disk_cache.c', 'v3d_fence.c', 'v3d_formats.c', 'v3d_job.c', diff --git a/src/gallium/drivers/v3d/v3d_context.h b/src/gallium/drivers/v3d/v3d_context.h index d955cf8cccc..84c4ede9868 100644 --- a/src/gallium/drivers/v3d/v3d_context.h +++ b/src/gallium/drivers/v3d/v3d_context.h @@ -41,6 +41,7 @@ #include "broadcom/common/v3d_limits.h" #include "broadcom/simulator/v3d_simulator.h" +#include "broadcom/compiler/v3d_compiler.h" struct v3d_job; struct v3d_bo; @@ -794,6 +795,17 @@ void v3d_get_tile_buffer_size(bool is_msaa, uint32_t *tile_height, uint32_t *max_bpp); +#ifdef ENABLE_SHADER_CACHE +struct v3d_compiled_shader *v3d_disk_cache_retrieve(struct v3d_context *v3d, + const struct v3d_key *key); + +void v3d_disk_cache_store(struct v3d_context *v3d, + const struct v3d_key *key, + const struct v3d_compiled_shader *shader, + uint64_t *qpu_insts, + uint32_t qpu_size); +#endif /* ENABLE_SHADER_CACHE */ + #ifdef v3dX # include "v3dx_context.h" #else diff --git a/src/gallium/drivers/v3d/v3d_disk_cache.c b/src/gallium/drivers/v3d/v3d_disk_cache.c new file mode 100644 index 00000000000..67ac170f194 --- /dev/null +++ b/src/gallium/drivers/v3d/v3d_disk_cache.c @@ -0,0 +1,218 @@ +/* + * Copyright © 2022 Raspberry Pi Ltd + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/** + * V3D on-disk shader cache. + */ + +#include "v3d_context.h" + +#include "compiler/nir/nir_serialize.h" +#include "util/blob.h" +#include "util/u_upload_mgr.h" + +#ifdef ENABLE_SHADER_CACHE + +static uint32_t +v3d_key_size(gl_shader_stage stage) +{ + static const int key_size[] = { + [MESA_SHADER_VERTEX] = sizeof(struct v3d_vs_key), + [MESA_SHADER_GEOMETRY] = sizeof(struct v3d_gs_key), + [MESA_SHADER_FRAGMENT] = sizeof(struct v3d_fs_key), + [MESA_SHADER_COMPUTE] = sizeof(struct v3d_key), + }; + + assert(stage >= 0 && + stage < ARRAY_SIZE(key_size) && + key_size[stage]); + + return key_size[stage]; +} + +void v3d_disk_cache_init(struct v3d_screen *screen) +{ + char *renderer; + + ASSERTED int len = + asprintf(&renderer, "V3D %d.%d", + screen->devinfo.ver / 10, + screen->devinfo.ver % 10); + assert(len > 0); + + const struct build_id_note *note = + build_id_find_nhdr_for_addr(v3d_disk_cache_init); + assert(note && build_id_length(note) == 20); + + const uint8_t *id_sha1 = build_id_data(note); + assert(id_sha1); + + char timestamp[41]; + _mesa_sha1_format(timestamp, id_sha1); + + screen->disk_cache = disk_cache_create(renderer, timestamp, 0); +} + +static void +v3d_disk_cache_compute_key(struct disk_cache *cache, + const struct v3d_key *key, + cache_key cache_key) +{ + assert(cache); + + struct v3d_uncompiled_shader *uncompiled = key->shader_state; + assert(uncompiled->base.type == PIPE_SHADER_IR_NIR); + nir_shader *nir = uncompiled->base.ir.nir; + + struct blob blob; + blob_init(&blob); + + uint32_t ckey_size = v3d_key_size(nir->info.stage); + struct v3d_key *ckey = malloc(ckey_size); + memcpy(ckey, key, ckey_size); + ckey->shader_state = NULL; + + blob_write_bytes(&blob, ckey, ckey_size); + + nir_serialize(&blob, nir, true); + + disk_cache_compute_key(cache, blob.data, blob.size, cache_key); + + blob_finish(&blob); + free(ckey); +} + +struct v3d_compiled_shader * +v3d_disk_cache_retrieve(struct v3d_context *v3d, + const struct v3d_key *key) +{ + struct v3d_screen *screen = v3d->screen; + struct disk_cache *cache = screen->disk_cache; + + if (!cache) + return NULL; + + struct v3d_uncompiled_shader *uncompiled = key->shader_state; + assert(uncompiled->base.type == PIPE_SHADER_IR_NIR); + nir_shader *nir = uncompiled->base.ir.nir; + + cache_key cache_key; + v3d_disk_cache_compute_key(cache, key, cache_key); + + size_t buffer_size; + void *buffer = disk_cache_get(cache, cache_key, &buffer_size); + + if (!buffer) + return NULL; + + /* Load data */ + struct blob_reader blob; + blob_reader_init(&blob, buffer, buffer_size); + + uint32_t prog_data_size = v3d_prog_data_size(nir->info.stage); + const void *prog_data = blob_read_bytes(&blob, prog_data_size); + if (blob.overrun) + return NULL; + + uint32_t ulist_count = blob_read_uint32(&blob); + uint32_t ulist_contents_size = ulist_count * sizeof(enum quniform_contents); + const void *ulist_contents = blob_read_bytes(&blob, ulist_contents_size); + if (blob.overrun) + return NULL; + + uint32_t ulist_data_size = ulist_count * sizeof(uint32_t); + const void *ulist_data = blob_read_bytes(&blob, ulist_data_size); + if (blob.overrun) + return NULL; + + uint32_t qpu_size = blob_read_uint32(&blob); + const void *qpu_insts = + blob_read_bytes(&blob, qpu_size); + if (blob.overrun) + return NULL; + + /* Assemble data */ + struct v3d_compiled_shader *shader = rzalloc(NULL, struct v3d_compiled_shader); + + shader->prog_data.base = rzalloc_size(shader, prog_data_size); + memcpy(shader->prog_data.base, prog_data, prog_data_size); + + shader->prog_data.base->uniforms.count = ulist_count; + + shader->prog_data.base->uniforms.contents = + ralloc_array(shader->prog_data.base, enum quniform_contents, ulist_count); + memcpy(shader->prog_data.base->uniforms.contents, ulist_contents, ulist_contents_size); + + shader->prog_data.base->uniforms.data = + ralloc_array(shader->prog_data.base, uint32_t, ulist_count); + memcpy(shader->prog_data.base->uniforms.data, ulist_data, ulist_data_size); + + u_upload_data(v3d->state_uploader, 0, qpu_size, 8, + qpu_insts, &shader->offset, &shader->resource); + + free(buffer); + + return shader; +} + +void +v3d_disk_cache_store(struct v3d_context *v3d, + const struct v3d_key *key, + const struct v3d_compiled_shader *shader, + uint64_t *qpu_insts, + uint32_t qpu_size) +{ + struct v3d_screen *screen = v3d->screen; + struct disk_cache *cache = screen->disk_cache; + + if (!cache) + return; + + struct v3d_uncompiled_shader *uncompiled = key->shader_state; + assert(uncompiled->base.type == PIPE_SHADER_IR_NIR); + nir_shader *nir = uncompiled->base.ir.nir; + + cache_key cache_key; + v3d_disk_cache_compute_key(cache, key, cache_key); + + struct blob blob; + blob_init(&blob); + + blob_write_bytes(&blob, shader->prog_data.base, v3d_prog_data_size(nir->info.stage)); + uint32_t ulist_count = shader->prog_data.base->uniforms.count; + blob_write_uint32(&blob, ulist_count); + blob_write_bytes(&blob, + shader->prog_data.base->uniforms.contents, + ulist_count * sizeof(enum quniform_contents)); + blob_write_bytes(&blob, + shader->prog_data.base->uniforms.data, + ulist_count * sizeof(uint32_t)); + + blob_write_uint32(&blob, qpu_size); + blob_write_bytes(&blob, qpu_insts, qpu_size); + + disk_cache_put(cache, cache_key, blob.data, blob.size, NULL); +} + +#endif /* ENABLE_SHADER_CACHE */ + diff --git a/src/gallium/drivers/v3d/v3d_program.c b/src/gallium/drivers/v3d/v3d_program.c index 8737dde551b..fdcf4fc99a5 100644 --- a/src/gallium/drivers/v3d/v3d_program.c +++ b/src/gallium/drivers/v3d/v3d_program.c @@ -380,30 +380,42 @@ v3d_get_compiled_shader(struct v3d_context *v3d, if (entry) return entry->data; - struct v3d_compiled_shader *shader = - rzalloc(NULL, struct v3d_compiled_shader); - - int program_id = shader_state->program_id; int variant_id = p_atomic_inc_return(&shader_state->compiled_variant_count); - uint64_t *qpu_insts; - uint32_t shader_size; - qpu_insts = v3d_compile(v3d->screen->compiler, key, - &shader->prog_data.base, s, - v3d_shader_debug_output, - v3d, - program_id, variant_id, &shader_size); - ralloc_steal(shader, shader->prog_data.base); + struct v3d_compiled_shader *shader = NULL; - v3d_set_shader_uniform_dirty_flags(shader); +#ifdef ENABLE_SHADER_CACHE + shader = v3d_disk_cache_retrieve(v3d, key); +#endif - if (shader_size) { - u_upload_data(v3d->state_uploader, 0, shader_size, 8, - qpu_insts, &shader->offset, &shader->resource); + if (!shader) { + shader = rzalloc(NULL, struct v3d_compiled_shader); + + int program_id = shader_state->program_id; + uint64_t *qpu_insts; + uint32_t shader_size; + + qpu_insts = v3d_compile(v3d->screen->compiler, key, + &shader->prog_data.base, s, + v3d_shader_debug_output, + v3d, + program_id, variant_id, &shader_size); + ralloc_steal(shader, shader->prog_data.base); + + if (shader_size) { + u_upload_data(v3d->state_uploader, 0, shader_size, 8, + qpu_insts, &shader->offset, &shader->resource); + } + +#ifdef ENABLE_SHADER_CACHE + v3d_disk_cache_store(v3d, key, shader, qpu_insts, shader_size); +#endif + + free(qpu_insts); } - free(qpu_insts); + v3d_set_shader_uniform_dirty_flags(shader); if (ht) { struct v3d_key *dup_key; diff --git a/src/gallium/drivers/v3d/v3d_screen.c b/src/gallium/drivers/v3d/v3d_screen.c index 5b47c5a2d22..f3f0828fe69 100644 --- a/src/gallium/drivers/v3d/v3d_screen.c +++ b/src/gallium/drivers/v3d/v3d_screen.c @@ -83,6 +83,12 @@ v3d_screen_destroy(struct pipe_screen *pscreen) v3d_simulator_destroy(screen->sim_file); v3d_compiler_free(screen->compiler); + +#ifdef ENABLE_SHADER_CACHE + if (screen->disk_cache) + disk_cache_destroy(screen->disk_cache); +#endif + u_transfer_helper_destroy(pscreen->transfer_helper); close(screen->fd); @@ -868,6 +874,10 @@ v3d_screen_create(int fd, const struct pipe_screen_config *config, screen->compiler = v3d_compiler_init(&screen->devinfo); +#ifdef ENABLE_SHADER_CACHE + v3d_disk_cache_init(screen); +#endif + pscreen->get_name = v3d_screen_get_name; pscreen->get_vendor = v3d_screen_get_vendor; pscreen->get_device_vendor = v3d_screen_get_vendor; diff --git a/src/gallium/drivers/v3d/v3d_screen.h b/src/gallium/drivers/v3d/v3d_screen.h index 9bf2a065487..de2ce2c1601 100644 --- a/src/gallium/drivers/v3d/v3d_screen.h +++ b/src/gallium/drivers/v3d/v3d_screen.h @@ -28,6 +28,7 @@ #include "renderonly/renderonly.h" #include "os/os_thread.h" #include "frontend/drm_driver.h" +#include "util/disk_cache.h" #include "util/list.h" #include "util/slab.h" #include "broadcom/common/v3d_debug.h" @@ -84,6 +85,10 @@ struct v3d_screen { bool nonmsaa_texture_size_limit; struct v3d_simulator_file *sim_file; + +#ifdef ENABLE_SHADER_CACHE + struct disk_cache *disk_cache; +#endif }; static inline struct v3d_screen * @@ -99,4 +104,9 @@ struct pipe_screen *v3d_screen_create(int fd, void v3d_fence_init(struct v3d_screen *screen); +#ifdef ENABLE_SHADER_CACHE +void +v3d_disk_cache_init(struct v3d_screen *screen); +#endif + #endif /* V3D_SCREEN_H */