fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2026-04-03 10:40:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

zink: uber shaders logic

Browse source 
Introduce the logic to implement uber shaders.

The way variants is handled changes significantly: a uber program is
expected to be compiling asynchronously and is used whenever possible.

Specialized variant shaders are compiled asynchronously, though they
might be compiled synchronously if the uber program can't be used.

Each variant is a separate program as that simplifies gpl/obj caching.

A new key is introduced, the st_key, that keeps track of the state of
the features emulated by the uber shader.

This is split in a dynamic part, always sent through push constants, and
a more compact part that is used as a key for caching optimized
variants.
This commit is contained in:
antonino 2023-07-28 11:11:15 +02:00 • committed by Anna Maniscalco
parent 446e49f145
commit f2ecf95fa7
12 changed files with 766 additions and 254 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
src/gallium/drivers/zink/zink_compiler.c
View file

@ -3879,9 +3879,25 @@ remove_interpolate_at_sample(struct nir_builder *b, nir_intrinsic_instr *interp,
return true;
}
static void
zink_optimized_st_emulation_passes(nir_shader *nir, struct zink_shader *zs,
const struct zink_st_variant_key *key)
{
if (!nir->info.io_lowered)
return;
}
static void
zink_emulation_passes(nir_shader *nir, struct zink_shader *zs)
{
if (!nir->info.io_lowered)
return;
}
struct zink_shader_object
zink_shader_compile(struct zink_screen *screen, bool can_shobj, struct zink_shader *zs,
nir_shader *nir, const struct zink_shader_key *key, const void *extra_data, struct zink_program *pg)
zink_shader_compile(struct zink_screen *screen, bool can_shobj, struct zink_shader *zs, nir_shader *nir,
const struct zink_shader_key *key, const struct zink_st_variant_key *st_key,
bool compile_uber, const void *extra_data, struct zink_program *pg)
{
bool need_optimize = true;
bool inlined_uniforms = false;
@ -3891,8 +3907,18 @@ zink_shader_compile(struct zink_screen *screen, bool can_shobj, struct zink_shad
NIR_PASS(_, nir, nir_lower_sample_shading);
}
if (compile_uber) {
zink_emulation_passes(nir, zs);
need_optimize = true;
}
else if (st_key) {
zink_optimized_st_emulation_passes(nir, zs, st_key);
need_optimize = true;
}
NIR_PASS(_, nir, add_derefs);
NIR_PASS(_, nir, nir_lower_fragcolor, nir->info.fs.color_is_dual_source ? 1 : 8);
if (key) {
if (key->inline_uniforms) {
NIR_PASS(_, nir, nir_inline_uniforms,
@ -4077,7 +4103,7 @@ zink_shader_compile(struct zink_screen *screen, bool can_shobj, struct zink_shad
}
struct zink_shader_object
zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs)
zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs, bool compile_uber)
{
nir_shader *nir = zs->nir;
/* TODO: maybe compile multiple variants for different set counts for compact mode? */
@ -4107,6 +4133,10 @@ zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs)
default: break;
}
}
if (compile_uber)
zink_emulation_passes(nir, zs);
NIR_PASS(_, nir, add_derefs);
NIR_PASS(_, nir, nir_lower_fragcolor, nir->info.fs.color_is_dual_source ? 1 : 8);
if (screen->driconf.inline_uniforms) {
@ -4114,6 +4144,7 @@ zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs)
NIR_PASS(_, nir, rewrite_bo_access, screen);
NIR_PASS(_, nir, remove_bo_access, zs);
}
optimize_nir(nir, zs, true);
zink_descriptor_shader_init(screen, zs);
nir_shader *nir_clone = NULL;
@ -4128,7 +4159,7 @@ zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs)
zs->non_fs.generated_tcs = zink_shader_tcs_create(screen, 32);
zink_shader_tcs_init(screen, zs->non_fs.generated_tcs, nir_clone, &nir_tcs);
nir_tcs->info.separate_shader = true;
zs->non_fs.generated_tcs->precompile.obj = zink_shader_compile_separate(screen, zs->non_fs.generated_tcs);
zs->non_fs.generated_tcs->precompile.obj = zink_shader_compile_separate(screen, zs->non_fs.generated_tcs, compile_uber);
ralloc_free(nir_tcs);
zs->non_fs.generated_tcs->nir = NULL;
}
@ -6448,11 +6479,13 @@ gfx_shader_prune(struct zink_screen *screen, struct zink_shader *shader)
prog->base.removed = true;
simple_mtx_unlock(lock);
for (int i = 0; i < ARRAY_SIZE(prog->pipelines); ++i) {
hash_table_foreach(&prog->pipelines[i], table_entry) {
struct zink_gfx_pipeline_cache_entry *pc_entry = table_entry->data;
for (int r = 0; r < ARRAY_SIZE(prog->pipelines); ++r) {
for (int i = 0; i < ARRAY_SIZE(prog->pipelines[0]); ++i) {
hash_table_foreach(&prog->pipelines[r][i], table_entry) {
struct zink_gfx_pipeline_cache_entry *pc_entry = table_entry->data;
util_queue_fence_wait(&pc_entry->fence);
util_queue_fence_wait(&pc_entry->fence);
}
}
}
}
@ -6468,7 +6501,10 @@ gfx_shader_prune(struct zink_screen *screen, struct zink_shader *shader)
prog->shaders[MESA_SHADER_GEOMETRY]->non_fs.parent == shader) {
prog->shaders[MESA_SHADER_GEOMETRY] = NULL;
}
zink_gfx_program_reference(screen, &prog, NULL);
/* variant programs are owned and destroyed by their parent */
if (!prog->is_variant_program)
zink_gfx_program_reference(screen, &prog, NULL);
return true;
}

6
src/gallium/drivers/zink/zink_compiler.h
View file

@ -63,9 +63,11 @@ void
zink_compiler_assign_io(struct zink_screen *screen, nir_shader *producer, nir_shader *consumer);
/* pass very large shader key data with extra_data */
struct zink_shader_object
zink_shader_compile(struct zink_screen *screen, bool can_shobj, struct zink_shader *zs, nir_shader *nir, const struct zink_shader_key *key, const void *extra_data, struct zink_program *pg);
zink_shader_compile(struct zink_screen *screen, bool can_shobj, struct zink_shader *zs, nir_shader *nir,
const struct zink_shader_key *key, const struct zink_st_variant_key *st_key,
bool compile_uber, const void *extra_data, struct zink_program *pg);
struct zink_shader_object
zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs);
zink_shader_compile_separate(struct zink_screen *screen, struct zink_shader *zs, bool compile_uber);
struct zink_shader *
zink_shader_create(struct zink_screen *screen, struct nir_shader *nir);
void

4
src/gallium/drivers/zink/zink_context.c
View file

@ -3721,8 +3721,8 @@ zink_update_descriptor_refs(struct zink_context *ctx, bool compute)
res->obj->unordered_read = false;
}
}
if (ctx->curr_program)
zink_batch_reference_program(ctx, &ctx->curr_program->base);
if (ctx->curr_program_uber || ctx->curr_program)
zink_batch_reference_program(ctx, &ctx->curr_program_uber->base);
}
if (ctx->di.bindless_refs_dirty) {
ctx->di.bindless_refs_dirty = false;

13
src/gallium/drivers/zink/zink_draw.cpp
View file

@ -265,11 +265,11 @@ update_gfx_pipeline(struct zink_context *ctx, struct zink_batch_state *bs, enum
zink_gfx_program_update(ctx);
bool pipeline_changed = false;
VkPipeline pipeline = VK_NULL_HANDLE;
if (!ctx->curr_program->base.uses_shobj) {
if (!(ctx->gfx_pipeline_state.uber_required ? ctx->curr_program_uber : ctx->curr_program)->base.uses_shobj) {
if (screen->info.have_EXT_graphics_pipeline_library)
pipeline = zink_get_gfx_pipeline<DYNAMIC_STATE, true, false>(ctx, ctx->curr_program, &ctx->gfx_pipeline_state, mode);
pipeline = zink_get_gfx_pipeline<DYNAMIC_STATE, true, false>(ctx, ctx->curr_program_uber, ctx->curr_program, &ctx->gfx_pipeline_state, mode);
else
pipeline = zink_get_gfx_pipeline<DYNAMIC_STATE, false, false>(ctx, ctx->curr_program, &ctx->gfx_pipeline_state, mode);
pipeline = zink_get_gfx_pipeline<DYNAMIC_STATE, false, false>(ctx, ctx->curr_program, ctx->curr_program, &ctx->gfx_pipeline_state, mode);
assert(pipeline);
pipeline_changed = prev_pipeline != pipeline || ctx->shobj_draw;
if (BATCH_CHANGED || pipeline_changed)
@ -285,7 +285,8 @@ update_gfx_pipeline(struct zink_context *ctx, struct zink_batch_state *bs, enum
VK_SHADER_STAGE_FRAGMENT_BIT,
};
/* always rebind all stages */
VKCTX(CmdBindShadersEXT)(bs->cmdbuf, ZINK_GFX_SHADER_COUNT, stages, ctx->curr_program->objects);
VKCTX(CmdBindShadersEXT)(bs->cmdbuf, ZINK_GFX_SHADER_COUNT, stages,
ctx->gfx_pipeline_state.uber_required ? ctx->curr_program_uber->objects : ctx->curr_program->objects);
if (screen->info.have_EXT_mesh_shader) {
/* must always unbind mesh stages */
VkShaderStageFlagBits mesh_stages[] = {
@ -994,9 +995,9 @@ update_mesh_pipeline(struct zink_context *ctx, struct zink_batch_state *bs)
VkPipeline pipeline = VK_NULL_HANDLE;
if (!ctx->mesh_program->base.uses_shobj) {
if (screen->info.have_EXT_graphics_pipeline_library)
pipeline = zink_get_gfx_pipeline<ZINK_DYNAMIC_STATE3, true, true>(ctx, ctx->mesh_program, &ctx->gfx_pipeline_state, MESA_PRIM_COUNT);
pipeline = zink_get_gfx_pipeline<ZINK_DYNAMIC_STATE3, true, true>(ctx, ctx->mesh_program, ctx->mesh_program, &ctx->gfx_pipeline_state, MESA_PRIM_COUNT);
else
pipeline = zink_get_gfx_pipeline<ZINK_DYNAMIC_STATE3, false, true>(ctx, ctx->mesh_program, &ctx->gfx_pipeline_state, MESA_PRIM_COUNT);
pipeline = zink_get_gfx_pipeline<ZINK_DYNAMIC_STATE3, false, true>(ctx, ctx->mesh_program, ctx->mesh_program, &ctx->gfx_pipeline_state, MESA_PRIM_COUNT);
assert(pipeline);
pipeline_changed = prev_pipeline != pipeline || ctx->shobj_draw;
if (BATCH_CHANGED || pipeline_changed)

4
src/gallium/drivers/zink/zink_pipeline.c
View file

@ -884,10 +884,10 @@ create_gfx_pipeline_library(struct zink_screen *screen, struct zink_shader_objec
}
VkPipeline
zink_create_gfx_pipeline_library(struct zink_screen *screen, struct zink_gfx_program *prog)
zink_create_gfx_pipeline_library(struct zink_screen *screen, struct zink_shader_object *objs, struct zink_gfx_program *prog)
{
u_rwlock_wrlock(&prog->base.pipeline_cache_lock);
VkPipeline pipeline = create_gfx_pipeline_library(screen, prog->objs, prog->stages_present, prog->base.layout, prog->base.pipeline_cache);
VkPipeline pipeline = create_gfx_pipeline_library(screen, objs, prog->stages_present, prog->base.layout, prog->base.pipeline_cache);
u_rwlock_wrunlock(&prog->base.pipeline_cache_lock);
return pipeline;
}

2
src/gallium/drivers/zink/zink_pipeline.h
View file

@ -58,7 +58,7 @@ zink_create_gfx_pipeline_input(struct zink_screen *screen,
const uint8_t *binding_map,
VkPrimitiveTopology primitive_topology);
VkPipeline
zink_create_gfx_pipeline_library(struct zink_screen *screen, struct zink_gfx_program *prog);
zink_create_gfx_pipeline_library(struct zink_screen *screen, struct zink_shader_object *objs, struct zink_gfx_program *prog);
VkPipeline
zink_create_gfx_pipeline_output(struct zink_screen *screen, struct zink_gfx_pipeline_state *state);
VkPipeline

727
src/gallium/drivers/zink/zink_program.c
View file

File diff suppressed because it is too large Load diff

33
src/gallium/drivers/zink/zink_program.h
View file

@ -128,7 +128,7 @@ zink_mesh_program_update_optimal(struct zink_context *ctx);
struct zink_gfx_library_key *
zink_create_pipeline_lib(struct zink_screen *screen, struct zink_gfx_program *prog, struct zink_gfx_pipeline_state *state);
zink_create_pipeline_lib(struct zink_screen *screen, struct zink_gfx_program *prog, struct zink_gfx_pipeline_state *state, bool compile_uber);
uint32_t hash_gfx_output(const void *key);
uint32_t hash_gfx_output_ds3(const void *key);
uint32_t hash_gfx_input(const void *key);
@ -159,7 +159,7 @@ zink_create_gfx_program(struct zink_context *ctx,
struct zink_shader **stages,
unsigned vertices_per_patch,
uint32_t gfx_hash,
bool is_mesh);
bool is_mesh, bool variant);
void
zink_destroy_gfx_program(struct zink_screen *screen,
@ -405,6 +405,27 @@ zink_set_zs_needs_shader_swizzle_key(struct zink_context *ctx, mesa_shader_stage
zink_set_shader_key_base(ctx, pstage)->needs_zs_shader_swizzle = enable;
}
static inline const union zink_st_small_key *
zink_get_st_small_key(struct zink_context *ctx)
{
assert(zink_screen(ctx->base.screen)->optimal_keys);
return &ctx->gfx_pipeline_state.st_key.small_key;
}
static inline union zink_st_small_key *
zink_set_st_small_key(struct zink_context *ctx)
{
ctx->dirty_gfx_stages |= ctx->shader_stages & (MESA_SHADER_VERTEX | MESA_SHADER_GEOMETRY | MESA_SHADER_FRAGMENT);
assert(zink_screen(ctx->base.screen)->optimal_keys);
return &ctx->gfx_pipeline_state.st_key.small_key;
}
static inline bool
needs_st_emulation(struct zink_context *ctx)
{
return ctx->gfx_pipeline_state.st_key.small_key.val != 0;
}
ALWAYS_INLINE static bool
zink_can_use_pipeline_libs(const struct zink_context *ctx)
{
@ -464,6 +485,14 @@ zink_can_use_shader_objects_mesh(const struct zink_context *ctx)
!ctx->fb_state.viewmask;
}
ALWAYS_INLINE static bool
zink_can_use_uber(struct zink_context *ctx)
{
bool generated_tcs = ctx->gfx_stages[MESA_SHADER_TESS_EVAL] && !ctx->gfx_stages[MESA_SHADER_TESS_CTRL];
return zink_shader_key_optimal_no_tcs(ctx->gfx_pipeline_state.optimal_key) == ZINK_SHADER_KEY_OPTIMAL_DEFAULT &&
zink_can_use_pipeline_libs(ctx) && (!generated_tcs || ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch == 3);
}
bool
zink_set_rasterizer_discard(struct zink_context *ctx, bool disable);
void

99
src/gallium/drivers/zink/zink_program_state.hpp
View file

@ -102,6 +102,7 @@ template <zink_dynamic_state DYNAMIC_STATE, bool HAVE_LIB, bool IS_MESH>
VkPipeline
zink_get_gfx_pipeline(struct zink_context *ctx,
struct zink_gfx_program *prog,
struct zink_gfx_program *variant_prog,
struct zink_gfx_pipeline_state *state,
enum mesa_prim mode)
{
@ -113,7 +114,7 @@ zink_get_gfx_pipeline(struct zink_context *ctx,
const unsigned idx = IS_MESH || screen->info.dynamic_state3_props.dynamicPrimitiveTopologyUnrestricted ?
0 :
get_pipeline_idx<DYNAMIC_STATE >= ZINK_DYNAMIC_STATE>(mode, vkmode);
assert(idx <= ARRAY_SIZE(prog->pipelines));
assert(idx <= ARRAY_SIZE(prog->pipelines[0]));
if (IS_MESH) {
if (!state->mesh_dirty && !state->mesh_modules_changed)
return state->mesh_pipeline;
@ -144,27 +145,28 @@ zink_get_gfx_pipeline(struct zink_context *ctx,
}
}
/* extra safety asserts for optimal path to catch refactoring bugs */
if (prog->optimal_keys) {
if (variant_prog->optimal_keys) {
ASSERTED const union zink_shader_key_optimal *opt = (union zink_shader_key_optimal*)&prog->last_variant_hash;
ASSERTED union zink_shader_key_optimal sanitized = {};
if (IS_MESH) {
sanitized.val = zink_sanitize_optimal_key_mesh(ctx->gfx_stages, ctx->gfx_pipeline_state.shader_keys_optimal.key.val);
assert(opt->val == sanitized.val);
assert(state->mesh_optimal_key == sanitized.val);
} else {
} else if (!state->uber_required) {
sanitized.val = zink_sanitize_optimal_key(ctx->gfx_stages, ctx->gfx_pipeline_state.shader_keys_optimal.key.val);
assert(opt->val == sanitized.val);
assert(state->optimal_key == sanitized.val);
}
assert(opt->val == sanitized.val);
}
if (IS_MESH) {
state->mesh_modules_changed = false;
if (prog->last_finalized_hash[idx] == state->mesh_final_hash &&
!prog->inline_variants && likely(prog->last_pipeline[idx]) &&
if (prog->last_finalized_hash[0][idx] == state->mesh_final_hash &&
!prog->inline_variants && likely(prog->last_pipeline[0][idx]) &&
/* this data is too big to compare in the fast-path */
likely(!prog->shaders[MESA_SHADER_FRAGMENT]->fs.legacy_shadow_mask)) {
state->mesh_pipeline = prog->last_pipeline[idx]->pipeline;
state->mesh_pipeline = prog->last_pipeline[0][idx]->pipeline;
return state->mesh_pipeline;
}
} else {
@ -202,23 +204,22 @@ zink_get_gfx_pipeline(struct zink_context *ctx,
/* shortcut for reusing previous pipeline across program changes */
if (DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT || DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT2) {
if (prog->last_finalized_hash[idx] == state->final_hash &&
!prog->inline_variants && likely(prog->last_pipeline[idx]) &&
/* this data is too big to compare in the fast-path */
likely(!prog->shaders[MESA_SHADER_FRAGMENT]->fs.legacy_shadow_mask)) {
state->pipeline = prog->last_pipeline[idx]->pipeline;
if (variant_prog->last_finalized_hash[state->uber_required][idx] == state->final_hash &&
!variant_prog->inline_variants && likely(variant_prog->last_pipeline[state->uber_required][idx]) &&
/* this data is too big to compare in the fast-path */
likely(!variant_prog->shaders[MESA_SHADER_FRAGMENT]->fs.legacy_shadow_mask)) {
state->pipeline = variant_prog->last_pipeline[state->uber_required][idx]->pipeline;
return state->pipeline;
}
}
}
unsigned final_hash = IS_MESH ? state->mesh_final_hash : state->final_hash;
entry = _mesa_hash_table_search_pre_hashed(&prog->pipelines[idx], final_hash, state);
entry = _mesa_hash_table_search_pre_hashed(&variant_prog->pipelines[state->uber_required][idx], final_hash, state);
if (!entry) {
bool can_gpl = IS_MESH ? zink_can_use_pipeline_libs_mesh(ctx) : zink_can_use_pipeline_libs(ctx);
/* always wait on async precompile/cache fence */
util_queue_fence_wait(&prog->base.cache_fence);
util_queue_fence_wait(&variant_prog->base.cache_fence);
struct zink_gfx_pipeline_cache_entry *pc_entry = CALLOC_STRUCT(zink_gfx_pipeline_cache_entry);
if (!pc_entry)
return VK_NULL_HANDLE;
@ -227,28 +228,47 @@ zink_get_gfx_pipeline(struct zink_context *ctx,
*/
memcpy(&pc_entry->state, state, sizeof(*state));
pc_entry->state.rendering_info.pColorAttachmentFormats = pc_entry->state.rendering_formats;
pc_entry->prog = prog;
pc_entry->prog = state->uber_required ? prog : variant_prog;
/* init the optimized background compile fence */
util_queue_fence_init(&pc_entry->fence);
entry = _mesa_hash_table_insert_pre_hashed(&prog->pipelines[idx], final_hash, pc_entry, pc_entry);
if (prog->base.uses_shobj && !prog->is_separable) {
memcpy(pc_entry->shobjs, prog->objs, sizeof(prog->objs));
entry = _mesa_hash_table_insert_pre_hashed(&variant_prog->pipelines[state->uber_required][idx], final_hash, pc_entry, pc_entry);
if (variant_prog->base.uses_shobj && !variant_prog->is_separable) {
memcpy(pc_entry->shobjs, variant_prog->objs, sizeof(variant_prog->objs));
zink_gfx_program_compile_queue(ctx, pc_entry);
} else if (HAVE_LIB && can_gpl) {
uint32_t optimal_key = IS_MESH ? ctx->gfx_pipeline_state.mesh_optimal_key : ctx->gfx_pipeline_state.optimal_key;
/* this is the graphics pipeline library path: find/construct all partial pipelines */
simple_mtx_lock(&prog->libs->lock);
struct set_entry *he = _mesa_set_search(&prog->libs->libs, &optimal_key);
struct zink_gfx_library_key *gkey;
if (he) {
gkey = (struct zink_gfx_library_key *)he->key;
if (IS_MESH) {
simple_mtx_lock(&prog->libs->lock);
struct set_entry *he = _mesa_set_search(&prog->libs->libs, &optimal_key);
if (he) {
gkey = (struct zink_gfx_library_key *)he->key;
} else {
assert(!prog->is_separable);
gkey = zink_create_pipeline_lib(screen, prog, &ctx->gfx_pipeline_state, false);
}
simple_mtx_unlock(&prog->libs->lock);
} else {
assert(!prog->is_separable);
gkey = zink_create_pipeline_lib(screen, prog, &ctx->gfx_pipeline_state);
if (state->uber_required) {
simple_mtx_lock(&prog->libs->lock);
assert(prog->libs->lib);
gkey = prog->libs->lib;
simple_mtx_unlock(&prog->libs->lock);
} else {
simple_mtx_lock(&variant_prog->libs->lock);
if (variant_prog->libs->lib) {
gkey = variant_prog->libs->lib;
assert(gkey->optimal_key == optimal_key);
assert(gkey->st_key == state->st_key.small_key.val);
} else {
assert(!variant_prog->is_separable);
gkey = zink_create_pipeline_lib(screen, variant_prog, &ctx->gfx_pipeline_state, false);
}
simple_mtx_unlock(&variant_prog->libs->lock);
}
}
simple_mtx_unlock(&prog->libs->lock);
struct zink_gfx_input_key *ikey = IS_MESH ? NULL :
DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT ?
struct zink_gfx_input_key *ikey = DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT ?
zink_find_or_create_input_dynamic(ctx, vkmode) :
zink_find_or_create_input(ctx, vkmode);
struct zink_gfx_output_key *okey = DYNAMIC_STATE >= ZINK_DYNAMIC_STATE3 && screen->have_full_ds3 ?
@ -259,29 +279,30 @@ zink_get_gfx_pipeline(struct zink_context *ctx,
pc_entry->gpl.gkey = gkey;
pc_entry->gpl.okey = okey;
/* try to hit optimized compile cache first if possible */
if (!prog->is_separable)
pc_entry->pipeline = zink_create_gfx_pipeline_combined(screen, prog, ikey ? ikey->pipeline : VK_NULL_HANDLE, &gkey->pipeline, 1, okey->pipeline, true, true);
if (!variant_prog->is_separable)
pc_entry->pipeline = zink_create_gfx_pipeline_combined(screen, variant_prog, ikey ? ikey->pipeline : VK_NULL_HANDLE, &gkey->pipeline, 1, okey->pipeline, true, true);
if (!pc_entry->pipeline) {
/* create the non-optimized pipeline first using fast-linking to avoid stuttering */
pc_entry->pipeline = zink_create_gfx_pipeline_combined(screen, prog, ikey ? ikey->pipeline : VK_NULL_HANDLE, &gkey->pipeline, 1, okey->pipeline, false, false);
if (!prog->is_separable)
pc_entry->pipeline = zink_create_gfx_pipeline_combined(screen, variant_prog, ikey ? ikey->pipeline : VK_NULL_HANDLE, &gkey->pipeline, 1, okey->pipeline, false, false);
if (!variant_prog->is_separable)
/* trigger async optimized pipeline compile if this was the fast-linked unoptimized pipeline */
zink_gfx_program_compile_queue(ctx, pc_entry);
}
} else {
struct zink_shader_object *objs = state->uber_required ? prog->objs : variant_prog->objs;
/* optimize by default only when expecting precompiles in order to reduce stuttering */
if (DYNAMIC_STATE != ZINK_DYNAMIC_VERTEX_INPUT2 && DYNAMIC_STATE != ZINK_DYNAMIC_VERTEX_INPUT && !IS_MESH)
pc_entry->pipeline = zink_create_gfx_pipeline(screen, prog, prog->objs, state, state->element_state->binding_map, vkmode, !HAVE_LIB);
pc_entry->pipeline = zink_create_gfx_pipeline(screen, variant_prog, objs, state, state->element_state->binding_map, vkmode, !HAVE_LIB);
else
pc_entry->pipeline = zink_create_gfx_pipeline(screen, prog, prog->objs, state, NULL, vkmode, !HAVE_LIB);
if (HAVE_LIB && !prog->is_separable)
pc_entry->pipeline = zink_create_gfx_pipeline(screen, variant_prog, objs, state, NULL, vkmode, !HAVE_LIB);
if (HAVE_LIB && !variant_prog->is_separable)
/* trigger async optimized pipeline compile if this was an unoptimized pipeline */
zink_gfx_program_compile_queue(ctx, pc_entry);
}
if (pc_entry->pipeline == VK_NULL_HANDLE)
return VK_NULL_HANDLE;
zink_screen_update_pipeline_cache(screen, &prog->base, false);
zink_screen_update_pipeline_cache(screen, &variant_prog->base, false);
}
struct zink_gfx_pipeline_cache_entry *cache_entry = (struct zink_gfx_pipeline_cache_entry *)entry->data;
@ -291,8 +312,8 @@ zink_get_gfx_pipeline(struct zink_context *ctx,
state->pipeline = cache_entry->pipeline;
/* update states for fastpath */
if (DYNAMIC_STATE >= ZINK_DYNAMIC_VERTEX_INPUT) {
prog->last_finalized_hash[idx] = final_hash;
prog->last_pipeline[idx] = cache_entry;
variant_prog->last_finalized_hash[state->uber_required][idx] = final_hash;
variant_prog->last_pipeline[state->uber_required][idx] = cache_entry;
}
return IS_MESH ? state->mesh_pipeline : state->pipeline;
}
@ -355,6 +376,8 @@ equals_gfx_pipeline_state(const void *a, const void *b)
if (STAGE_MASK & STAGE_MASK_OPTIMAL) {
if (sa->optimal_key != sb->optimal_key)
return false;
if (sa->st_key.small_key.val != sb->st_key.small_key.val)
return false;
if (STAGE_MASK & STAGE_MASK_OPTIMAL_SHADOW) {
if (sa->shadow != sb->shadow)
return false;

35
src/gallium/drivers/zink/zink_shader_keys.h
View file

@ -28,6 +28,41 @@
#include "compiler/shader_info.h"
union zink_st_small_key {
struct {
/** for ARB_color_buffer_float */
uint8_t clamp_color:1;
/* for user-defined clip-planes */
uint8_t lower_ucp:1;
/* Whether st_variant::driver_shader is for the draw module,
* not for the driver.
*/
uint8_t is_draw_shader:1;
uint8_t lower_flatshade:1;
uint8_t lower_alpha_test:1;
uint16_t pad: 11; // from here not key
};
uint16_t val;
};
struct zink_st_variant_key
{
union zink_st_small_key small_key;
uint8_t ucp_enables: 8;
unsigned lower_alpha_func:3;
uint32_t pad2: 5; //next array aligned to uint32 for easy access
/* bitmask of sampler units; PIPE_CAP_GL_CLAMP */
uint32_t gl_clamp[3];
/* needs more than 128 bytes */
struct pipe_clip_state ucp_state;
};
struct zink_vs_key_base {
bool last_vertex_stage : 1;
bool clip_halfz : 1;

15
src/gallium/drivers/zink/zink_state.c
View file

@ -722,6 +722,21 @@ zink_bind_rasterizer_state(struct pipe_context *pctx, void *cso)
if (!screen->optimal_keys)
zink_update_gs_key_rectangular_line(ctx);
if (screen->optimal_keys) {
struct zink_st_variant_key *key = &ctx->gfx_pipeline_state.st_key;
if ((zink_get_st_small_key(ctx)->clamp_color) != ctx->rast_state->base.clamp_fragment_color)
zink_set_st_small_key(ctx)->clamp_color = ctx->rast_state->base.clamp_fragment_color;
if ((zink_get_st_small_key(ctx)->lower_flatshade) != ctx->rast_state->base.flatshade)
zink_set_st_small_key(ctx)->lower_flatshade = ctx->rast_state->base.flatshade;
key->ucp_enables = ctx->rast_state->base.clip_plane_enable;
if ((zink_get_st_small_key(ctx)->lower_ucp) != !!key->ucp_enables)
zink_set_st_small_key(ctx)->lower_ucp = !!key->ucp_enables;
}
}
}

28
src/gallium/drivers/zink/zink_types.h
View file

@ -795,6 +795,7 @@ struct zink_shader_object {
VkShaderModule mod;
};
struct spirv_shader *spirv;
VkPipeline gpl;
};
struct zink_shader {
@ -824,6 +825,7 @@ struct zink_shader {
bool has_uniforms;
bool has_edgeflags;
bool needs_inlining;
bool is_uber;
struct spirv_shader *spirv;
struct {
@ -932,6 +934,7 @@ struct zink_gfx_pipeline_state {
uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
struct zink_vertex_elements_hw_state *element_state;
struct zink_zs_swizzle_key *shadow;
bool uber_required; // emulation needed && !async compilation done
enum mesa_prim shader_rast_prim, rast_prim; /* reduced type or max for unknown */
union {
struct {
@ -942,6 +945,7 @@ struct zink_gfx_pipeline_state {
union zink_shader_key_optimal key;
} shader_keys_optimal;
};
struct zink_st_variant_key st_key;
struct zink_blend_state *blend_state;
VkFormat rendering_formats[PIPE_MAX_COLOR_BUFS];
VkPipelineRenderingCreateInfo rendering_info;
@ -1008,6 +1012,7 @@ enum zink_gfx_push_constant_member {
*/
struct zink_shader_module {
struct zink_shader_object obj;
struct util_queue_fence fence;
uint32_t hash;
bool shobj;
bool default_variant;
@ -1049,6 +1054,7 @@ typedef bool (*equals_gfx_pipeline_state_func)(const void *a, const void *b);
struct zink_gfx_library_key {
uint32_t optimal_key; //equals_pipeline_lib_optimal
uint32_t st_key;
VkShaderModule modules[MESA_SHADER_MESH_STAGES];
VkPipeline pipeline;
};
@ -1115,6 +1121,13 @@ struct zink_gfx_lib_cache {
simple_mtx_t lock;
struct set libs; //zink_gfx_library_key -> VkPipeline
struct zink_gfx_library_key *lib; //zink_gfx_library_key -> VkPipeline
};
struct zink_gfx_program_variant_key {
uint32_t optimal_key; //equals_pipeline_lib_optimal
uint32_t st_key;
struct zink_gfx_program *prog;
};
struct zink_gfx_program {
@ -1135,21 +1148,29 @@ struct zink_gfx_program {
uint32_t module_hash[MESA_SHADER_MESH_STAGES];
struct blob blobs[MESA_SHADER_MESH_STAGES];
struct util_dynarray shader_cache[MESA_SHADER_MESH_STAGES][2][2]; //normal, nonseamless cubes, inline uniforms
struct util_dynarray uber_modules;
struct set variants;
struct zink_gfx_program *base_variant; //quick access to base varitant (only !NULL when done compiling)
struct zink_gfx_program *uber_variant;
unsigned inlined_variant_count[MESA_SHADER_MESH_STAGES];
uint32_t default_variant_hash;
uint8_t inline_variants; //which stages are using inlined uniforms
bool needs_inlining; // whether this program requires some uniforms to be inlined
bool has_edgeflags;
bool optimal_keys;
bool started_compiling;
bool is_uber_program;
bool is_variant_program;
/* separable */
struct zink_gfx_program *full_prog;
struct hash_table pipelines[11]; // [number of draw modes we support]
struct hash_table pipelines[2][11]; // [uber_emulation][number of draw modes we support]
uint32_t last_variant_hash;
uint32_t st_key;
uint32_t last_finalized_hash[4]; //[primtype idx]
struct zink_gfx_pipeline_cache_entry *last_pipeline[4]; //[primtype idx]
uint32_t last_finalized_hash[2][4]; //[uber_emulation][primtype idx]
struct zink_gfx_pipeline_cache_entry *last_pipeline[2][4]; //[uber_emulation][primtype idx]
struct zink_gfx_lib_cache *libs;
};
@ -1787,6 +1808,7 @@ struct zink_context {
simple_mtx_t program_lock[8];
uint32_t gfx_hash;
struct zink_gfx_program *curr_program;
struct zink_gfx_program *curr_program_uber;
struct set gfx_inputs;
struct set gfx_outputs;