fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2025-12-24 13:10:10 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1
mesa/src/gallium/drivers/zink/zink_program.c
Mike Blumenkrantz 1542f3eb47 zink: decompose vertex attribs into single components when not supported 
this avoids vbuf in a lot more cases on radv where 3component attribs aren't supported

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12771>
2021-09-14 01:55:56 +00:00

1107 lines
38 KiB
C
Raw Blame History

/*
* Copyright 2018 Collabora Ltd.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* on the rights to use, copy, modify, merge, publish, distribute, sub
* license, and/or sell copies of the Software, and to permit persons to whom
* the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
* THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "zink_program.h"
#include "zink_compiler.h"
#include "zink_context.h"
#include "zink_descriptors.h"
#include "zink_helpers.h"
#include "zink_render_pass.h"
#include "zink_resource.h"
#include "zink_screen.h"
#include "zink_state.h"
#include "zink_inlines.h"
#include "util/hash_table.h"
#include "util/set.h"
#include "util/u_debug.h"
#include "util/u_memory.h"
#include "util/u_prim.h"
#include "tgsi/tgsi_from_mesa.h"
/* for pipeline cache */
#define XXH_INLINE_ALL
#include "util/xxhash.h"
struct gfx_pipeline_cache_entry {
struct zink_gfx_pipeline_state state;
VkPipeline pipeline;
};
struct compute_pipeline_cache_entry {
struct zink_compute_pipeline_state state;
VkPipeline pipeline;
};
void
debug_describe_zink_gfx_program(char *buf, const struct zink_gfx_program *ptr)
{
sprintf(buf, "zink_gfx_program");
}
void
debug_describe_zink_compute_program(char *buf, const struct zink_compute_program *ptr)
{
sprintf(buf, "zink_compute_program");
}
/* copied from iris */
struct keybox {
uint16_t size;
gl_shader_stage stage;
uint8_t data[0];
};
static struct keybox *
make_keybox(void *mem_ctx, gl_shader_stage stage, const void *key, uint32_t key_size, void *base, uint32_t base_size)
{
struct keybox *keybox =
ralloc_size(mem_ctx, sizeof(struct keybox) + key_size + base_size);
keybox->stage = stage;
keybox->size = key_size + base_size;
memcpy(keybox->data, key, key_size);
if (base_size)
memcpy(&keybox->data[key_size], base, base_size);
return keybox;
}
static uint32_t
keybox_hash(const void *void_key)
{
const struct keybox *key = void_key;
return _mesa_hash_data(&key->stage, key->size + sizeof(key->stage));
}
static bool
keybox_equals(const void *void_a, const void *void_b)
{
const struct keybox *a = void_a, *b = void_b;
if (a->size != b->size)
return false;
return memcmp(a->data, b->data, a->size) == 0;
}
static void
shader_key_vs_base_gen(struct zink_context *ctx, struct zink_shader *zs,
struct zink_shader *shaders[ZINK_SHADER_COUNT], struct zink_shader_key *key)
{
struct zink_vs_key_base *vs_key = &key->key.vs_base;
key->size = sizeof(struct zink_vs_key_base);
vs_key->clip_halfz = ctx->rast_state && ctx->rast_state->base.clip_halfz;
switch (zs->nir->info.stage) {
case MESA_SHADER_VERTEX:
vs_key->last_vertex_stage = !shaders[PIPE_SHADER_TESS_EVAL] && !shaders[PIPE_SHADER_GEOMETRY];
vs_key->push_drawid = ctx->gfx_pipeline_state.drawid_broken;
break;
case MESA_SHADER_TESS_EVAL:
vs_key->last_vertex_stage = !shaders[PIPE_SHADER_GEOMETRY];
break;
case MESA_SHADER_GEOMETRY:
vs_key->last_vertex_stage = true;
break;
default:
unreachable("impossible case");
}
}
static void
shader_key_vs_gen(struct zink_context *ctx, struct zink_shader *zs,
struct zink_shader *shaders[ZINK_SHADER_COUNT], struct zink_shader_key *key)
{
struct zink_vs_key *vs_key = &key->key.vs;
shader_key_vs_base_gen(ctx, zs, shaders, key);
vs_key->decomposed_attrs = ctx->element_state->decomposed_attrs;
vs_key->decomposed_attrs_without_w = ctx->element_state->decomposed_attrs_without_w;
key->size += 2 * 4;
}
static void
shader_key_fs_gen(struct zink_context *ctx, struct zink_shader *zs,
struct zink_shader *shaders[ZINK_SHADER_COUNT], struct zink_shader_key *key)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_fs_key *fs_key = &key->key.fs;
key->size = sizeof(struct zink_fs_key);
/* if gl_SampleMask[] is written to, we have to ensure that we get a shader with the same sample count:
* in GL, rast_samples==1 means ignore gl_SampleMask[]
* in VK, gl_SampleMask[] is never ignored
*/
if (zs->nir->info.outputs_written & (1 << FRAG_RESULT_SAMPLE_MASK))
fs_key->samples = ctx->gfx_pipeline_state.render_pass->state.samples;
fs_key->force_dual_color_blend = screen->driconf.dual_color_blend_by_location &&
ctx->gfx_pipeline_state.blend_state &&
ctx->gfx_pipeline_state.blend_state->dual_src_blend &&
ctx->gfx_pipeline_state.blend_state->attachments[1].blendEnable;
if (ctx->gfx_pipeline_state.has_points && ctx->gfx_pipeline_state.coord_replace_bits) {
fs_key->coord_replace_bits = ctx->gfx_pipeline_state.coord_replace_bits;
fs_key->coord_replace_yinvert = ctx->gfx_pipeline_state.coord_replace_yinvert;
}
}
static void
shader_key_tcs_gen(struct zink_context *ctx, struct zink_shader *zs,
struct zink_shader *shaders[ZINK_SHADER_COUNT], struct zink_shader_key *key)
{
key->size = 0;
}
typedef void (*zink_shader_key_gen)(struct zink_context *ctx, struct zink_shader *zs,
struct zink_shader *shaders[ZINK_SHADER_COUNT],
struct zink_shader_key *key);
static zink_shader_key_gen shader_key_vtbl[] =
{
[MESA_SHADER_VERTEX] = shader_key_vs_gen,
[MESA_SHADER_TESS_CTRL] = shader_key_tcs_gen,
/* reusing vs key for now since we're only using clip_halfz */
[MESA_SHADER_TESS_EVAL] = shader_key_vs_base_gen,
[MESA_SHADER_GEOMETRY] = shader_key_vs_base_gen,
[MESA_SHADER_FRAGMENT] = shader_key_fs_gen,
};
/* return pointer to make function reusable */
static inline struct zink_shader_module **
get_default_shader_module_ptr(struct zink_gfx_program *prog, struct zink_shader *zs, struct zink_shader_key *key)
{
if (zs->nir->info.stage == MESA_SHADER_VERTEX ||
zs->nir->info.stage == MESA_SHADER_TESS_EVAL) {
/* no streamout or halfz */
if (!zink_vs_key_base(key)->last_vertex_stage)
return &prog->default_variants[zs->nir->info.stage][1];
}
return &prog->default_variants[zs->nir->info.stage][0];
}
static struct zink_shader_module *
get_shader_module_for_stage(struct zink_context *ctx, struct zink_shader *zs, struct zink_gfx_program *prog)
{
gl_shader_stage stage = zs->nir->info.stage;
enum pipe_shader_type pstage = pipe_shader_type_from_mesa(stage);
struct zink_shader_key key = {0};
VkShaderModule mod;
struct zink_shader_module *zm;
struct zink_shader_module **default_zm = NULL;
struct keybox *keybox;
uint32_t hash;
unsigned base_size = 0;
shader_key_vtbl[stage](ctx, zs, prog->shaders, &key);
/* this is default variant if there is no default or it matches the default */
if (pstage != PIPE_SHADER_TESS_CTRL && prog->default_variant_key[pstage]) {
const struct keybox *tmp = prog->default_variant_key[pstage];
/* if comparing against the existing default, use the base variant key size since
* we're only checking the stage-specific data
*/
key.is_default_variant = tmp->size == key.size && !memcmp(tmp->data, &key, key.size);
} else
key.is_default_variant = true;
if (zs->nir->info.num_inlinable_uniforms &&
ctx->inlinable_uniforms_valid_mask & BITFIELD64_BIT(pstage)) {
key.inline_uniforms = true;
memcpy(key.base.inlined_uniform_values,
ctx->inlinable_uniforms[pstage],
zs->nir->info.num_inlinable_uniforms * 4);
base_size = zs->nir->info.num_inlinable_uniforms * sizeof(uint32_t);
key.is_default_variant = false;
}
if (key.is_default_variant) {
default_zm = get_default_shader_module_ptr(prog, zs, &key);
if (*default_zm)
return *default_zm;
}
keybox = make_keybox(prog, stage, &key, key.size, &key.base, base_size);
hash = keybox_hash(keybox);
struct hash_entry *entry = _mesa_hash_table_search_pre_hashed(&prog->base.shader_cache[pstage],
hash, keybox);
if (entry) {
ralloc_free(keybox);
zm = entry->data;
} else {
zm = CALLOC_STRUCT(zink_shader_module);
if (!zm) {
ralloc_free(keybox);
return NULL;
}
zm->hash = hash;
mod = zink_shader_compile(zink_screen(ctx->base.screen), zs, prog->nir[stage], &key);
if (!mod) {
ralloc_free(keybox);
FREE(zm);
return NULL;
}
zm->shader = mod;
_mesa_hash_table_insert_pre_hashed(&prog->base.shader_cache[pstage], hash, keybox, zm);
if (key.is_default_variant) {
/* previously returned */
*default_zm = zm;
zm->default_variant = true;
prog->default_variant_key[pstage] = keybox;
}
}
return zm;
}
static void
zink_destroy_shader_module(struct zink_screen *screen, struct zink_shader_module *zm)
{
VKSCR(DestroyShaderModule)(screen->dev, zm->shader, NULL);
free(zm);
}
static void
destroy_shader_cache(struct zink_screen *screen, struct hash_table *sc)
{
hash_table_foreach(sc, entry) {
struct zink_shader_module *zm = entry->data;
zink_destroy_shader_module(screen, zm);
}
}
static void
update_shader_modules(struct zink_context *ctx, struct zink_gfx_program *prog, uint32_t mask)
{
bool hash_changed = false;
bool default_variants = true;
bool first = !prog->modules[PIPE_SHADER_VERTEX];
uint32_t variant_hash = prog->last_variant_hash;
u_foreach_bit(pstage, mask) {
assert(prog->shaders[pstage]);
struct zink_shader_module *zm = get_shader_module_for_stage(ctx, prog->shaders[pstage], prog);
if (prog->modules[pstage] != zm) {
if (prog->modules[pstage])
variant_hash ^= prog->modules[pstage]->hash;
hash_changed = true;
}
default_variants &= zm->default_variant;
prog->modules[pstage] = zm;
variant_hash ^= prog->modules[pstage]->hash;
ctx->gfx_pipeline_state.modules[pstage] = zm->shader;
}
if (hash_changed) {
if (!first && likely(ctx->gfx_pipeline_state.pipeline)) //avoid on first hash
ctx->gfx_pipeline_state.final_hash ^= prog->last_variant_hash;
if (default_variants && !first)
prog->last_variant_hash = prog->default_variant_hash;
else
prog->last_variant_hash = variant_hash;
ctx->gfx_pipeline_state.final_hash ^= prog->last_variant_hash;
ctx->gfx_pipeline_state.modules_changed = true;
}
}
static uint32_t
hash_gfx_pipeline_state(const void *key)
{
const struct zink_gfx_pipeline_state *state = key;
uint32_t hash = _mesa_hash_data(key, offsetof(struct zink_gfx_pipeline_state, hash));
if (!state->have_EXT_extended_dynamic_state2)
hash = XXH32(&state->primitive_restart, 1, hash);
if (state->have_EXT_extended_dynamic_state)
return hash;
return XXH32(&state->dyn_state1, sizeof(state->dyn_state1), hash);
}
static bool
equals_gfx_pipeline_state(const void *a, const void *b)
{
const struct zink_gfx_pipeline_state *sa = a;
const struct zink_gfx_pipeline_state *sb = b;
if (!sa->have_EXT_extended_dynamic_state) {
if (sa->vertex_buffers_enabled_mask != sb->vertex_buffers_enabled_mask)
return false;
/* if we don't have dynamic states, we have to hash the enabled vertex buffer bindings */
uint32_t mask_a = sa->vertex_buffers_enabled_mask;
uint32_t mask_b = sb->vertex_buffers_enabled_mask;
while (mask_a || mask_b) {
unsigned idx_a = u_bit_scan(&mask_a);
unsigned idx_b = u_bit_scan(&mask_b);
if (sa->vertex_strides[idx_a] != sb->vertex_strides[idx_b])
return false;
}
if (sa->dyn_state1.front_face != sb->dyn_state1.front_face)
return false;
if (!!sa->dyn_state1.depth_stencil_alpha_state != !!sb->dyn_state1.depth_stencil_alpha_state ||
(sa->dyn_state1.depth_stencil_alpha_state &&
memcmp(sa->dyn_state1.depth_stencil_alpha_state, sb->dyn_state1.depth_stencil_alpha_state,
sizeof(struct zink_depth_stencil_alpha_hw_state))))
return false;
}
if (!sa->have_EXT_extended_dynamic_state2) {
if (sa->primitive_restart != sb->primitive_restart)
return false;
}
return !memcmp(sa->modules, sb->modules, sizeof(sa->modules)) &&
!memcmp(a, b, offsetof(struct zink_gfx_pipeline_state, hash));
}
void
zink_update_gfx_program(struct zink_context *ctx, struct zink_gfx_program *prog)
{
update_shader_modules(ctx, prog, ctx->dirty_shader_stages & prog->stages_present);
}
VkPipelineLayout
zink_pipeline_layout_create(struct zink_screen *screen, struct zink_program *pg)
{
VkPipelineLayoutCreateInfo plci = {0};
plci.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;
plci.pSetLayouts = pg->dsl;
plci.setLayoutCount = pg->num_dsl;
VkPushConstantRange pcr[2] = {0};
if (pg->is_compute) {
if (((struct zink_compute_program*)pg)->shader->nir->info.stage == MESA_SHADER_KERNEL) {
pcr[0].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
pcr[0].offset = 0;
pcr[0].size = sizeof(struct zink_cs_push_constant);
plci.pushConstantRangeCount = 1;
}
} else {
pcr[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
pcr[0].offset = offsetof(struct zink_gfx_push_constant, draw_mode_is_indexed);
pcr[0].size = 2 * sizeof(unsigned);
pcr[1].stageFlags = VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT;
pcr[1].offset = offsetof(struct zink_gfx_push_constant, default_inner_level);
pcr[1].size = sizeof(float) * 6;
plci.pushConstantRangeCount = 2;
}
plci.pPushConstantRanges = &pcr[0];
VkPipelineLayout layout;
if (VKSCR(CreatePipelineLayout)(screen->dev, &plci, NULL, &layout) != VK_SUCCESS) {
debug_printf("vkCreatePipelineLayout failed!\n");
return VK_NULL_HANDLE;
}
return layout;
}
static void
assign_io(struct zink_gfx_program *prog, struct zink_shader *stages[ZINK_SHADER_COUNT])
{
struct zink_shader *shaders[PIPE_SHADER_TYPES];
/* build array in pipeline order */
for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++)
shaders[tgsi_processor_to_shader_stage(i)] = stages[i];
for (unsigned i = 0; i < MESA_SHADER_FRAGMENT;) {
nir_shader *producer = shaders[i]->nir;
for (unsigned j = i + 1; j < ZINK_SHADER_COUNT; i++, j++) {
struct zink_shader *consumer = shaders[j];
if (!consumer)
continue;
if (!prog->nir[producer->info.stage])
prog->nir[producer->info.stage] = nir_shader_clone(prog, producer);
if (!prog->nir[j])
prog->nir[j] = nir_shader_clone(prog, consumer->nir);
zink_compiler_assign_io(prog->nir[producer->info.stage], prog->nir[j]);
i = j;
break;
}
}
}
struct zink_gfx_program *
zink_create_gfx_program(struct zink_context *ctx,
struct zink_shader *stages[ZINK_SHADER_COUNT])
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_gfx_program *prog = rzalloc(NULL, struct zink_gfx_program);
if (!prog)
goto fail;
pipe_reference_init(&prog->base.reference, 1);
for (int i = 0; i < ZINK_SHADER_COUNT; ++i) {
if (stages[i]) {
_mesa_hash_table_init(&prog->base.shader_cache[i], prog, keybox_hash, keybox_equals);
prog->shaders[i] = stages[i];
prog->stages_present |= BITFIELD_BIT(i);
}
}
if (stages[PIPE_SHADER_TESS_EVAL] && !stages[PIPE_SHADER_TESS_CTRL]) {
prog->shaders[PIPE_SHADER_TESS_EVAL]->generated =
prog->shaders[PIPE_SHADER_TESS_CTRL] =
zink_shader_tcs_create(ctx, stages[PIPE_SHADER_VERTEX]);
_mesa_hash_table_init(&prog->base.shader_cache[PIPE_SHADER_TESS_CTRL], prog, keybox_hash, keybox_equals);
prog->stages_present |= BITFIELD_BIT(PIPE_SHADER_TESS_CTRL);
}
assign_io(prog, prog->shaders);
update_shader_modules(ctx, prog, prog->stages_present);
prog->default_variant_hash = prog->last_variant_hash;
if (stages[PIPE_SHADER_GEOMETRY])
prog->last_vertex_stage = stages[PIPE_SHADER_GEOMETRY];
else if (stages[PIPE_SHADER_TESS_EVAL])
prog->last_vertex_stage = stages[PIPE_SHADER_TESS_EVAL];
else
prog->last_vertex_stage = stages[PIPE_SHADER_VERTEX];
for (int i = 0; i < ARRAY_SIZE(prog->pipelines); ++i) {
_mesa_hash_table_init(&prog->pipelines[i], prog, NULL, equals_gfx_pipeline_state);
/* only need first 3/4 for point/line/tri/patch */
if (screen->info.have_EXT_extended_dynamic_state &&
i == (prog->last_vertex_stage->nir->info.stage == MESA_SHADER_TESS_EVAL ? 4 : 3))
break;
}
struct mesa_sha1 sctx;
_mesa_sha1_init(&sctx);
for (int i = 0; i < ZINK_SHADER_COUNT; ++i) {
if (prog->modules[i]) {
simple_mtx_lock(&prog->shaders[i]->lock);
_mesa_set_add(prog->shaders[i]->programs, prog);
simple_mtx_unlock(&prog->shaders[i]->lock);
zink_gfx_program_reference(screen, NULL, prog);
_mesa_sha1_update(&sctx, prog->shaders[i]->base.sha1, sizeof(prog->shaders[i]->base.sha1));
}
}
_mesa_sha1_final(&sctx, prog->base.sha1);
p_atomic_dec(&prog->base.reference.count);
if (!screen->descriptor_program_init(ctx, &prog->base))
goto fail;
zink_screen_get_pipeline_cache(screen, &prog->base);
return prog;
fail:
if (prog)
zink_destroy_gfx_program(screen, prog);
return NULL;
}
static uint32_t
hash_compute_pipeline_state(const void *key)
{
const struct zink_compute_pipeline_state *state = key;
uint32_t hash = _mesa_hash_data(state, offsetof(struct zink_compute_pipeline_state, hash));
if (state->use_local_size)
hash = XXH32(&state->local_size[0], sizeof(state->local_size), hash);
return hash;
}
void
zink_program_update_compute_pipeline_state(struct zink_context *ctx, struct zink_compute_program *comp, const uint block[3])
{
struct zink_shader *zs = comp->shader;
bool use_local_size = !(zs->nir->info.workgroup_size[0] ||
zs->nir->info.workgroup_size[1] ||
zs->nir->info.workgroup_size[2]);
if (ctx->compute_pipeline_state.use_local_size != use_local_size)
ctx->compute_pipeline_state.dirty = true;
ctx->compute_pipeline_state.use_local_size = use_local_size;
if (ctx->compute_pipeline_state.use_local_size) {
for (int i = 0; i < ARRAY_SIZE(ctx->compute_pipeline_state.local_size); i++) {
if (ctx->compute_pipeline_state.local_size[i] != block[i])
ctx->compute_pipeline_state.dirty = true;
ctx->compute_pipeline_state.local_size[i] = block[i];
}
} else
ctx->compute_pipeline_state.local_size[0] =
ctx->compute_pipeline_state.local_size[1] =
ctx->compute_pipeline_state.local_size[2] = 0;
}
static bool
equals_compute_pipeline_state(const void *a, const void *b)
{
return memcmp(a, b, offsetof(struct zink_compute_pipeline_state, hash)) == 0;
}
struct zink_compute_program *
zink_create_compute_program(struct zink_context *ctx, struct zink_shader *shader)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
struct zink_compute_program *comp = rzalloc(NULL, struct zink_compute_program);
if (!comp)
goto fail;
pipe_reference_init(&comp->base.reference, 1);
comp->base.is_compute = true;
/* TODO: cs shader keys placeholder for now */
_mesa_hash_table_init(&comp->base.shader_cache[0], comp, _mesa_hash_pointer, _mesa_key_pointer_equal);
comp->module = CALLOC_STRUCT(zink_shader_module);
assert(comp->module);
comp->module->shader = zink_shader_compile(screen, shader, shader->nir, NULL);
assert(comp->module->shader);
_mesa_hash_table_insert(&comp->base.shader_cache[0], shader, comp->module);
comp->pipelines = _mesa_hash_table_create(NULL, hash_compute_pipeline_state,
equals_compute_pipeline_state);
_mesa_set_add(shader->programs, comp);
comp->shader = shader;
memcpy(comp->base.sha1, shader->base.sha1, sizeof(shader->base.sha1));
if (!screen->descriptor_program_init(ctx, &comp->base))
goto fail;
zink_screen_get_pipeline_cache(screen, &comp->base);
return comp;
fail:
if (comp)
zink_destroy_compute_program(screen, comp);
return NULL;
}
uint32_t
zink_program_get_descriptor_usage(struct zink_context *ctx, enum pipe_shader_type stage, enum zink_descriptor_type type)
{
struct zink_shader *zs = NULL;
switch (stage) {
case PIPE_SHADER_VERTEX:
case PIPE_SHADER_TESS_CTRL:
case PIPE_SHADER_TESS_EVAL:
case PIPE_SHADER_GEOMETRY:
case PIPE_SHADER_FRAGMENT:
zs = ctx->gfx_stages[stage];
break;
case PIPE_SHADER_COMPUTE: {
zs = ctx->compute_stage;
break;
}
default:
unreachable("unknown shader type");
}
if (!zs)
return 0;
switch (type) {
case ZINK_DESCRIPTOR_TYPE_UBO:
return zs->ubos_used;
case ZINK_DESCRIPTOR_TYPE_SSBO:
return zs->ssbos_used;
case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
return BITSET_TEST_RANGE(zs->nir->info.textures_used, 0, PIPE_MAX_SAMPLERS - 1);
case ZINK_DESCRIPTOR_TYPE_IMAGE:
return zs->nir->info.images_used;
default:
unreachable("unknown descriptor type!");
}
return 0;
}
bool
zink_program_descriptor_is_buffer(struct zink_context *ctx, enum pipe_shader_type stage, enum zink_descriptor_type type, unsigned i)
{
struct zink_shader *zs = NULL;
switch (stage) {
case PIPE_SHADER_VERTEX:
case PIPE_SHADER_TESS_CTRL:
case PIPE_SHADER_TESS_EVAL:
case PIPE_SHADER_GEOMETRY:
case PIPE_SHADER_FRAGMENT:
zs = ctx->gfx_stages[stage];
break;
case PIPE_SHADER_COMPUTE: {
zs = ctx->compute_stage;
break;
}
default:
unreachable("unknown shader type");
}
if (!zs)
return false;
return zink_shader_descriptor_is_buffer(zs, type, i);
}
static unsigned
get_num_bindings(struct zink_shader *zs, enum zink_descriptor_type type)
{
switch (type) {
case ZINK_DESCRIPTOR_TYPE_UBO:
case ZINK_DESCRIPTOR_TYPE_SSBO:
return zs->num_bindings[type];
default:
break;
}
unsigned num_bindings = 0;
for (int i = 0; i < zs->num_bindings[type]; i++)
num_bindings += zs->bindings[type][i].size;
return num_bindings;
}
unsigned
zink_program_num_bindings_typed(const struct zink_program *pg, enum zink_descriptor_type type, bool is_compute)
{
unsigned num_bindings = 0;
if (is_compute) {
struct zink_compute_program *comp = (void*)pg;
return get_num_bindings(comp->shader, type);
}
struct zink_gfx_program *prog = (void*)pg;
for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++) {
if (prog->shaders[i])
num_bindings += get_num_bindings(prog->shaders[i], type);
}
return num_bindings;
}
unsigned
zink_program_num_bindings(const struct zink_program *pg, bool is_compute)
{
unsigned num_bindings = 0;
for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++)
num_bindings += zink_program_num_bindings_typed(pg, i, is_compute);
return num_bindings;
}
void
zink_destroy_gfx_program(struct zink_screen *screen,
struct zink_gfx_program *prog)
{
if (prog->base.layout)
VKSCR(DestroyPipelineLayout)(screen->dev, prog->base.layout, NULL);
for (int i = 0; i < ZINK_SHADER_COUNT; ++i) {
if (prog->shaders[i]) {
_mesa_set_remove_key(prog->shaders[i]->programs, prog);
prog->shaders[i] = NULL;
}
destroy_shader_cache(screen, &prog->base.shader_cache[i]);
ralloc_free(prog->nir[i]);
}
unsigned max_idx = ARRAY_SIZE(prog->pipelines);
if (screen->info.have_EXT_extended_dynamic_state) {
/* only need first 3/4 for point/line/tri/patch */
if ((prog->stages_present &
(BITFIELD_BIT(PIPE_SHADER_TESS_EVAL) | BITFIELD_BIT(PIPE_SHADER_GEOMETRY))) ==
BITFIELD_BIT(PIPE_SHADER_TESS_EVAL))
max_idx = 4;
else
max_idx = 3;
max_idx++;
}
for (int i = 0; i < max_idx; ++i) {
hash_table_foreach(&prog->pipelines[i], entry) {
struct gfx_pipeline_cache_entry *pc_entry = entry->data;
VKSCR(DestroyPipeline)(screen->dev, pc_entry->pipeline, NULL);
free(pc_entry);
}
}
if (prog->base.pipeline_cache)
VKSCR(DestroyPipelineCache)(screen->dev, prog->base.pipeline_cache, NULL);
screen->descriptor_program_deinit(screen, &prog->base);
ralloc_free(prog);
}
void
zink_destroy_compute_program(struct zink_screen *screen,
struct zink_compute_program *comp)
{
if (comp->base.layout)
VKSCR(DestroyPipelineLayout)(screen->dev, comp->base.layout, NULL);
if (comp->shader)
_mesa_set_remove_key(comp->shader->programs, comp);
hash_table_foreach(comp->pipelines, entry) {
struct compute_pipeline_cache_entry *pc_entry = entry->data;
VKSCR(DestroyPipeline)(screen->dev, pc_entry->pipeline, NULL);
free(pc_entry);
}
_mesa_hash_table_destroy(comp->pipelines, NULL);
destroy_shader_cache(screen, &comp->base.shader_cache[0]);
if (comp->base.pipeline_cache)
VKSCR(DestroyPipelineCache)(screen->dev, comp->base.pipeline_cache, NULL);
screen->descriptor_program_deinit(screen, &comp->base);
ralloc_free(comp);
}
static unsigned
get_pipeline_idx(bool have_EXT_extended_dynamic_state, enum pipe_prim_type mode, VkPrimitiveTopology vkmode)
{
/* VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY_EXT specifies that the topology state in
* VkPipelineInputAssemblyStateCreateInfo only specifies the topology class,
* and the specific topology order and adjacency must be set dynamically
* with vkCmdSetPrimitiveTopologyEXT before any drawing commands.
*/
if (have_EXT_extended_dynamic_state) {
if (mode == PIPE_PRIM_PATCHES)
return 3;
switch (u_reduced_prim(mode)) {
case PIPE_PRIM_POINTS:
return 0;
case PIPE_PRIM_LINES:
return 1;
default:
return 2;
}
}
return vkmode;
}
VkPipeline
zink_get_gfx_pipeline(struct zink_context *ctx,
struct zink_gfx_program *prog,
struct zink_gfx_pipeline_state *state,
enum pipe_prim_type mode)
{
struct zink_screen *screen = zink_screen(ctx->base.screen);
const bool have_EXT_vertex_input_dynamic_state = screen->info.have_EXT_vertex_input_dynamic_state;
const bool have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state;
VkPrimitiveTopology vkmode = zink_primitive_topology(mode);
const unsigned idx = get_pipeline_idx(screen->info.have_EXT_extended_dynamic_state, mode, vkmode);
assert(idx <= ARRAY_SIZE(prog->pipelines));
if (!state->dirty && !state->modules_changed &&
(have_EXT_vertex_input_dynamic_state || !ctx->vertex_state_changed) &&
idx == state->idx)
return state->pipeline;
struct hash_entry *entry = NULL;
if (state->dirty) {
if (state->pipeline) //avoid on first hash
state->final_hash ^= state->hash;
state->hash = hash_gfx_pipeline_state(state);
state->final_hash ^= state->hash;
state->dirty = false;
}
if (!have_EXT_vertex_input_dynamic_state && ctx->vertex_state_changed) {
if (state->pipeline)
state->final_hash ^= state->vertex_hash;
if (!have_EXT_extended_dynamic_state) {
uint32_t hash = 0;
/* if we don't have dynamic states, we have to hash the enabled vertex buffer bindings */
uint32_t vertex_buffers_enabled_mask = state->vertex_buffers_enabled_mask;
hash = XXH32(&vertex_buffers_enabled_mask, sizeof(uint32_t), hash);
for (unsigned i = 0; i < state->element_state->num_bindings; i++) {
struct pipe_vertex_buffer *vb = ctx->vertex_buffers + ctx->element_state->binding_map[i];
state->vertex_strides[i] = vb->buffer.resource ? vb->stride : 0;
hash = XXH32(&state->vertex_strides[i], sizeof(uint32_t), hash);
}
state->vertex_hash = hash ^ state->element_state->hash;
} else
state->vertex_hash = state->element_state->hash;
state->final_hash ^= state->vertex_hash;
}
state->modules_changed = false;
ctx->vertex_state_changed = false;
entry = _mesa_hash_table_search_pre_hashed(&prog->pipelines[idx], state->final_hash, state);
if (!entry) {
util_queue_fence_wait(&prog->base.cache_fence);
VkPipeline pipeline = zink_create_gfx_pipeline(screen, prog,
state, vkmode);
if (pipeline == VK_NULL_HANDLE)
return VK_NULL_HANDLE;
struct gfx_pipeline_cache_entry *pc_entry = CALLOC_STRUCT(gfx_pipeline_cache_entry);
if (!pc_entry)
return VK_NULL_HANDLE;
memcpy(&pc_entry->state, state, sizeof(*state));
pc_entry->pipeline = pipeline;
entry = _mesa_hash_table_insert_pre_hashed(&prog->pipelines[idx], state->final_hash, pc_entry, pc_entry);
assert(entry);
}
struct gfx_pipeline_cache_entry *cache_entry = entry->data;
state->pipeline = cache_entry->pipeline;
state->idx = idx;
return state->pipeline;
}
VkPipeline
zink_get_compute_pipeline(struct zink_screen *screen,
struct zink_compute_program *comp,
struct zink_compute_pipeline_state *state)
{
struct hash_entry *entry = NULL;
if (!state->dirty)
return state->pipeline;
if (state->dirty) {
state->hash = hash_compute_pipeline_state(state);
state->dirty = false;
}
entry = _mesa_hash_table_search_pre_hashed(comp->pipelines, state->hash, state);
if (!entry) {
util_queue_fence_wait(&comp->base.cache_fence);
VkPipeline pipeline = zink_create_compute_pipeline(screen, comp, state);
if (pipeline == VK_NULL_HANDLE)
return VK_NULL_HANDLE;
struct compute_pipeline_cache_entry *pc_entry = CALLOC_STRUCT(compute_pipeline_cache_entry);
if (!pc_entry)
return VK_NULL_HANDLE;
memcpy(&pc_entry->state, state, sizeof(*state));
pc_entry->pipeline = pipeline;
entry = _mesa_hash_table_insert_pre_hashed(comp->pipelines, state->hash, pc_entry, pc_entry);
assert(entry);
}
struct compute_pipeline_cache_entry *cache_entry = entry->data;
state->pipeline = cache_entry->pipeline;
return state->pipeline;
}
static inline void
bind_stage(struct zink_context *ctx, enum pipe_shader_type stage,
struct zink_shader *shader)
{
if (shader && shader->nir->info.num_inlinable_uniforms)
ctx->shader_has_inlinable_uniforms_mask |= 1 << stage;
else
ctx->shader_has_inlinable_uniforms_mask &= ~(1 << stage);
if (stage == PIPE_SHADER_COMPUTE) {
if (shader && shader != ctx->compute_stage) {
struct hash_entry *entry = _mesa_hash_table_search(&ctx->compute_program_cache, shader);
if (entry) {
ctx->compute_pipeline_state.dirty = true;
ctx->curr_compute = entry->data;
} else {
struct zink_compute_program *comp = zink_create_compute_program(ctx, shader);
_mesa_hash_table_insert(&ctx->compute_program_cache, comp->shader, comp);
ctx->compute_pipeline_state.dirty = true;
ctx->curr_compute = comp;
zink_batch_reference_program(&ctx->batch, &ctx->curr_compute->base);
}
} else if (!shader)
ctx->curr_compute = NULL;
ctx->compute_stage = shader;
zink_select_launch_grid(ctx);
} else {
if (ctx->gfx_stages[stage])
ctx->gfx_hash ^= ctx->gfx_stages[stage]->hash;
ctx->gfx_stages[stage] = shader;
ctx->gfx_dirty = ctx->gfx_stages[PIPE_SHADER_FRAGMENT] && ctx->gfx_stages[PIPE_SHADER_VERTEX];
ctx->gfx_pipeline_state.modules_changed = true;
if (shader) {
ctx->shader_stages |= BITFIELD_BIT(stage);
ctx->gfx_hash ^= ctx->gfx_stages[stage]->hash;
} else {
ctx->gfx_pipeline_state.modules[stage] = VK_NULL_HANDLE;
if (ctx->curr_program)
ctx->gfx_pipeline_state.final_hash ^= ctx->curr_program->last_variant_hash;
ctx->curr_program = NULL;
ctx->shader_stages &= ~BITFIELD_BIT(stage);
}
}
}
static void
zink_bind_vs_state(struct pipe_context *pctx,
void *cso)
{
struct zink_context *ctx = zink_context(pctx);
bind_stage(ctx, PIPE_SHADER_VERTEX, cso);
if (!ctx->gfx_stages[PIPE_SHADER_GEOMETRY] &&
!ctx->gfx_stages[PIPE_SHADER_TESS_EVAL]) {
ctx->last_vertex_stage = cso;
}
if (cso) {
struct zink_shader *zs = cso;
ctx->shader_reads_drawid = BITSET_TEST(zs->nir->info.system_values_read, SYSTEM_VALUE_DRAW_ID);
ctx->shader_reads_basevertex = BITSET_TEST(zs->nir->info.system_values_read, SYSTEM_VALUE_BASE_VERTEX);
} else {
ctx->shader_reads_drawid = false;
ctx->shader_reads_basevertex = false;
}
}
static void
zink_bind_fs_state(struct pipe_context *pctx,
void *cso)
{
struct zink_context *ctx = zink_context(pctx);
bind_stage(ctx, PIPE_SHADER_FRAGMENT, cso);
ctx->fbfetch_outputs = 0;
if (cso) {
nir_shader *nir = ctx->gfx_stages[PIPE_SHADER_FRAGMENT]->nir;
if (nir->info.fs.uses_fbfetch_output) {
nir_foreach_shader_out_variable(var, ctx->gfx_stages[PIPE_SHADER_FRAGMENT]->nir) {
if (var->data.fb_fetch_output)
ctx->fbfetch_outputs |= BITFIELD_BIT(var->data.location - FRAG_RESULT_DATA0);
}
}
}
zink_update_fbfetch(ctx);
}
static void
zink_bind_gs_state(struct pipe_context *pctx,
void *cso)
{
struct zink_context *ctx = zink_context(pctx);
bool had_points = ctx->gfx_stages[PIPE_SHADER_GEOMETRY] ? ctx->gfx_stages[PIPE_SHADER_GEOMETRY]->nir->info.gs.output_primitive == GL_POINTS : false;
if (!!ctx->gfx_stages[PIPE_SHADER_GEOMETRY] != !!cso)
ctx->dirty_shader_stages |= BITFIELD_BIT(PIPE_SHADER_VERTEX) |
BITFIELD_BIT(PIPE_SHADER_TESS_EVAL);
bind_stage(ctx, PIPE_SHADER_GEOMETRY, cso);
if (cso) {
ctx->last_vertex_stage = cso;
if (!had_points && ctx->last_vertex_stage->nir->info.gs.output_primitive == GL_POINTS)
ctx->gfx_pipeline_state.has_points++;
} else {
if (ctx->gfx_stages[PIPE_SHADER_TESS_EVAL])
ctx->last_vertex_stage = ctx->gfx_stages[PIPE_SHADER_TESS_EVAL];
else
ctx->last_vertex_stage = ctx->gfx_stages[PIPE_SHADER_VERTEX];
if (had_points)
ctx->gfx_pipeline_state.has_points--;
}
}
static void
zink_bind_tcs_state(struct pipe_context *pctx,
void *cso)
{
bind_stage(zink_context(pctx), PIPE_SHADER_TESS_CTRL, cso);
}
static void
zink_bind_tes_state(struct pipe_context *pctx,
void *cso)
{
struct zink_context *ctx = zink_context(pctx);
if (!!ctx->gfx_stages[PIPE_SHADER_TESS_EVAL] != !!cso) {
if (!cso) {
/* if unsetting a TESS that uses a generated TCS, ensure the TCS is unset */
if (ctx->gfx_stages[PIPE_SHADER_TESS_EVAL]->generated)
ctx->gfx_stages[PIPE_SHADER_TESS_CTRL] = NULL;
}
ctx->dirty_shader_stages |= BITFIELD_BIT(PIPE_SHADER_VERTEX);
}
bind_stage(ctx, PIPE_SHADER_TESS_EVAL, cso);
if (!ctx->gfx_stages[PIPE_SHADER_GEOMETRY]) {
if (cso)
ctx->last_vertex_stage = cso;
else
ctx->last_vertex_stage = ctx->gfx_stages[PIPE_SHADER_VERTEX];
}
}
static void *
zink_create_cs_state(struct pipe_context *pctx,
const struct pipe_compute_state *shader)
{
struct nir_shader *nir;
if (shader->ir_type != PIPE_SHADER_IR_NIR)
nir = zink_tgsi_to_nir(pctx->screen, shader->prog);
else
nir = (struct nir_shader *)shader->prog;
return zink_shader_create(zink_screen(pctx->screen), nir, NULL);
}
static void
zink_bind_cs_state(struct pipe_context *pctx,
void *cso)
{
bind_stage(zink_context(pctx), PIPE_SHADER_COMPUTE, cso);
}
void
zink_delete_shader_state(struct pipe_context *pctx, void *cso)
{
zink_shader_free(zink_context(pctx), cso);
}
void *
zink_create_gfx_shader_state(struct pipe_context *pctx, const struct pipe_shader_state *shader)
{
nir_shader *nir;
if (shader->type != PIPE_SHADER_IR_NIR)
nir = zink_tgsi_to_nir(pctx->screen, shader->tokens);
else
nir = (struct nir_shader *)shader->ir.nir;
return zink_shader_create(zink_screen(pctx->screen), nir, &shader->stream_output);
}
static void
zink_delete_cached_shader_state(struct pipe_context *pctx, void *cso)
{
struct zink_screen *screen = zink_screen(pctx->screen);
util_shader_reference(pctx, &screen->shaders, &cso, NULL);
}
static void *
zink_create_cached_shader_state(struct pipe_context *pctx, const struct pipe_shader_state *shader)
{
bool cache_hit;
struct zink_screen *screen = zink_screen(pctx->screen);
return util_live_shader_cache_get(pctx, &screen->shaders, shader, &cache_hit);
}
void
zink_program_init(struct zink_context *ctx)
{
ctx->base.create_vs_state = zink_create_cached_shader_state;
ctx->base.bind_vs_state = zink_bind_vs_state;
ctx->base.delete_vs_state = zink_delete_cached_shader_state;
ctx->base.create_fs_state = zink_create_cached_shader_state;
ctx->base.bind_fs_state = zink_bind_fs_state;
ctx->base.delete_fs_state = zink_delete_cached_shader_state;
ctx->base.create_gs_state = zink_create_cached_shader_state;
ctx->base.bind_gs_state = zink_bind_gs_state;
ctx->base.delete_gs_state = zink_delete_cached_shader_state;
ctx->base.create_tcs_state = zink_create_cached_shader_state;
ctx->base.bind_tcs_state = zink_bind_tcs_state;
ctx->base.delete_tcs_state = zink_delete_cached_shader_state;
ctx->base.create_tes_state = zink_create_cached_shader_state;
ctx->base.bind_tes_state = zink_bind_tes_state;
ctx->base.delete_tes_state = zink_delete_cached_shader_state;
ctx->base.create_compute_state = zink_create_cs_state;
ctx->base.bind_compute_state = zink_bind_cs_state;
ctx->base.delete_compute_state = zink_delete_shader_state;
}
Reference in a new issue View git blame Copy permalink