fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2026-04-20 01:20:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

iris: Reduce dependency on brw_*_prog_data structs

Browse source 
Once the brw_*_prog_data are available, copy down all the relevant
fields to iris_compiled_shader (and iris_*_data corresponding structs)
so that most of Iris code will be independent of brw types.

Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27646>
This commit is contained in:
Caio Oliveira 2024-02-06 11:05:33 -08:00 • committed by Marge Bot
parent 9fd4b7460f
commit 04364768f2
6 changed files with 712 additions and 292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

220
src/gallium/drivers/iris/iris_context.h
View file

@ -304,6 +304,137 @@ PRAGMA_DIAGNOSTIC_POP
/** @} */
struct iris_ubo_range
{
uint16_t block;
/* In units of 32-byte registers */
uint8_t start;
uint8_t length;
};
struct iris_fs_data {
int urb_setup[VARYING_SLOT_MAX];
uint8_t urb_setup_attribs[VARYING_SLOT_MAX];
uint8_t urb_setup_attribs_count;
uint64_t inputs;
unsigned num_varying_inputs;
unsigned msaa_flags_param;
uint32_t flat_inputs;
uint8_t computed_depth_mode;
uint8_t max_polygons;
uint8_t dispatch_multi;
bool computed_stencil;
bool early_fragment_tests;
bool post_depth_coverage;
bool inner_coverage;
bool dispatch_8;
bool dispatch_16;
bool dispatch_32;
bool dual_src_blend;
bool uses_pos_offset;
bool uses_omask;
bool uses_kill;
bool uses_src_depth;
bool uses_src_w;
bool uses_sample_mask;
bool uses_vmask;
bool has_side_effects;
bool pulls_bary;
bool uses_nonperspective_interp_modes;
bool is_per_sample;
};
struct iris_push_const_block {
unsigned dwords; /* Dword count, not reg aligned */
unsigned regs;
unsigned size; /* Bytes, register aligned */
};
struct iris_cs_data {
struct {
struct iris_push_const_block cross_thread;
struct iris_push_const_block per_thread;
} push;
unsigned local_size[3];
unsigned prog_offset[3];
unsigned prog_mask;
uint8_t generate_local_id;
enum intel_compute_walk_order walk_order;
bool uses_barrier;
bool first_param_is_builtin_subgroup_id;
};
static inline uint32_t
iris_cs_data_prog_offset(const struct iris_cs_data *prog_data,
unsigned dispatch_width)
{
assert(dispatch_width == 8 ||
dispatch_width == 16 ||
dispatch_width == 32);
const unsigned index = dispatch_width / 16;
assert(prog_data->prog_mask & (1 << index));
return prog_data->prog_offset[index];
}
struct iris_vue_data {
struct intel_vue_map vue_map;
unsigned urb_read_length;
uint32_t cull_distance_mask;
unsigned urb_entry_size;
enum intel_shader_dispatch_mode dispatch_mode;
bool include_vue_handles;
};
struct iris_vs_data {
struct iris_vue_data base;
bool uses_vertexid;
bool uses_instanceid;
bool uses_firstvertex;
bool uses_baseinstance;
bool uses_drawid;
};
struct iris_tcs_data {
struct iris_vue_data base;
int instances;
int patch_count_threshold;
bool include_primitive_id;
};
struct iris_tes_data {
struct iris_vue_data base;
enum intel_tess_partitioning partitioning;
enum intel_tess_output_topology output_topology;
enum intel_tess_domain domain;
bool include_primitive_id;
};
struct iris_gs_data {
struct iris_vue_data base;
unsigned vertices_in;
unsigned output_vertex_size_hwords;
unsigned output_topology;
unsigned control_data_header_size_hwords;
unsigned control_data_format;
int static_vertex_count;
int invocations;
bool include_primitive_id;
};
struct iris_depth_stencil_alpha_state;
/**
@ -533,7 +664,7 @@ struct iris_compiled_shader {
void *map;
/** The program data (owned by the program cache hash table) */
struct brw_stage_prog_data *prog_data;
struct brw_stage_prog_data *brw_prog_data;
/** A list of system values to be uploaded as uniforms. */
enum brw_param_builtin *system_values;
@ -555,6 +686,29 @@ struct iris_compiled_shader {
gl_shader_stage stage;
/**
* Data derived from prog_data.
*/
struct iris_ubo_range ubo_ranges[4];
unsigned nr_params;
unsigned total_scratch;
unsigned total_shared;
unsigned program_size;
unsigned const_data_offset;
unsigned dispatch_grf_start_reg;
bool has_ubo_pull;
bool use_alt_mode;
union {
struct iris_fs_data fs;
struct iris_cs_data cs;
struct iris_vs_data vs;
struct iris_tcs_data tcs;
struct iris_tes_data tes;
struct iris_gs_data gs;
};
/**
* Shader packets and other data derived from prog_data. These must be
* completely determined from prog_data.
@ -569,6 +723,43 @@ KSP(const struct iris_compiled_shader *shader)
return iris_bo_offset_from_base_address(res->bo) + shader->assembly.offset;
}
#define DEFINE_IRIS_SHADER_DATA(TYPE, STAGE, FIELD) \
static inline TYPE * \
iris_ ## FIELD ## _data(struct iris_compiled_shader *shader) \
{ \
assert(shader->stage == STAGE); \
return &shader->FIELD; \
} \
static inline const TYPE * \
iris_ ## FIELD ## _data_const(const struct iris_compiled_shader *shader) \
{ \
assert(shader->stage == STAGE); \
return &shader->FIELD; \
}
DEFINE_IRIS_SHADER_DATA(struct iris_fs_data, MESA_SHADER_FRAGMENT, fs)
DEFINE_IRIS_SHADER_DATA(struct iris_cs_data, MESA_SHADER_COMPUTE, cs)
DEFINE_IRIS_SHADER_DATA(struct iris_vs_data, MESA_SHADER_VERTEX, vs)
DEFINE_IRIS_SHADER_DATA(struct iris_tcs_data, MESA_SHADER_TESS_CTRL, tcs)
DEFINE_IRIS_SHADER_DATA(struct iris_tes_data, MESA_SHADER_TESS_EVAL, tes)
DEFINE_IRIS_SHADER_DATA(struct iris_gs_data, MESA_SHADER_GEOMETRY, gs)
#undef DEFINE_IRIS_SHADER_DATA
static inline struct iris_vue_data *
iris_vue_data(struct iris_compiled_shader *shader)
{
switch (shader->stage) {
case MESA_SHADER_VERTEX: return &shader->vs.base;
case MESA_SHADER_TESS_CTRL: return &shader->tcs.base;
case MESA_SHADER_TESS_EVAL: return &shader->tes.base;
case MESA_SHADER_GEOMETRY: return &shader->gs.base;
default:
unreachable("invalid shader stage for vue prog data");
return NULL;
}
}
/**
* API context state that is replicated per shader stage.
*/
@ -1033,7 +1224,8 @@ void iris_init_resource_functions(struct pipe_context *ctx);
void iris_init_perfquery_functions(struct pipe_context *ctx);
void iris_update_compiled_shaders(struct iris_context *ice);
void iris_update_compiled_compute_shader(struct iris_context *ice);
void iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data,
void iris_fill_cs_push_const_buffer(struct iris_screen *screen,
struct iris_compiled_shader *shader,
unsigned threads,
uint32_t *dst);
@ -1116,6 +1308,18 @@ uint32_t iris_group_index_to_bti(const struct iris_binding_table *bt,
uint32_t iris_bti_to_group_index(const struct iris_binding_table *bt,
enum iris_surface_group group,
uint32_t bti);
void iris_apply_brw_prog_data(struct iris_compiled_shader *shader,
const struct brw_stage_prog_data *prog_data);
struct intel_cs_dispatch_info
iris_get_cs_dispatch_info(const struct intel_device_info *devinfo,
const struct iris_compiled_shader *shader,
const uint32_t block[3]);
unsigned
iris_cs_push_const_total_size(const struct iris_compiled_shader *shader,
unsigned threads);
uint32_t
iris_fs_barycentric_modes(const struct iris_compiled_shader *shader,
enum intel_msaa_flags pushed_msaa_flags);
/* iris_disk_cache.c */
@ -1149,7 +1353,7 @@ struct iris_compiled_shader *iris_create_shader_variant(const struct iris_screen
const void *key);
void iris_finalize_program(struct iris_compiled_shader *shader,
struct brw_stage_prog_data *prog_data,
struct brw_stage_prog_data *brw_prog_data,
uint32_t *streamout,
enum brw_param_builtin *system_values,
unsigned num_system_values,
@ -1248,8 +1452,8 @@ iris_execute_indirect_draw_supported(const struct iris_context *ice,
const struct pipe_draw_info *draw)
{
const struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
const struct brw_vs_prog_data *vs_prog_data = (void *)
ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data;
const struct iris_vs_data *vs_data =
iris_vs_data(ice->shaders.prog[MESA_SHADER_VERTEX]);
const bool is_multiview = draw->view_mask != 0;
const size_t struct_size = draw->index_size ?
sizeof(uint32_t) * 5 :
@ -1262,9 +1466,9 @@ iris_execute_indirect_draw_supported(const struct iris_context *ice,
(indirect &&
!indirect->count_from_stream_output) &&
!is_multiview &&
!(vs_prog_data->uses_firstvertex ||
vs_prog_data->uses_baseinstance ||
vs_prog_data->uses_drawid));
!(vs_data->uses_firstvertex ||
vs_data->uses_baseinstance ||
vs_data->uses_drawid));
}
#ifdef genX

14
src/gallium/drivers/iris/iris_disk_cache.c
View file

@ -86,7 +86,7 @@ iris_disk_cache_store(struct disk_cache *cache,
return;
gl_shader_stage stage = ish->nir->info.stage;
const struct brw_stage_prog_data *prog_data = shader->prog_data;
const struct brw_stage_prog_data *prog_data = shader->brw_prog_data;
cache_key cache_key;
iris_disk_cache_compute_key(cache, ish, prog_key, prog_key_size, cache_key);
@ -116,12 +116,12 @@ iris_disk_cache_store(struct disk_cache *cache,
size_t prog_data_s = brw_prog_data_size(stage);
union brw_any_prog_data serializable;
assert(prog_data_s <= sizeof(serializable));
memcpy(&serializable, shader->prog_data, prog_data_s);
memcpy(&serializable, shader->brw_prog_data, prog_data_s);
serializable.base.param = NULL;
serializable.base.relocs = NULL;
blob_write_bytes(&blob, &serializable, prog_data_s);
blob_write_bytes(&blob, shader->map, shader->prog_data->program_size);
blob_write_bytes(&blob, shader->map, shader->program_size);
blob_write_uint32(&blob, shader->num_system_values);
blob_write_bytes(&blob, shader->system_values,
shader->num_system_values * sizeof(enum brw_param_builtin));
@ -228,9 +228,9 @@ iris_disk_cache_retrieve(struct iris_screen *screen,
if (stage == MESA_SHADER_VERTEX ||
stage == MESA_SHADER_TESS_EVAL ||
stage == MESA_SHADER_GEOMETRY) {
struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output,
&vue_prog_data->vue_map);
struct intel_vue_map *vue_map =
&brw_vue_prog_data(prog_data)->vue_map;
so_decls = screen->vtbl.create_so_decl_list(&ish->stream_output, vue_map);
}
/* System values and uniforms are stored in constant buffer 0, the
@ -245,6 +245,8 @@ iris_disk_cache_retrieve(struct iris_screen *screen,
if (num_system_values || kernel_input_size)
num_cbufs++;
iris_apply_brw_prog_data(shader, prog_data);
iris_finalize_program(shader, prog_data, so_decls, system_values,
num_system_values, kernel_input_size, num_cbufs,
&bt);

31
src/gallium/drivers/iris/iris_indirect_gen.c
View file

@ -263,14 +263,14 @@ emit_indirect_generate_draw(struct iris_batch *batch,
raster.CullMode = CULLMODE_NONE;
}
const struct brw_wm_prog_data *wm_prog_data = (void *)
ice->draw.generation.shader->prog_data;
const struct iris_compiled_shader *shader = ice->draw.generation.shader;
const struct iris_fs_data *fs_data = iris_fs_data_const(shader);
iris_emit_cmd(batch, GENX(3DSTATE_SBE), sbe) {
sbe.VertexURBEntryReadOffset = 1;
sbe.NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs;
sbe.VertexURBEntryReadLength = MAX2((wm_prog_data->num_varying_inputs + 1) / 2, 1);
sbe.ConstantInterpolationEnable = wm_prog_data->flat_inputs;
sbe.NumberofSFOutputAttributes = fs_data->num_varying_inputs;
sbe.VertexURBEntryReadLength = MAX2((fs_data->num_varying_inputs + 1) / 2, 1);
sbe.ConstantInterpolationEnable = fs_data->flat_inputs;
sbe.ForceVertexURBEntryReadLength = true;
sbe.ForceVertexURBEntryReadOffset = true;
#if GFX_VER >= 9
@ -280,21 +280,22 @@ emit_indirect_generate_draw(struct iris_batch *batch,
}
iris_emit_cmd(batch, GENX(3DSTATE_WM), wm) {
if (wm_prog_data->has_side_effects || wm_prog_data->uses_kill)
if (fs_data->has_side_effects || fs_data->uses_kill)
wm.ForceThreadDispatchEnable = ForceON;
}
iris_emit_cmd(batch, GENX(3DSTATE_PS), ps) {
struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(shader->brw_prog_data);
intel_set_ps_dispatch_state(&ps, devinfo, wm_prog_data,
1 /* rasterization_samples */,
0 /* msaa_flags */);
ps.VectorMaskEnable = wm_prog_data->uses_vmask;
ps.VectorMaskEnable = fs_data->uses_vmask;
ps.BindingTableEntryCount = GFX_VER == 9 ? 1 : 0;
#if GFX_VER < 20
ps.PushConstantEnable = wm_prog_data->base.nr_params > 0 ||
wm_prog_data->base.ubo_ranges[0].length;
ps.PushConstantEnable = shader->nr_params > 0 ||
shader->ubo_ranges[0].length;
#endif
ps.DispatchGRFStartRegisterForConstantSetupData0 =
@ -321,17 +322,17 @@ emit_indirect_generate_draw(struct iris_batch *batch,
iris_emit_cmd(batch, GENX(3DSTATE_PS_EXTRA), psx) {
psx.PixelShaderValid = true;
#if GFX_VER < 20
psx.AttributeEnable = wm_prog_data->num_varying_inputs > 0;
psx.AttributeEnable = fs_data->num_varying_inputs > 0;
#endif
psx.PixelShaderIsPerSample = wm_prog_data->persample_dispatch;
psx.PixelShaderComputedDepthMode = wm_prog_data->computed_depth_mode;
psx.PixelShaderIsPerSample = fs_data->is_per_sample;
psx.PixelShaderComputedDepthMode = fs_data->computed_depth_mode;
#if GFX_VER >= 9
#if GFX_VER >= 20
assert(!wm_prog_data->pulls_bary);
assert(!fs_data->pulls_bary);
#else
psx.PixelShaderPullsBary = wm_prog_data->pulls_bary;
psx.PixelShaderPullsBary = fs_data->pulls_bary;
#endif
psx.PixelShaderComputesStencil = wm_prog_data->computed_stencil;
psx.PixelShaderComputesStencil = fs_data->computed_stencil;
#endif
psx.PixelShaderHasUAV = GFX_VER == 8;
}

387
src/gallium/drivers/iris/iris_program.c
View file

@ -70,9 +70,205 @@ get_new_program_id(struct iris_screen *screen)
return p_atomic_inc_return(&screen->program_id);
}
static void
iris_apply_brw_wm_prog_data(struct iris_compiled_shader *shader,
const struct brw_wm_prog_data *brw)
{
assert(shader->stage == MESA_SHADER_FRAGMENT);
struct iris_fs_data *iris = &shader->fs;
STATIC_ASSERT(ARRAY_SIZE(iris->urb_setup) == ARRAY_SIZE(brw->urb_setup));
STATIC_ASSERT(ARRAY_SIZE(iris->urb_setup_attribs) == ARRAY_SIZE(brw->urb_setup_attribs));
memcpy(iris->urb_setup, brw->urb_setup, sizeof(iris->urb_setup));
memcpy(iris->urb_setup_attribs, brw->urb_setup_attribs, brw->urb_setup_attribs_count);
iris->urb_setup_attribs_count = brw->urb_setup_attribs_count;
iris->num_varying_inputs = brw->num_varying_inputs;
iris->msaa_flags_param = brw->msaa_flags_param;
iris->flat_inputs = brw->flat_inputs;
iris->inputs = brw->inputs;
iris->computed_depth_mode = brw->computed_depth_mode;
iris->max_polygons = brw->max_polygons;
iris->dispatch_multi = brw->dispatch_multi;
iris->computed_stencil = brw->computed_stencil;
iris->early_fragment_tests = brw->early_fragment_tests;
iris->post_depth_coverage = brw->post_depth_coverage;
iris->inner_coverage = brw->inner_coverage;
iris->dispatch_8 = brw->dispatch_8;
iris->dispatch_16 = brw->dispatch_16;
iris->dispatch_32 = brw->dispatch_32;
iris->dual_src_blend = brw->dual_src_blend;
iris->uses_pos_offset = brw->uses_pos_offset;
iris->uses_omask = brw->uses_omask;
iris->uses_kill = brw->uses_kill;
iris->uses_src_depth = brw->uses_src_depth;
iris->uses_src_w = brw->uses_src_w;
iris->uses_sample_mask = brw->uses_sample_mask;
iris->uses_vmask = brw->uses_vmask;
iris->has_side_effects = brw->has_side_effects;
iris->pulls_bary = brw->pulls_bary;
iris->uses_nonperspective_interp_modes = brw->uses_nonperspective_interp_modes;
iris->is_per_sample = brw_wm_prog_data_is_persample(brw, 0);
}
static void
iris_apply_brw_cs_prog_data(struct iris_compiled_shader *shader,
const struct brw_cs_prog_data *brw)
{
assert(shader->stage == MESA_SHADER_COMPUTE);
struct iris_cs_data *iris = &shader->cs;
iris->push.cross_thread.dwords = brw->push.cross_thread.dwords;
iris->push.cross_thread.regs = brw->push.cross_thread.regs;
iris->push.cross_thread.size = brw->push.cross_thread.size;
iris->push.per_thread.dwords = brw->push.per_thread.dwords;
iris->push.per_thread.regs = brw->push.per_thread.regs;
iris->push.per_thread.size = brw->push.per_thread.size;
iris->local_size[0] = brw->local_size[0];
iris->local_size[1] = brw->local_size[1];
iris->local_size[2] = brw->local_size[2];
iris->prog_offset[0] = brw->prog_offset[0];
iris->prog_offset[1] = brw->prog_offset[1];
iris->prog_offset[2] = brw->prog_offset[2];
iris->generate_local_id = brw->generate_local_id;
iris->walk_order = brw->walk_order;
iris->uses_barrier = brw->uses_barrier;
iris->prog_mask = brw->prog_mask;
iris->first_param_is_builtin_subgroup_id =
brw->base.nr_params > 0 &&
brw->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID;
}
static void
iris_apply_brw_vue_prog_data(const struct brw_vue_prog_data *brw,
struct iris_vue_data *iris)
{
memcpy(&iris->vue_map, &brw->vue_map, sizeof(struct intel_vue_map));
iris->urb_read_length = brw->urb_read_length;
iris->cull_distance_mask = brw->cull_distance_mask;
iris->urb_entry_size = brw->urb_entry_size;
iris->dispatch_mode = brw->dispatch_mode;
iris->include_vue_handles = brw->include_vue_handles;
}
static void
iris_apply_brw_vs_prog_data(struct iris_compiled_shader *shader,
const struct brw_vs_prog_data *brw)
{
assert(shader->stage == MESA_SHADER_VERTEX);
struct iris_vs_data *iris = &shader->vs;
iris_apply_brw_vue_prog_data(&brw->base, &iris->base);
iris->uses_vertexid = brw->uses_vertexid;
iris->uses_instanceid = brw->uses_instanceid;
iris->uses_firstvertex = brw->uses_firstvertex;
iris->uses_baseinstance = brw->uses_baseinstance;
iris->uses_drawid = brw->uses_drawid;
}
static void
iris_apply_brw_tcs_prog_data(struct iris_compiled_shader *shader,
const struct brw_tcs_prog_data *brw)
{
assert(shader->stage == MESA_SHADER_TESS_CTRL);
struct iris_tcs_data *iris = &shader->tcs;
iris_apply_brw_vue_prog_data(&brw->base, &iris->base);
iris->instances = brw->instances;
iris->patch_count_threshold = brw->patch_count_threshold;
iris->include_primitive_id = brw->include_primitive_id;
}
static void
iris_apply_brw_tes_prog_data(struct iris_compiled_shader *shader,
const struct brw_tes_prog_data *brw)
{
assert(shader->stage == MESA_SHADER_TESS_EVAL);
struct iris_tes_data *iris = &shader->tes;
iris_apply_brw_vue_prog_data(&brw->base, &iris->base);
iris->partitioning = brw->partitioning;
iris->output_topology = brw->output_topology;
iris->domain = brw->domain;
iris->include_primitive_id = brw->include_primitive_id;
}
static void
iris_apply_brw_gs_prog_data(struct iris_compiled_shader *shader,
const struct brw_gs_prog_data *brw)
{
assert(shader->stage == MESA_SHADER_GEOMETRY);
struct iris_gs_data *iris = &shader->gs;
iris_apply_brw_vue_prog_data(&brw->base, &iris->base);
iris->vertices_in = brw->vertices_in;
iris->output_vertex_size_hwords = brw->output_vertex_size_hwords;
iris->output_topology = brw->output_topology;
iris->control_data_header_size_hwords = brw->control_data_header_size_hwords;
iris->control_data_format = brw->control_data_format;
iris->static_vertex_count = brw->static_vertex_count;
iris->invocations = brw->invocations;
iris->include_primitive_id = brw->include_primitive_id;
}
void
iris_apply_brw_prog_data(struct iris_compiled_shader *shader,
const struct brw_stage_prog_data *brw)
{
STATIC_ASSERT(ARRAY_SIZE(brw->ubo_ranges) == ARRAY_SIZE(shader->ubo_ranges));
for (int i = 0; i < ARRAY_SIZE(shader->ubo_ranges); i++) {
shader->ubo_ranges[i].block = brw->ubo_ranges[i].block;
shader->ubo_ranges[i].start = brw->ubo_ranges[i].start;
shader->ubo_ranges[i].length = brw->ubo_ranges[i].length;
}
shader->nr_params = brw->nr_params;
shader->total_scratch = brw->total_scratch;
shader->total_shared = brw->total_shared;
shader->program_size = brw->program_size;
shader->const_data_offset = brw->const_data_offset;
shader->dispatch_grf_start_reg = brw->dispatch_grf_start_reg;
shader->has_ubo_pull = brw->has_ubo_pull;
shader->use_alt_mode = brw->use_alt_mode;
switch (shader->stage) {
case MESA_SHADER_FRAGMENT:
iris_apply_brw_wm_prog_data(shader, brw_wm_prog_data_const(brw));
break;
case MESA_SHADER_COMPUTE:
iris_apply_brw_cs_prog_data(shader, brw_cs_prog_data_const(brw));
break;
case MESA_SHADER_VERTEX:
iris_apply_brw_vs_prog_data(shader, brw_vs_prog_data_const(brw));
break;
case MESA_SHADER_TESS_CTRL:
iris_apply_brw_tcs_prog_data(shader, brw_tcs_prog_data_const(brw));
break;
case MESA_SHADER_TESS_EVAL:
iris_apply_brw_tes_prog_data(shader, brw_tes_prog_data_const(brw));
break;
case MESA_SHADER_GEOMETRY:
iris_apply_brw_gs_prog_data(shader, brw_gs_prog_data_const(brw));
break;
default:
unreachable("invalid shader stage");
}
}
void
iris_finalize_program(struct iris_compiled_shader *shader,
struct brw_stage_prog_data *prog_data,
struct brw_stage_prog_data *brw_prog_data,
uint32_t *streamout,
enum brw_param_builtin *system_values,
unsigned num_system_values,
@ -80,7 +276,7 @@ iris_finalize_program(struct iris_compiled_shader *shader,
unsigned num_cbufs,
const struct iris_binding_table *bt)
{
shader->prog_data = prog_data;
shader->brw_prog_data = brw_prog_data;
shader->streamout = streamout;
shader->system_values = system_values;
shader->num_system_values = num_system_values;
@ -88,9 +284,9 @@ iris_finalize_program(struct iris_compiled_shader *shader,
shader->num_cbufs = num_cbufs;
shader->bt = *bt;
ralloc_steal(shader, shader->prog_data);
ralloc_steal(shader->prog_data, (void *)prog_data->relocs);
ralloc_steal(shader->prog_data, prog_data->param);
ralloc_steal(shader, shader->brw_prog_data);
ralloc_steal(shader->brw_prog_data, (void *)brw_prog_data->relocs);
ralloc_steal(shader->brw_prog_data, brw_prog_data->param);
ralloc_steal(shader, shader->streamout);
ralloc_steal(shader, shader->system_values);
}
@ -1293,10 +1489,6 @@ iris_compile_vs(struct iris_screen *screen,
const struct brw_compiler *compiler = screen->compiler;
const struct intel_device_info *devinfo = screen->devinfo;
void *mem_ctx = ralloc_context(NULL);
struct brw_vs_prog_data *vs_prog_data =
rzalloc(mem_ctx, struct brw_vs_prog_data);
struct brw_vue_prog_data *vue_prog_data = &vs_prog_data->base;
struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
enum brw_param_builtin *system_values;
unsigned num_system_values;
unsigned num_cbufs;
@ -1316,8 +1508,6 @@ iris_compile_vs(struct iris_screen *screen,
}
}
prog_data->use_alt_mode = nir->info.use_legacy_math_rules;
iris_setup_uniforms(devinfo, mem_ctx, nir, 0, &system_values,
&num_system_values, &num_cbufs);
@ -1325,10 +1515,15 @@ iris_compile_vs(struct iris_screen *screen,
iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
num_system_values, num_cbufs);
brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges);
struct brw_vs_prog_data *brw_prog_data =
rzalloc(mem_ctx, struct brw_vs_prog_data);
brw_prog_data->base.base.use_alt_mode = nir->info.use_legacy_math_rules;
brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges);
brw_compute_vue_map(devinfo,
&vue_prog_data->vue_map, nir->info.outputs_written,
&brw_prog_data->base.vue_map, nir->info.outputs_written,
nir->info.separate_shader, /* pos_slots */ 1);
struct brw_vs_prog_key brw_key = iris_to_brw_vs_key(screen, key);
@ -1341,7 +1536,7 @@ iris_compile_vs(struct iris_screen *screen,
.source_hash = ish->source_hash,
},
.key = &brw_key,
.prog_data = vs_prog_data,
.prog_data = brw_prog_data,
};
const unsigned *program = brw_compile_vs(compiler, &params);
@ -1359,12 +1554,14 @@ iris_compile_vs(struct iris_screen *screen,
iris_debug_recompile(screen, dbg, ish, &brw_key.base);
iris_apply_brw_prog_data(shader, &brw_prog_data->base.base);
uint32_t *so_decls =
screen->vtbl.create_so_decl_list(&ish->stream_output,
&vue_prog_data->vue_map);
&iris_vue_data(shader)->vue_map);
iris_finalize_program(shader, prog_data, so_decls, system_values,
num_system_values, 0, num_cbufs, &bt);
iris_finalize_program(shader, &brw_prog_data->base.base, so_decls,
system_values, num_system_values, 0, num_cbufs, &bt);
iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_VS,
sizeof(*key), key, program);
@ -1414,7 +1611,7 @@ iris_update_compiled_vs(struct iris_context *ice)
shs->sysvals_need_upload = true;
unsigned urb_entry_size = shader ?
((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0;
iris_vue_data(shader)->urb_entry_size : 0;
check_urb_size(ice, urb_entry_size, MESA_SHADER_VERTEX);
}
}
@ -1478,10 +1675,6 @@ iris_compile_tcs(struct iris_screen *screen,
{
const struct brw_compiler *compiler = screen->compiler;
void *mem_ctx = ralloc_context(NULL);
struct brw_tcs_prog_data *tcs_prog_data =
rzalloc(mem_ctx, struct brw_tcs_prog_data);
struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base;
struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
const struct intel_device_info *devinfo = screen->devinfo;
enum brw_param_builtin *system_values = NULL;
unsigned num_system_values = 0;
@ -1507,7 +1700,10 @@ iris_compile_tcs(struct iris_screen *screen,
&num_system_values, &num_cbufs);
iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
num_system_values, num_cbufs);
brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges);
struct brw_tcs_prog_data *brw_prog_data =
rzalloc(mem_ctx, struct brw_tcs_prog_data);
brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges);
struct brw_compile_tcs_params params = {
.base = {
@ -1517,7 +1713,7 @@ iris_compile_tcs(struct iris_screen *screen,
.source_hash = source_hash,
},
.key = &brw_key,
.prog_data = tcs_prog_data,
.prog_data = brw_prog_data,
};
const unsigned *program = brw_compile_tcs(compiler, &params);
@ -1535,7 +1731,9 @@ iris_compile_tcs(struct iris_screen *screen,
iris_debug_recompile(screen, dbg, ish, &brw_key.base);
iris_finalize_program(shader, prog_data, NULL, system_values,
iris_apply_brw_prog_data(shader, &brw_prog_data->base.base);
iris_finalize_program(shader, &brw_prog_data->base.base, NULL, system_values,
num_system_values, 0, num_cbufs, &bt);
iris_upload_shader(screen, ish, shader, passthrough_ht, uploader,
@ -1622,7 +1820,7 @@ iris_update_compiled_tcs(struct iris_context *ice)
shs->sysvals_need_upload = true;
unsigned urb_entry_size = shader ?
((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0;
iris_vue_data(shader)->urb_entry_size : 0;
check_urb_size(ice, urb_entry_size, MESA_SHADER_TESS_CTRL);
}
}
@ -1639,10 +1837,6 @@ iris_compile_tes(struct iris_screen *screen,
{
const struct brw_compiler *compiler = screen->compiler;
void *mem_ctx = ralloc_context(NULL);
struct brw_tes_prog_data *tes_prog_data =
rzalloc(mem_ctx, struct brw_tes_prog_data);
struct brw_vue_prog_data *vue_prog_data = &tes_prog_data->base;
struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
enum brw_param_builtin *system_values;
const struct intel_device_info *devinfo = screen->devinfo;
unsigned num_system_values;
@ -1668,7 +1862,10 @@ iris_compile_tes(struct iris_screen *screen,
iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
num_system_values, num_cbufs);
brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges);
struct brw_tes_prog_data *brw_prog_data =
rzalloc(mem_ctx, struct brw_tes_prog_data);
brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges);
struct intel_vue_map input_vue_map;
brw_compute_tess_vue_map(&input_vue_map, key->inputs_read,
@ -1684,7 +1881,7 @@ iris_compile_tes(struct iris_screen *screen,
.source_hash = ish->source_hash,
},
.key = &brw_key,
.prog_data = tes_prog_data,
.prog_data = brw_prog_data,
.input_vue_map = &input_vue_map,
};
@ -1703,11 +1900,13 @@ iris_compile_tes(struct iris_screen *screen,
iris_debug_recompile(screen, dbg, ish, &brw_key.base);
iris_apply_brw_prog_data(shader, &brw_prog_data->base.base);
uint32_t *so_decls =
screen->vtbl.create_so_decl_list(&ish->stream_output,
&vue_prog_data->vue_map);
&iris_vue_data(shader)->vue_map);
iris_finalize_program(shader, prog_data, so_decls, system_values,
iris_finalize_program(shader, &brw_prog_data->base.base, so_decls, system_values,
num_system_values, 0, num_cbufs, &bt);
iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_TES,
@ -1758,7 +1957,7 @@ iris_update_compiled_tes(struct iris_context *ice)
shs->sysvals_need_upload = true;
unsigned urb_entry_size = shader ?
((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0;
iris_vue_data(shader)->urb_entry_size : 0;
check_urb_size(ice, urb_entry_size, MESA_SHADER_TESS_EVAL);
}
@ -1783,10 +1982,6 @@ iris_compile_gs(struct iris_screen *screen,
const struct brw_compiler *compiler = screen->compiler;
const struct intel_device_info *devinfo = screen->devinfo;
void *mem_ctx = ralloc_context(NULL);
struct brw_gs_prog_data *gs_prog_data =
rzalloc(mem_ctx, struct brw_gs_prog_data);
struct brw_vue_prog_data *vue_prog_data = &gs_prog_data->base;
struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
enum brw_param_builtin *system_values;
unsigned num_system_values;
unsigned num_cbufs;
@ -1811,10 +2006,13 @@ iris_compile_gs(struct iris_screen *screen,
iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
num_system_values, num_cbufs);
brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges);
struct brw_gs_prog_data *brw_prog_data =
rzalloc(mem_ctx, struct brw_gs_prog_data);
brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.base.ubo_ranges);
brw_compute_vue_map(devinfo,
&vue_prog_data->vue_map, nir->info.outputs_written,
&brw_prog_data->base.vue_map, nir->info.outputs_written,
nir->info.separate_shader, /* pos_slots */ 1);
struct brw_gs_prog_key brw_key = iris_to_brw_gs_key(screen, key);
@ -1827,7 +2025,7 @@ iris_compile_gs(struct iris_screen *screen,
.source_hash = ish->source_hash,
},
.key = &brw_key,
.prog_data = gs_prog_data,
.prog_data = brw_prog_data,
};
const unsigned *program = brw_compile_gs(compiler, &params);
@ -1845,11 +2043,13 @@ iris_compile_gs(struct iris_screen *screen,
iris_debug_recompile(screen, dbg, ish, &brw_key.base);
iris_apply_brw_prog_data(shader, &brw_prog_data->base.base);
uint32_t *so_decls =
screen->vtbl.create_so_decl_list(&ish->stream_output,
&vue_prog_data->vue_map);
&iris_vue_data(shader)->vue_map);
iris_finalize_program(shader, prog_data, so_decls, system_values,
iris_finalize_program(shader, &brw_prog_data->base.base, so_decls, system_values,
num_system_values, 0, num_cbufs, &bt);
iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_GS,
@ -1903,7 +2103,7 @@ iris_update_compiled_gs(struct iris_context *ice)
shs->sysvals_need_upload = true;
unsigned urb_entry_size = shader ?
((struct brw_vue_prog_data *) shader->prog_data)->urb_entry_size : 0;
iris_vue_data(shader)->urb_entry_size : 0;
check_urb_size(ice, urb_entry_size, MESA_SHADER_GEOMETRY);
}
}
@ -1921,9 +2121,6 @@ iris_compile_fs(struct iris_screen *screen,
{
const struct brw_compiler *compiler = screen->compiler;
void *mem_ctx = ralloc_context(NULL);
struct brw_wm_prog_data *fs_prog_data =
rzalloc(mem_ctx, struct brw_wm_prog_data);
struct brw_stage_prog_data *prog_data = &fs_prog_data->base;
enum brw_param_builtin *system_values;
const struct intel_device_info *devinfo = screen->devinfo;
unsigned num_system_values;
@ -1932,7 +2129,10 @@ iris_compile_fs(struct iris_screen *screen,
nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
const struct iris_fs_prog_key *const key = &shader->key.fs;
prog_data->use_alt_mode = nir->info.use_legacy_math_rules;
struct brw_wm_prog_data *brw_prog_data =
rzalloc(mem_ctx, struct brw_wm_prog_data);
brw_prog_data->base.use_alt_mode = nir->info.use_legacy_math_rules;
iris_setup_uniforms(devinfo, mem_ctx, nir, 0, &system_values,
&num_system_values, &num_cbufs);
@ -1955,7 +2155,7 @@ iris_compile_fs(struct iris_screen *screen,
MAX2(key->nr_color_regions, null_rts),
num_system_values, num_cbufs);
brw_nir_analyze_ubo_ranges(compiler, nir, prog_data->ubo_ranges);
brw_nir_analyze_ubo_ranges(compiler, nir, brw_prog_data->base.ubo_ranges);
struct brw_wm_prog_key brw_key = iris_to_brw_fs_key(screen, key);
@ -1967,7 +2167,7 @@ iris_compile_fs(struct iris_screen *screen,
.source_hash = ish->source_hash,
},
.key = &brw_key,
.prog_data = fs_prog_data,
.prog_data = brw_prog_data,
.allow_spilling = true,
.max_polygons = UCHAR_MAX,
@ -1989,7 +2189,9 @@ iris_compile_fs(struct iris_screen *screen,
iris_debug_recompile(screen, dbg, ish, &brw_key.base);
iris_finalize_program(shader, prog_data, NULL, system_values,
iris_apply_brw_prog_data(shader, &brw_prog_data->base);
iris_finalize_program(shader, &brw_prog_data->base, NULL, system_values,
num_system_values, 0, num_cbufs, &bt);
iris_upload_shader(screen, ish, shader, NULL, uploader, IRIS_CACHE_FS,
@ -2017,7 +2219,7 @@ iris_update_compiled_fs(struct iris_context *ice)
screen->vtbl.populate_fs_key(ice, &ish->nir->info, &key);
struct intel_vue_map *last_vue_map =
&brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map;
&iris_vue_data(ice->shaders.last_vue_shader)->vue_map;
if (ish->nos & (1ull << IRIS_NOS_LAST_VUE_MAP))
key.input_slots_valid = last_vue_map->slots_valid;
@ -2061,10 +2263,10 @@ static void
update_last_vue_map(struct iris_context *ice,
struct iris_compiled_shader *shader)
{
struct brw_vue_prog_data *vue_prog_data = (void *) shader->prog_data;
struct intel_vue_map *vue_map = &vue_prog_data->vue_map;
struct intel_vue_map *old_map = !ice->shaders.last_vue_shader ? NULL :
&brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map;
const struct intel_vue_map *vue_map = &iris_vue_data(shader)->vue_map;
const struct intel_vue_map *old_map =
!ice->shaders.last_vue_shader ? NULL :
&iris_vue_data(ice->shaders.last_vue_shader)->vue_map;
const uint64_t changed_slots =
(old_map ? old_map->slots_valid : 0ull) ^ vue_map->slots_valid;
@ -2092,7 +2294,7 @@ iris_update_pull_constant_descriptors(struct iris_context *ice,
{
struct iris_compiled_shader *shader = ice->shaders.prog[stage];
if (!shader || !shader->prog_data->has_ubo_pull)
if (!shader || !shader->has_ubo_pull)
return;
struct iris_shader_state *shs = &ice->state.shaders[stage];
@ -2163,12 +2365,12 @@ iris_update_compiled_shaders(struct iris_context *ice)
bool points_or_lines = false;
if (gs) {
const struct brw_gs_prog_data *gs_prog_data = (void *) gs->prog_data;
const struct iris_gs_data *gs_data = iris_gs_data_const(gs);
points_or_lines =
gs_prog_data->output_topology == _3DPRIM_POINTLIST ||
gs_prog_data->output_topology == _3DPRIM_LINESTRIP;
gs_data->output_topology == _3DPRIM_POINTLIST ||
gs_data->output_topology == _3DPRIM_LINESTRIP;
} else if (tes) {
const struct brw_tes_prog_data *tes_data = (void *) tes->prog_data;
const struct iris_tes_data *tes_data = iris_tes_data_const(tes);
points_or_lines =
tes_data->output_topology == INTEL_TESS_OUTPUT_TOPOLOGY_LINE ||
tes_data->output_topology == INTEL_TESS_OUTPUT_TOPOLOGY_POINT;
@ -2217,9 +2419,6 @@ iris_compile_cs(struct iris_screen *screen,
{
const struct brw_compiler *compiler = screen->compiler;
void *mem_ctx = ralloc_context(NULL);
struct brw_cs_prog_data *cs_prog_data =
rzalloc(mem_ctx, struct brw_cs_prog_data);
struct brw_stage_prog_data *prog_data = &cs_prog_data->base;
enum brw_param_builtin *system_values;
const struct intel_device_info *devinfo = screen->devinfo;
unsigned num_system_values;
@ -2228,7 +2427,8 @@ iris_compile_cs(struct iris_screen *screen,
nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
const struct iris_cs_prog_key *const key = &shader->key.cs;
NIR_PASS_V(nir, brw_nir_lower_cs_intrinsics, devinfo, cs_prog_data);
NIR_PASS_V(nir, brw_nir_lower_cs_intrinsics, devinfo,
brw_cs_prog_data(shader->brw_prog_data));
iris_setup_uniforms(devinfo, mem_ctx, nir, ish->kernel_input_size,
&system_values, &num_system_values, &num_cbufs);
@ -2239,6 +2439,9 @@ iris_compile_cs(struct iris_screen *screen,
struct brw_cs_prog_key brw_key = iris_to_brw_cs_key(screen, key);
struct brw_cs_prog_data *brw_prog_data =
rzalloc(mem_ctx, struct brw_cs_prog_data);
struct brw_compile_cs_params params = {
.base = {
.mem_ctx = mem_ctx,
@ -2247,7 +2450,7 @@ iris_compile_cs(struct iris_screen *screen,
.source_hash = ish->source_hash,
},
.key = &brw_key,
.prog_data = cs_prog_data,
.prog_data = brw_prog_data,
};
const unsigned *program = brw_compile_cs(compiler, &params);
@ -2264,7 +2467,9 @@ iris_compile_cs(struct iris_screen *screen,
iris_debug_recompile(screen, dbg, ish, &brw_key.base);
iris_finalize_program(shader, prog_data, NULL, system_values,
iris_apply_brw_prog_data(shader, &brw_prog_data->base);
iris_finalize_program(shader, &brw_prog_data->base, NULL, system_values,
num_system_values, ish->kernel_input_size, num_cbufs,
&bt);
@ -2322,14 +2527,16 @@ iris_update_compiled_compute_shader(struct iris_context *ice)
}
void
iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data,
iris_fill_cs_push_const_buffer(struct iris_screen *screen,
struct iris_compiled_shader *shader,
unsigned threads,
uint32_t *dst)
{
assert(brw_cs_push_const_total_size(cs_prog_data, threads) > 0);
assert(cs_prog_data->push.cross_thread.size == 0);
assert(cs_prog_data->push.per_thread.dwords == 1);
assert(cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID);
struct iris_cs_data *cs_data = iris_cs_data(shader);
assert(iris_cs_push_const_total_size(shader, threads) > 0);
assert(cs_data->push.cross_thread.size == 0);
assert(cs_data->push.per_thread.dwords == 1);
assert(cs_data->first_param_is_builtin_subgroup_id);
for (unsigned t = 0; t < threads; t++)
dst[8 * t] = t;
}
@ -2536,7 +2743,7 @@ iris_get_compute_state_info(struct pipe_context *ctx, void *state,
list_for_each_entry_safe(struct iris_compiled_shader, shader,
&ish->variants, link) {
info->private_memory = MAX2(info->private_memory,
shader->prog_data->total_scratch);
shader->total_scratch);
}
}
@ -2547,7 +2754,6 @@ iris_get_compute_state_subgroup_size(struct pipe_context *ctx, void *state,
struct iris_context *ice = (void *) ctx;
struct iris_screen *screen = (void *) ctx->screen;
struct u_upload_mgr *uploader = ice->shaders.uploader_driver;
const struct intel_device_info *devinfo = screen->devinfo;
struct iris_uncompiled_shader *ish = state;
struct iris_cs_prog_key key = { KEY_INIT(base) };
@ -2563,9 +2769,7 @@ iris_get_compute_state_subgroup_size(struct pipe_context *ctx, void *state,
iris_compile_cs(screen, uploader, &ice->dbg, ish, shader);
}
struct brw_cs_prog_data *cs_prog_data = (void *) shader->prog_data;
return brw_cs_get_dispatch_info(devinfo, cs_prog_data, block).simd_size;
return iris_get_cs_dispatch_info(screen->devinfo, shader, block).simd_size;
}
static void
@ -3041,3 +3245,30 @@ iris_init_program_functions(struct pipe_context *ctx)
ctx->get_compute_state_info = iris_get_compute_state_info;
ctx->get_compute_state_subgroup_size = iris_get_compute_state_subgroup_size;
}
struct intel_cs_dispatch_info
iris_get_cs_dispatch_info(const struct intel_device_info *devinfo,
const struct iris_compiled_shader *shader,
const uint32_t block[3])
{
struct brw_cs_prog_data *cs_prog_data =
brw_cs_prog_data(shader->brw_prog_data);
return brw_cs_get_dispatch_info(devinfo, cs_prog_data, block);
}
unsigned
iris_cs_push_const_total_size(const struct iris_compiled_shader *shader,
unsigned threads)
{
struct brw_cs_prog_data *cs_prog_data =
brw_cs_prog_data(shader->brw_prog_data);
return brw_cs_push_const_total_size(cs_prog_data, threads);
}
uint32_t
iris_fs_barycentric_modes(const struct iris_compiled_shader *shader,
enum intel_msaa_flags pushed_msaa_flags)
{
const struct brw_wm_prog_data *brw = brw_wm_prog_data(shader->brw_prog_data);
return wm_prog_data_barycentric_modes(brw, pushed_msaa_flags);
}

15
src/gallium/drivers/iris/iris_program_cache.c
View file

@ -160,15 +160,15 @@ iris_upload_shader(struct iris_screen *screen,
{
const struct intel_device_info *devinfo = screen->devinfo;
u_upload_alloc(uploader, 0, shader->prog_data->program_size, 64,
u_upload_alloc(uploader, 0, shader->program_size, 64,
&shader->assembly.offset, &shader->assembly.res,
&shader->map);
memcpy(shader->map, assembly, shader->prog_data->program_size);
memcpy(shader->map, assembly, shader->program_size);
struct iris_resource *res = (void *) shader->assembly.res;
uint64_t shader_data_addr = res->bo->address +
shader->assembly.offset +
shader->prog_data->const_data_offset;
shader->const_data_offset;
struct brw_shader_reloc_value reloc_values[] = {
{
@ -181,7 +181,7 @@ iris_upload_shader(struct iris_screen *screen,
},
};
brw_write_shader_relocs(&screen->compiler->isa, shader->map,
shader->prog_data, reloc_values,
shader->brw_prog_data, reloc_values,
ARRAY_SIZE(reloc_values));
/* Store the 3DSTATE shader packets and other derived state. */
@ -212,7 +212,7 @@ iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
*kernel_out =
iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
*((void **) prog_data_out) = shader->prog_data;
*((void **) prog_data_out) = shader->brw_prog_data;
iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
@ -242,6 +242,8 @@ iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
iris_create_shader_variant(screen, ice->shaders.cache, stage,
IRIS_CACHE_BLORP, key_size, key);
iris_apply_brw_prog_data(shader, prog_data);
iris_finalize_program(shader, prog_data, NULL, NULL, 0, 0, 0, &bt);
iris_upload_shader(screen, NULL, shader, ice->shaders.cache,
@ -251,7 +253,7 @@ iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
*kernel_out =
iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
*((void **) prog_data_out) = shader->prog_data;
*((void **) prog_data_out) = shader->brw_prog_data;
iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
@ -419,6 +421,7 @@ iris_ensure_indirect_generation_shader(struct iris_batch *batch)
MESA_SHADER_FRAGMENT,
IRIS_CACHE_BLORP,
sizeof(key), &key);
iris_apply_brw_prog_data(shader, &prog_data->base);
iris_finalize_program(shader, &prog_data->base, NULL, NULL, 0, 0, 0, &bt);
iris_upload_shader(screen, NULL, shader, ice->shaders.cache,

337
src/gallium/drivers/iris/iris_state.c
View file

@ -96,7 +96,6 @@
#include "util/u_trace_gallium.h"
#include "drm-uapi/i915_drm.h"
#include "nir.h"
#include "intel/compiler/brw_compiler.h"
#include "intel/common/intel_aux_map.h"
#include "intel/common/intel_l3_config.h"
#include "intel/common/intel_sample_positions.h"
@ -1930,8 +1929,8 @@ want_pma_fix(struct iris_context *ice)
{
UNUSED struct iris_screen *screen = (void *) ice->ctx.screen;
UNUSED const struct intel_device_info *devinfo = screen->devinfo;
const struct brw_wm_prog_data *wm_prog_data = (void *)
ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
const struct iris_fs_data *fs_data =
iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]);
const struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
const struct iris_depth_stencil_alpha_state *cso_zsa = ice->state.cso_zsa;
const struct iris_blend_state *cso_blend = ice->state.cso_blend;
@ -2008,7 +2007,7 @@ want_pma_fix(struct iris_context *ice)
return false;
/* 3DSTATE_WM::EDSC_Mode != EDSC_PREPS */
if (wm_prog_data->early_fragment_tests)
if (fs_data->early_fragment_tests)
return false;
/* 3DSTATE_WM::ForceKillPix != ForceOff &&
@ -2018,7 +2017,7 @@ want_pma_fix(struct iris_context *ice)
* 3DSTATE_PS_BLEND::AlphaTestEnable ||
* 3DSTATE_WM_CHROMAKEY::ChromaKeyKillEnable)
*/
bool killpixels = wm_prog_data->uses_kill || wm_prog_data->uses_omask ||
bool killpixels = fs_data->uses_kill || fs_data->uses_omask ||
cso_blend->alpha_to_coverage || cso_zsa->alpha_enabled;
/* The Gfx8 depth PMA equation becomes:
@ -2042,7 +2041,7 @@ want_pma_fix(struct iris_context *ice)
if (!cso_zsa->depth_test_enabled)
return false;
return wm_prog_data->computed_depth_mode != PSCDEPTH_OFF ||
return fs_data->computed_depth_mode != PSCDEPTH_OFF ||
(killpixels && (cso_zsa->depth_writes_enabled ||
(sres && cso_zsa->stencil_writes_enabled)));
}
@ -4626,15 +4625,15 @@ iris_emit_sbe_swiz(struct iris_batch *batch,
unsigned sprite_coord_enables)
{
struct GENX(SF_OUTPUT_ATTRIBUTE_DETAIL) attr_overrides[16] = {};
const struct brw_wm_prog_data *wm_prog_data = (void *)
ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
const struct iris_fs_data *fs_data =
iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]);
const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
/* XXX: this should be generated when putting programs in place */
for (uint8_t idx = 0; idx < wm_prog_data->urb_setup_attribs_count; idx++) {
const uint8_t fs_attr = wm_prog_data->urb_setup_attribs[idx];
const int input_index = wm_prog_data->urb_setup[fs_attr];
for (uint8_t idx = 0; idx < fs_data->urb_setup_attribs_count; idx++) {
const uint8_t fs_attr = fs_data->urb_setup_attribs[idx];
const int input_index = fs_data->urb_setup[fs_attr];
if (input_index < 0 || input_index >= 16)
continue;
@ -4719,12 +4718,12 @@ iris_is_drawing_points(const struct iris_context *ice)
}
if (ice->shaders.prog[MESA_SHADER_GEOMETRY]) {
const struct brw_gs_prog_data *gs_prog_data =
(void *) ice->shaders.prog[MESA_SHADER_GEOMETRY]->prog_data;
return gs_prog_data->output_topology == _3DPRIM_POINTLIST;
const struct iris_gs_data *gs_data =
iris_gs_data(ice->shaders.prog[MESA_SHADER_GEOMETRY]);
return gs_data->output_topology == _3DPRIM_POINTLIST;
} else if (ice->shaders.prog[MESA_SHADER_TESS_EVAL]) {
const struct brw_tes_prog_data *tes_data =
(void *) ice->shaders.prog[MESA_SHADER_TESS_EVAL]->prog_data;
const struct iris_tes_data *tes_data =
iris_tes_data(ice->shaders.prog[MESA_SHADER_TESS_EVAL]);
return tes_data->output_topology == INTEL_TESS_OUTPUT_TOPOLOGY_POINT;
} else {
return ice->state.prim_mode == MESA_PRIM_POINTS;
@ -4732,18 +4731,18 @@ iris_is_drawing_points(const struct iris_context *ice)
}
static unsigned
iris_calculate_point_sprite_overrides(const struct brw_wm_prog_data *prog_data,
iris_calculate_point_sprite_overrides(const struct iris_fs_data *fs_data,
const struct iris_rasterizer_state *cso)
{
unsigned overrides = 0;
if (prog_data->urb_setup[VARYING_SLOT_PNTC] != -1)
overrides |= 1 << prog_data->urb_setup[VARYING_SLOT_PNTC];
if (fs_data->urb_setup[VARYING_SLOT_PNTC] != -1)
overrides |= 1 << fs_data->urb_setup[VARYING_SLOT_PNTC];
for (int i = 0; i < 8; i++) {
if ((cso->sprite_coord_enable & (1 << i)) &&
prog_data->urb_setup[VARYING_SLOT_TEX0 + i] != -1)
overrides |= 1 << prog_data->urb_setup[VARYING_SLOT_TEX0 + i];
fs_data->urb_setup[VARYING_SLOT_TEX0 + i] != -1)
overrides |= 1 << fs_data->urb_setup[VARYING_SLOT_TEX0 + i];
}
return overrides;
@ -4753,30 +4752,30 @@ static void
iris_emit_sbe(struct iris_batch *batch, const struct iris_context *ice)
{
const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
const struct brw_wm_prog_data *wm_prog_data = (void *)
ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
const struct iris_fs_data *fs_data =
iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]);
const struct intel_vue_map *last_vue_map =
&brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map;
&iris_vue_data(ice->shaders.last_vue_shader)->vue_map;
unsigned urb_read_offset, urb_read_length;
iris_compute_sbe_urb_read_interval(wm_prog_data->inputs,
iris_compute_sbe_urb_read_interval(fs_data->inputs,
last_vue_map,
cso_rast->light_twoside,
&urb_read_offset, &urb_read_length);
unsigned sprite_coord_overrides =
iris_is_drawing_points(ice) ?
iris_calculate_point_sprite_overrides(wm_prog_data, cso_rast) : 0;
iris_calculate_point_sprite_overrides(fs_data, cso_rast) : 0;
iris_emit_cmd(batch, GENX(3DSTATE_SBE), sbe) {
sbe.AttributeSwizzleEnable = true;
sbe.NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs;
sbe.NumberofSFOutputAttributes = fs_data->num_varying_inputs;
sbe.PointSpriteTextureCoordinateOrigin = cso_rast->sprite_coord_mode;
sbe.VertexURBEntryReadOffset = urb_read_offset;
sbe.VertexURBEntryReadLength = urb_read_length;
sbe.ForceVertexURBEntryReadOffset = true;
sbe.ForceVertexURBEntryReadLength = true;
sbe.ConstantInterpolationEnable = wm_prog_data->flat_inputs;
sbe.ConstantInterpolationEnable = fs_data->flat_inputs;
sbe.PointSpriteTextureCoordinateEnable = sprite_coord_overrides;
#if GFX_VER >= 9
for (int i = 0; i < 32; i++) {
@ -4787,10 +4786,10 @@ iris_emit_sbe(struct iris_batch *batch, const struct iris_context *ice)
/* Ask the hardware to supply PrimitiveID if the fragment shader
* reads it but a previous stage didn't write one.
*/
if ((wm_prog_data->inputs & VARYING_BIT_PRIMITIVE_ID) &&
if ((fs_data->inputs & VARYING_BIT_PRIMITIVE_ID) &&
last_vue_map->varying_to_slot[VARYING_SLOT_PRIMITIVE_ID] == -1) {
sbe.PrimitiveIDOverrideAttributeSelect =
wm_prog_data->urb_setup[VARYING_SLOT_PRIMITIVE_ID];
fs_data->urb_setup[VARYING_SLOT_PRIMITIVE_ID];
sbe.PrimitiveIDOverrideComponentX = true;
sbe.PrimitiveIDOverrideComponentY = true;
sbe.PrimitiveIDOverrideComponentZ = true;
@ -4922,17 +4921,17 @@ encode_sampler_count(const struct iris_compiled_shader *shader)
pkt.KernelStartPointer = KSP(shader); \
pkt.BindingTableEntryCount = shader->bt.size_bytes / 4; \
pkt.SamplerCount = encode_sampler_count(shader); \
pkt.FloatingPointMode = prog_data->use_alt_mode; \
pkt.FloatingPointMode = shader->use_alt_mode; \
\
pkt.DispatchGRFStartRegisterForURBData = \
prog_data->dispatch_grf_start_reg; \
pkt.prefix##URBEntryReadLength = vue_prog_data->urb_read_length; \
shader->dispatch_grf_start_reg; \
pkt.prefix##URBEntryReadLength = vue_data->urb_read_length; \
pkt.prefix##URBEntryReadOffset = 0; \
\
pkt.StatisticsEnable = true; \
pkt.Enable = true; \
\
if (prog_data->total_scratch) { \
if (shader->total_scratch) { \
INIT_THREAD_SCRATCH_SIZE(pkt) \
}
@ -4948,7 +4947,7 @@ encode_sampler_count(const struct iris_compiled_shader *shader)
}
#else
#define INIT_THREAD_SCRATCH_SIZE(pkt) \
pkt.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11;
pkt.PerThreadScratchSpace = ffs(shader->total_scratch) - 11;
#define MERGE_SCRATCH_ADDR(name) \
{ \
uint32_t pkt2[GENX(name##_length)] = {0}; \
@ -4968,8 +4967,7 @@ static void
iris_store_vs_state(const struct intel_device_info *devinfo,
struct iris_compiled_shader *shader)
{
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
struct iris_vue_data *vue_data = iris_vue_data(shader);
iris_pack_command(GENX(3DSTATE_VS), shader->derived_data, vs) {
INIT_THREAD_DISPATCH_FIELDS(vs, Vertex, MESA_SHADER_VERTEX);
@ -4978,7 +4976,7 @@ iris_store_vs_state(const struct intel_device_info *devinfo,
vs.SIMD8DispatchEnable = true;
#endif
vs.UserClipDistanceCullTestEnableBitmask =
vue_prog_data->cull_distance_mask;
vue_data->cull_distance_mask;
}
}
@ -4989,9 +4987,8 @@ static void
iris_store_tcs_state(const struct intel_device_info *devinfo,
struct iris_compiled_shader *shader)
{
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
struct brw_tcs_prog_data *tcs_prog_data = (void *) prog_data;
struct iris_tcs_data *tcs_data = iris_tcs_data(shader);
struct iris_vue_data *vue_data = &tcs_data->base;
iris_pack_command(GENX(3DSTATE_HS), shader->derived_data, hs) {
INIT_THREAD_DISPATCH_FIELDS(hs, Vertex, MESA_SHADER_TESS_CTRL);
@ -5003,12 +5000,12 @@ iris_store_tcs_state(const struct intel_device_info *devinfo,
* the number of instance count. The number of max threads must be
* more than 2 times the number of instance count.
*/
assert((devinfo->max_tcs_threads / 2) > tcs_prog_data->instances);
hs.DispatchGRFStartRegisterForURBData = prog_data->dispatch_grf_start_reg & 0x1f;
hs.DispatchGRFStartRegisterForURBData5 = prog_data->dispatch_grf_start_reg >> 5;
assert((devinfo->max_tcs_threads / 2) > tcs_data->instances);
hs.DispatchGRFStartRegisterForURBData = shader->dispatch_grf_start_reg & 0x1f;
hs.DispatchGRFStartRegisterForURBData5 = shader->dispatch_grf_start_reg >> 5;
#endif
hs.InstanceCount = tcs_prog_data->instances - 1;
hs.InstanceCount = tcs_data->instances - 1;
hs.MaximumNumberofThreads = devinfo->max_tcs_threads - 1;
hs.IncludeVertexHandles = true;
@ -5016,14 +5013,14 @@ iris_store_tcs_state(const struct intel_device_info *devinfo,
/* Patch Count threshold specifies the maximum number of patches that
* will be accumulated before a thread dispatch is forced.
*/
hs.PatchCountThreshold = tcs_prog_data->patch_count_threshold;
hs.PatchCountThreshold = tcs_data->patch_count_threshold;
#endif
#if GFX_VER >= 9
#if GFX_VER < 20
hs.DispatchMode = vue_prog_data->dispatch_mode;
hs.DispatchMode = vue_data->dispatch_mode;
#endif
hs.IncludePrimitiveID = tcs_prog_data->include_primitive_id;
hs.IncludePrimitiveID = tcs_data->include_primitive_id;
#endif
}
}
@ -5035,9 +5032,8 @@ static void
iris_store_tes_state(const struct intel_device_info *devinfo,
struct iris_compiled_shader *shader)
{
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
struct brw_tes_prog_data *tes_prog_data = (void *) prog_data;
struct iris_tes_data *tes_data = iris_tes_data(shader);
struct iris_vue_data *vue_data = &tes_data->base;
uint32_t *ds_state = (void *) shader->derived_data;
uint32_t *te_state = ds_state + GENX(3DSTATE_DS_length);
@ -5048,19 +5044,19 @@ iris_store_tes_state(const struct intel_device_info *devinfo,
ds.DispatchMode = DISPATCH_MODE_SIMD8_SINGLE_PATCH;
ds.MaximumNumberofThreads = devinfo->max_tes_threads - 1;
ds.ComputeWCoordinateEnable =
tes_prog_data->domain == INTEL_TESS_DOMAIN_TRI;
tes_data->domain == INTEL_TESS_DOMAIN_TRI;
#if GFX_VER >= 12
ds.PrimitiveIDNotRequired = !tes_prog_data->include_primitive_id;
ds.PrimitiveIDNotRequired = !tes_data->include_primitive_id;
#endif
ds.UserClipDistanceCullTestEnableBitmask =
vue_prog_data->cull_distance_mask;
vue_data->cull_distance_mask;
}
iris_pack_command(GENX(3DSTATE_TE), te_state, te) {
te.Partitioning = tes_prog_data->partitioning;
te.OutputTopology = tes_prog_data->output_topology;
te.TEDomain = tes_prog_data->domain;
te.Partitioning = tes_data->partitioning;
te.OutputTopology = tes_data->output_topology;
te.TEDomain = tes_data->domain;
te.TEEnable = true;
te.MaximumTessellationFactorOdd = 63.0;
te.MaximumTessellationFactorNotOdd = 64.0;
@ -5096,42 +5092,38 @@ static void
iris_store_gs_state(const struct intel_device_info *devinfo,
struct iris_compiled_shader *shader)
{
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
struct brw_gs_prog_data *gs_prog_data = (void *) prog_data;
struct iris_gs_data *gs_data = iris_gs_data(shader);
struct iris_vue_data *vue_data = &gs_data->base;
iris_pack_command(GENX(3DSTATE_GS), shader->derived_data, gs) {
INIT_THREAD_DISPATCH_FIELDS(gs, Vertex, MESA_SHADER_GEOMETRY);
gs.OutputVertexSize = gs_prog_data->output_vertex_size_hwords * 2 - 1;
gs.OutputTopology = gs_prog_data->output_topology;
gs.ControlDataHeaderSize =
gs_prog_data->control_data_header_size_hwords;
gs.InstanceControl = gs_prog_data->invocations - 1;
gs.OutputVertexSize = gs_data->output_vertex_size_hwords * 2 - 1;
gs.OutputTopology = gs_data->output_topology;
gs.ControlDataHeaderSize = gs_data->control_data_header_size_hwords;
gs.InstanceControl = gs_data->invocations - 1;
#if GFX_VER < 20
gs.DispatchMode = DISPATCH_MODE_SIMD8;
#endif
gs.IncludePrimitiveID = gs_prog_data->include_primitive_id;
gs.ControlDataFormat = gs_prog_data->control_data_format;
gs.IncludePrimitiveID = gs_data->include_primitive_id;
gs.ControlDataFormat = gs_data->control_data_format;
gs.ReorderMode = TRAILING;
gs.ExpectedVertexCount = gs_prog_data->vertices_in;
gs.ExpectedVertexCount = gs_data->vertices_in;
gs.MaximumNumberofThreads =
GFX_VER == 8 ? (devinfo->max_gs_threads / 2 - 1)
: (devinfo->max_gs_threads - 1);
if (gs_prog_data->static_vertex_count != -1) {
if (gs_data->static_vertex_count != -1) {
gs.StaticOutput = true;
gs.StaticOutputVertexCount = gs_prog_data->static_vertex_count;
gs.StaticOutputVertexCount = gs_data->static_vertex_count;
}
gs.IncludeVertexHandles = vue_prog_data->include_vue_handles;
gs.IncludeVertexHandles = vue_data->include_vue_handles;
gs.UserClipDistanceCullTestEnableBitmask =
vue_prog_data->cull_distance_mask;
gs.UserClipDistanceCullTestEnableBitmask = vue_data->cull_distance_mask;
const int urb_entry_write_offset = 1;
const uint32_t urb_entry_output_length =
DIV_ROUND_UP(vue_prog_data->vue_map.num_slots, 2) -
urb_entry_write_offset;
DIV_ROUND_UP(vue_data->vue_map.num_slots, 2) - urb_entry_write_offset;
gs.VertexURBEntryOutputReadOffset = urb_entry_write_offset;
gs.VertexURBEntryOutputLength = MAX2(urb_entry_output_length, 1);
@ -5145,22 +5137,21 @@ static void
iris_store_fs_state(const struct intel_device_info *devinfo,
struct iris_compiled_shader *shader)
{
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_wm_prog_data *wm_prog_data = (void *) shader->prog_data;
struct iris_fs_data *fs_data = iris_fs_data(shader);
uint32_t *ps_state = (void *) shader->derived_data;
uint32_t *psx_state = ps_state + GENX(3DSTATE_PS_length);
iris_pack_command(GENX(3DSTATE_PS), ps_state, ps) {
ps.VectorMaskEnable = wm_prog_data->uses_vmask;
ps.VectorMaskEnable = fs_data->uses_vmask;
ps.BindingTableEntryCount = shader->bt.size_bytes / 4;
ps.SamplerCount = encode_sampler_count(shader);
ps.FloatingPointMode = prog_data->use_alt_mode;
ps.FloatingPointMode = shader->use_alt_mode;
ps.MaximumNumberofThreadsPerPSD =
devinfo->max_threads_per_psd - (GFX_VER == 8 ? 2 : 1);
#if GFX_VER < 20
ps.PushConstantEnable = prog_data->ubo_ranges[0].length > 0;
ps.PushConstantEnable = shader->ubo_ranges[0].length > 0;
#endif
/* From the documentation for this packet:
@ -5177,33 +5168,32 @@ iris_store_fs_state(const struct intel_device_info *devinfo,
* look useful at the moment. We might need this in future.
*/
ps.PositionXYOffsetSelect =
wm_prog_data->uses_pos_offset ? POSOFFSET_SAMPLE : POSOFFSET_NONE;
fs_data->uses_pos_offset ? POSOFFSET_SAMPLE : POSOFFSET_NONE;
if (prog_data->total_scratch) {
if (shader->total_scratch) {
INIT_THREAD_SCRATCH_SIZE(ps);
}
}
iris_pack_command(GENX(3DSTATE_PS_EXTRA), psx_state, psx) {
psx.PixelShaderValid = true;
psx.PixelShaderComputedDepthMode = wm_prog_data->computed_depth_mode;
psx.PixelShaderKillsPixel = wm_prog_data->uses_kill;
psx.PixelShaderComputedDepthMode = fs_data->computed_depth_mode;
psx.PixelShaderKillsPixel = fs_data->uses_kill;
#if GFX_VER < 20
psx.AttributeEnable = wm_prog_data->num_varying_inputs != 0;
psx.AttributeEnable = fs_data->num_varying_inputs != 0;
#endif
psx.PixelShaderUsesSourceDepth = wm_prog_data->uses_src_depth;
psx.PixelShaderUsesSourceW = wm_prog_data->uses_src_w;
psx.PixelShaderIsPerSample =
brw_wm_prog_data_is_persample(wm_prog_data, 0);
psx.oMaskPresenttoRenderTarget = wm_prog_data->uses_omask;
psx.PixelShaderUsesSourceDepth = fs_data->uses_src_depth;
psx.PixelShaderUsesSourceW = fs_data->uses_src_w;
psx.PixelShaderIsPerSample = fs_data->is_per_sample;
psx.oMaskPresenttoRenderTarget = fs_data->uses_omask;
#if GFX_VER >= 9
#if GFX_VER >= 20
assert(!wm_prog_data->pulls_bary);
assert(!fs_data->pulls_bary);
#else
psx.PixelShaderPullsBary = wm_prog_data->pulls_bary;
psx.PixelShaderPullsBary = fs_data->pulls_bary;
#endif
psx.PixelShaderComputesStencil = wm_prog_data->computed_stencil;
psx.PixelShaderComputesStencil = fs_data->computed_stencil;
#endif
}
}
@ -5217,20 +5207,20 @@ static void
iris_store_cs_state(const struct intel_device_info *devinfo,
struct iris_compiled_shader *shader)
{
struct brw_cs_prog_data *cs_prog_data = (void *) shader->prog_data;
struct iris_cs_data *cs_data = iris_cs_data(shader);
void *map = shader->derived_data;
iris_pack_state(GENX(INTERFACE_DESCRIPTOR_DATA), map, desc) {
#if GFX_VERx10 < 125
desc.ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs;
desc.ConstantURBEntryReadLength = cs_data->push.per_thread.regs;
desc.CrossThreadConstantDataReadLength =
cs_prog_data->push.cross_thread.regs;
cs_data->push.cross_thread.regs;
#else
assert(cs_prog_data->push.per_thread.regs == 0);
assert(cs_prog_data->push.cross_thread.regs == 0);
assert(cs_data->push.per_thread.regs == 0);
assert(cs_data->push.cross_thread.regs == 0);
#endif
#if GFX_VERx10 <= 125
desc.BarrierEnable = cs_prog_data->uses_barrier;
desc.BarrierEnable = cs_data->uses_barrier;
#endif
/* Typically set to 0 to avoid prefetching on every thread dispatch. */
desc.BindingTableEntryCount = devinfo->verx10 == 125 ?
@ -5597,7 +5587,6 @@ iris_populate_binding_table(struct iris_context *ice,
return;
struct iris_binding_table *bt = &shader->bt;
UNUSED struct brw_stage_prog_data *prog_data = shader->prog_data;
struct iris_shader_state *shs = &ice->state.shaders[stage];
uint32_t surf_base_offset = GFX_VER < 11 ? binder->bo->address : 0;
@ -5743,19 +5732,19 @@ pin_depth_and_stencil_buffers(struct iris_batch *batch,
static uint32_t
pin_scratch_space(struct iris_context *ice,
struct iris_batch *batch,
const struct brw_stage_prog_data *prog_data,
const struct iris_compiled_shader *shader,
gl_shader_stage stage)
{
uint32_t scratch_addr = 0;
if (prog_data->total_scratch > 0) {
if (shader->total_scratch > 0) {
struct iris_bo *scratch_bo =
iris_get_scratch_space(ice, prog_data->total_scratch, stage);
iris_get_scratch_space(ice, shader->total_scratch, stage);
iris_use_pinned_bo(batch, scratch_bo, true, IRIS_DOMAIN_NONE);
#if GFX_VERx10 >= 125
const struct iris_state_ref *ref =
iris_get_scratch_surf(ice, prog_data->total_scratch);
iris_get_scratch_surf(ice, shader->total_scratch);
iris_use_pinned_bo(batch, iris_resource_bo(ref->res),
false, IRIS_DOMAIN_NONE);
scratch_addr = ref->offset +
@ -5843,10 +5832,8 @@ iris_restore_render_saved_bos(struct iris_context *ice,
if (!shader)
continue;
struct brw_stage_prog_data *prog_data = (void *) shader->prog_data;
for (int i = 0; i < 4; i++) {
const struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
const struct iris_ubo_range *range = &shader->ubo_ranges[i];
if (range->length == 0)
continue;
@ -5890,7 +5877,7 @@ iris_restore_render_saved_bos(struct iris_context *ice,
struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
pin_scratch_space(ice, batch, shader->prog_data, stage);
pin_scratch_space(ice, batch, shader, stage);
}
}
}
@ -5963,7 +5950,7 @@ iris_restore_compute_saved_bos(struct iris_context *ice,
iris_use_pinned_bo(batch, curbe_bo, false, IRIS_DOMAIN_NONE);
}
pin_scratch_space(ice, batch, shader->prog_data, stage);
pin_scratch_space(ice, batch, shader, stage);
}
}
}
@ -6288,13 +6275,12 @@ setup_constant_buffers(struct iris_context *ice,
{
struct iris_shader_state *shs = &ice->state.shaders[stage];
struct iris_compiled_shader *shader = ice->shaders.prog[stage];
struct brw_stage_prog_data *prog_data = (void *) shader->prog_data;
uint32_t push_range_sum = 0;
int n = 0;
for (int i = 0; i < 4; i++) {
const struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
const struct iris_ubo_range *range = &shader->ubo_ranges[i];
if (range->length == 0)
continue;
@ -6564,7 +6550,7 @@ iris_preemption_streamout_wa(struct iris_context *ice,
static void
shader_program_needs_wa_14015055625(struct iris_context *ice,
struct iris_batch *batch,
const struct brw_stage_prog_data *prog_data,
struct iris_compiled_shader *shader,
gl_shader_stage stage,
bool *program_needs_wa_14015055625)
{
@ -6573,13 +6559,13 @@ shader_program_needs_wa_14015055625(struct iris_context *ice,
switch (stage) {
case MESA_SHADER_TESS_CTRL: {
struct brw_tcs_prog_data *tcs_prog_data = (void *) prog_data;
*program_needs_wa_14015055625 |= tcs_prog_data->include_primitive_id;
struct iris_tcs_data *tcs_data = iris_tcs_data(shader);
*program_needs_wa_14015055625 |= tcs_data->include_primitive_id;
break;
}
case MESA_SHADER_TESS_EVAL: {
struct brw_tes_prog_data *tes_prog_data = (void *) prog_data;
*program_needs_wa_14015055625 |= tes_prog_data->include_primitive_id;
struct iris_tes_data *tes_data = iris_tes_data(shader);
*program_needs_wa_14015055625 |= tes_data->include_primitive_id;
break;
}
default:
@ -6588,11 +6574,10 @@ shader_program_needs_wa_14015055625(struct iris_context *ice,
struct iris_compiled_shader *gs_shader =
ice->shaders.prog[MESA_SHADER_GEOMETRY];
const struct brw_gs_prog_data *gs_prog_data =
gs_shader ? (void *) gs_shader->prog_data : NULL;
const struct iris_gs_data *gs_data =
gs_shader ? iris_gs_data(gs_shader) : NULL;
*program_needs_wa_14015055625 |=
gs_prog_data && gs_prog_data->include_primitive_id;
*program_needs_wa_14015055625 |= gs_data && gs_data->include_primitive_id;
}
static void
@ -6700,8 +6685,8 @@ iris_upload_dirty_render_state(struct iris_context *ice,
struct iris_genx_state *genx = ice->state.genx;
struct iris_binder *binder = &ice->state.binder;
struct brw_wm_prog_data *wm_prog_data = (void *)
ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
struct iris_fs_data *fs_data =
iris_fs_data(ice->shaders.prog[MESA_SHADER_FRAGMENT]);
/* When MSAA is enabled, instead of using BLENDFACTOR_ZERO use
* CONST_COLOR, CONST_ALPHA and supply zero by using blend constants.
@ -6830,9 +6815,9 @@ iris_upload_dirty_render_state(struct iris_context *ice,
if (!ice->shaders.prog[i]) {
ice->shaders.urb.cfg.size[i] = 1;
} else {
struct brw_vue_prog_data *vue_prog_data =
(void *) ice->shaders.prog[i]->prog_data;
ice->shaders.urb.cfg.size[i] = vue_prog_data->urb_entry_size;
struct iris_vue_data *vue_data =
iris_vue_data(ice->shaders.prog[i]);
ice->shaders.urb.cfg.size[i] = vue_data->urb_entry_size;
}
assert(ice->shaders.urb.cfg.size[i] != 0);
}
@ -7135,8 +7120,8 @@ iris_upload_dirty_render_state(struct iris_context *ice,
#if INTEL_WA_14015055625_GFX_VER
/* Check if FS stage will use primitive ID overrides for Wa_14015055625. */
const struct intel_vue_map *last_vue_map =
&brw_vue_prog_data(ice->shaders.last_vue_shader->prog_data)->vue_map;
if ((wm_prog_data->inputs & VARYING_BIT_PRIMITIVE_ID) &&
&iris_vue_data(ice->shaders.last_vue_shader)->vue_map;
if ((fs_data->inputs & VARYING_BIT_PRIMITIVE_ID) &&
last_vue_map->varying_to_slot[VARYING_SLOT_PRIMITIVE_ID] == -1 &&
intel_needs_workaround(batch->screen->devinfo, 14015055625)) {
program_needs_wa_14015055625 = true;
@ -7150,15 +7135,14 @@ iris_upload_dirty_render_state(struct iris_context *ice,
struct iris_compiled_shader *shader = ice->shaders.prog[stage];
if (shader) {
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct iris_resource *cache = (void *) shader->assembly.res;
iris_use_pinned_bo(batch, cache->bo, false, IRIS_DOMAIN_NONE);
uint32_t scratch_addr =
pin_scratch_space(ice, batch, prog_data, stage);
pin_scratch_space(ice, batch, shader, stage);
#if INTEL_WA_14015055625_GFX_VER
shader_program_needs_wa_14015055625(ice, batch, prog_data, stage,
shader_program_needs_wa_14015055625(ice, batch, shader, stage,
&program_needs_wa_14015055625);
#endif
@ -7168,14 +7152,15 @@ iris_upload_dirty_render_state(struct iris_context *ice,
uint32_t ps_state[GENX(3DSTATE_PS_length)] = {0};
_iris_pack_command(batch, GENX(3DSTATE_PS), ps_state, ps) {
struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(shader->brw_prog_data);
intel_set_ps_dispatch_state(&ps, batch->screen->devinfo,
wm_prog_data, util_framebuffer_get_num_samples(cso_fb),
0 /* msaa_flags */);
#if GFX_VER == 12
assert(wm_prog_data->dispatch_multi == 0 ||
(wm_prog_data->dispatch_multi == 16 && wm_prog_data->max_polygons == 2));
ps.DualSIMD8DispatchEnable = wm_prog_data->dispatch_multi;
assert(fs_data->dispatch_multi == 0 ||
(fs_data->dispatch_multi == 16 && fs_data->max_polygons == 2));
ps.DualSIMD8DispatchEnable = fs_data->dispatch_multi;
/* XXX - No major improvement observed from enabling
* overlapping subspans, but it could be helpful
* in theory when the requirements listed on the
@ -7213,18 +7198,18 @@ iris_upload_dirty_render_state(struct iris_context *ice,
uint32_t psx_state[GENX(3DSTATE_PS_EXTRA_length)] = {0};
iris_pack_command(GENX(3DSTATE_PS_EXTRA), psx_state, psx) {
#if GFX_VER >= 9
if (!wm_prog_data->uses_sample_mask)
if (!fs_data->uses_sample_mask)
psx.InputCoverageMaskState = ICMS_NONE;
else if (wm_prog_data->post_depth_coverage)
else if (fs_data->post_depth_coverage)
psx.InputCoverageMaskState = ICMS_DEPTH_COVERAGE;
else if (wm_prog_data->inner_coverage &&
else if (fs_data->inner_coverage &&
cso->conservative_rasterization)
psx.InputCoverageMaskState = ICMS_INNER_CONSERVATIVE;
else
psx.InputCoverageMaskState = ICMS_NORMAL;
#else
psx.PixelShaderUsesInputCoverageMask =
wm_prog_data->uses_sample_mask;
fs_data->uses_sample_mask;
#endif
}
@ -7461,7 +7446,7 @@ iris_upload_dirty_render_state(struct iris_context *ice,
cl.PerspectiveDivideDisable = ice->state.window_space_position;
cl.ViewportXYClipTestEnable = !points_or_lines;
cl.NonPerspectiveBarycentricEnable = wm_prog_data->uses_nonperspective_interp_modes;
cl.NonPerspectiveBarycentricEnable = fs_data->uses_nonperspective_interp_modes;
cl.ForceZeroRTAIndexEnable = cso_fb->layers <= 1;
cl.MaximumVPIndex = ice->state.num_viewports - 1;
@ -7517,17 +7502,17 @@ iris_upload_dirty_render_state(struct iris_context *ice,
wm.StatisticsEnable = ice->state.statistics_counters_enabled;
wm.BarycentricInterpolationMode =
wm_prog_data_barycentric_modes(wm_prog_data, 0);
iris_fs_barycentric_modes(ice->shaders.prog[MESA_SHADER_FRAGMENT], 0);
if (wm_prog_data->early_fragment_tests)
if (fs_data->early_fragment_tests)
wm.EarlyDepthStencilControl = EDSC_PREPS;
else if (wm_prog_data->has_side_effects)
else if (fs_data->has_side_effects)
wm.EarlyDepthStencilControl = EDSC_PSEXEC;
else
wm.EarlyDepthStencilControl = EDSC_NORMAL;
/* We could skip this bit if color writes are enabled. */
if (wm_prog_data->has_side_effects || wm_prog_data->uses_kill)
if (fs_data->has_side_effects || fs_data->uses_kill)
wm.ForceThreadDispatchEnable = ForceON;
}
iris_emit_merge(batch, cso->wm, dynamic_wm, ARRAY_SIZE(cso->wm));
@ -7570,7 +7555,7 @@ iris_upload_dirty_render_state(struct iris_context *ice,
* undefined anyway, so simply disable blending to avoid the hang.
*/
pb.ColorBufferBlendEnable = (cso_blend->blend_enables & 1) &&
(!cso_blend->dual_color_blending || wm_prog_data->dual_src_blend);
(!cso_blend->dual_color_blending || fs_data->dual_src_blend);
}
iris_emit_merge(batch, cso_blend->ps_blend, dynamic_pb,
@ -7964,19 +7949,19 @@ iris_upload_dirty_render_state(struct iris_context *ice,
}
if (dirty & IRIS_DIRTY_VF_SGVS) {
const struct brw_vs_prog_data *vs_prog_data = (void *)
ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data;
const struct iris_vs_data *vs_data =
iris_vs_data(ice->shaders.prog[MESA_SHADER_VERTEX]);
struct iris_vertex_element_state *cso = ice->state.cso_vertex_elements;
iris_emit_cmd(batch, GENX(3DSTATE_VF_SGVS), sgv) {
if (vs_prog_data->uses_vertexid) {
if (vs_data->uses_vertexid) {
sgv.VertexIDEnable = true;
sgv.VertexIDComponentNumber = 2;
sgv.VertexIDElementOffset =
cso->count - ice->state.vs_needs_edge_flag;
}
if (vs_prog_data->uses_instanceid) {
if (vs_data->uses_instanceid) {
sgv.InstanceIDEnable = true;
sgv.InstanceIDComponentNumber = 3;
sgv.InstanceIDElementOffset =
@ -8750,13 +8735,10 @@ static void iris_emit_execute_indirect_dispatch(struct iris_context *ice,
const struct GENX(INTERFACE_DESCRIPTOR_DATA) idd)
{
const struct iris_screen *screen = batch->screen;
const struct intel_device_info *devinfo = screen->devinfo;
struct iris_compiled_shader *shader =
ice->shaders.prog[MESA_SHADER_COMPUTE];
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_cs_prog_data *cs_prog_data = (void *) prog_data;
const struct intel_cs_dispatch_info dispatch =
brw_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block);
iris_get_cs_dispatch_info(screen->devinfo, shader, grid->block);
struct iris_bo *indirect = iris_resource_bo(grid->indirect);
const int dispatch_size = dispatch.simd_size / 16;
@ -8794,10 +8776,9 @@ iris_upload_compute_walker(struct iris_context *ice,
struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_COMPUTE];
struct iris_compiled_shader *shader =
ice->shaders.prog[MESA_SHADER_COMPUTE];
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_cs_prog_data *cs_prog_data = (void *) prog_data;
const struct iris_cs_data *cs_data = iris_cs_data(shader);
const struct intel_cs_dispatch_info dispatch =
brw_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block);
iris_get_cs_dispatch_info(devinfo, shader, grid->block);
trace_intel_begin_compute(&batch->trace);
@ -8805,7 +8786,7 @@ iris_upload_compute_walker(struct iris_context *ice,
iris_emit_cmd(batch, GENX(CFE_STATE), cfe) {
cfe.MaximumNumberofThreads =
devinfo->max_cs_threads * devinfo->subslice_total;
uint32_t scratch_addr = pin_scratch_space(ice, batch, prog_data,
uint32_t scratch_addr = pin_scratch_space(ice, batch, shader,
MESA_SHADER_COMPUTE);
cfe.ScratchSpaceBuffer = scratch_addr >> 4;
}
@ -8815,7 +8796,7 @@ iris_upload_compute_walker(struct iris_context *ice,
idd.KernelStartPointer = KSP(shader);
idd.NumberofThreadsinGPGPUThreadGroup = dispatch.threads;
idd.SharedLocalMemorySize =
encode_slm_size(GFX_VER, prog_data->total_shared);
encode_slm_size(GFX_VER, shader->total_shared);
idd.SamplerStatePointer = shs->sampler_table.offset;
idd.SamplerCount = encode_sampler_count(shader),
idd.BindingTablePointer = binder->bt_offset[MESA_SHADER_COMPUTE];
@ -8823,7 +8804,7 @@ iris_upload_compute_walker(struct iris_context *ice,
idd.BindingTableEntryCount = devinfo->verx10 == 125 ?
0 : MIN2(shader->bt.size_bytes / 4, 31);
idd.PreferredSLMAllocationSize = preferred_slm_allocation_size(devinfo);
idd.NumberOfBarriers = cs_prog_data->uses_barrier;
idd.NumberOfBarriers = cs_data->uses_barrier;
iris_measure_snapshot(ice, batch, INTEL_SNAPSHOT_COMPUTE, NULL, NULL, NULL);
@ -8853,14 +8834,14 @@ iris_upload_compute_walker(struct iris_context *ice,
cw.InterfaceDescriptor = idd;
#if GFX_VERx10 >= 125
cw.GenerateLocalID = cs_prog_data->generate_local_id != 0;
cw.EmitLocal = cs_prog_data->generate_local_id;
cw.WalkOrder = cs_prog_data->walk_order;
cw.TileLayout = cs_prog_data->walk_order == INTEL_WALK_ORDER_YXZ ?
cw.GenerateLocalID = cs_data->generate_local_id != 0;
cw.EmitLocal = cs_data->generate_local_id;
cw.WalkOrder = cs_data->walk_order;
cw.TileLayout = cs_data->walk_order == INTEL_WALK_ORDER_YXZ ?
TileY32bpe : Linear;
#endif
assert(brw_cs_push_const_total_size(cs_prog_data, dispatch.threads) == 0);
assert(iris_cs_push_const_total_size(shader, dispatch.threads) == 0);
}
}
@ -8883,15 +8864,14 @@ iris_upload_gpgpu_walker(struct iris_context *ice,
ice->shaders.uncompiled[MESA_SHADER_COMPUTE];
struct iris_compiled_shader *shader =
ice->shaders.prog[MESA_SHADER_COMPUTE];
struct brw_stage_prog_data *prog_data = shader->prog_data;
struct brw_cs_prog_data *cs_prog_data = (void *) prog_data;
struct iris_cs_data *cs_data = iris_cs_data(shader);
const struct intel_cs_dispatch_info dispatch =
brw_cs_get_dispatch_info(devinfo, cs_prog_data, grid->block);
iris_get_cs_dispatch_info(screen->devinfo, shader, grid->block);
trace_intel_begin_compute(&batch->trace);
if ((stage_dirty & IRIS_STAGE_DIRTY_CS) ||
cs_prog_data->local_size[0] == 0 /* Variable local group size */) {
cs_data->local_size[0] == 0 /* Variable local group size */) {
/* The MEDIA_VFE_STATE documentation for Gfx8+ says:
*
* "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless
@ -8905,11 +8885,11 @@ iris_upload_gpgpu_walker(struct iris_context *ice,
PIPE_CONTROL_CS_STALL);
iris_emit_cmd(batch, GENX(MEDIA_VFE_STATE), vfe) {
if (prog_data->total_scratch) {
if (shader->total_scratch) {
uint32_t scratch_addr =
pin_scratch_space(ice, batch, prog_data, MESA_SHADER_COMPUTE);
pin_scratch_space(ice, batch, shader, MESA_SHADER_COMPUTE);
vfe.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11;
vfe.PerThreadScratchSpace = ffs(shader->total_scratch) - 11;
vfe.ScratchSpaceBasePointer =
rw_bo(NULL, scratch_addr, IRIS_DOMAIN_NONE);
}
@ -8927,20 +8907,20 @@ iris_upload_gpgpu_walker(struct iris_context *ice,
vfe.URBEntryAllocationSize = 2;
vfe.CURBEAllocationSize =
ALIGN(cs_prog_data->push.per_thread.regs * dispatch.threads +
cs_prog_data->push.cross_thread.regs, 2);
ALIGN(cs_data->push.per_thread.regs * dispatch.threads +
cs_data->push.cross_thread.regs, 2);
}
}
/* TODO: Combine subgroup-id with cbuf0 so we can push regular uniforms */
if ((stage_dirty & IRIS_STAGE_DIRTY_CS) ||
cs_prog_data->local_size[0] == 0 /* Variable local group size */) {
cs_data->local_size[0] == 0 /* Variable local group size */) {
uint32_t curbe_data_offset = 0;
assert(cs_prog_data->push.cross_thread.dwords == 0 &&
cs_prog_data->push.per_thread.dwords == 1 &&
cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID);
assert(cs_data->push.cross_thread.dwords == 0 &&
cs_data->push.per_thread.dwords == 1 &&
cs_data->first_param_is_builtin_subgroup_id);
const unsigned push_const_size =
brw_cs_push_const_total_size(cs_prog_data, dispatch.threads);
iris_cs_push_const_total_size(shader, dispatch.threads);
uint32_t *curbe_data_map =
stream_state(batch, ice->state.dynamic_uploader,
&ice->state.last_res.cs_thread_ids,
@ -8948,7 +8928,7 @@ iris_upload_gpgpu_walker(struct iris_context *ice,
&curbe_data_offset);
assert(curbe_data_map);
memset(curbe_data_map, 0x5a, ALIGN(push_const_size, 64));
iris_fill_cs_push_const_buffer(cs_prog_data, dispatch.threads,
iris_fill_cs_push_const_buffer(screen, shader, dispatch.threads,
curbe_data_map);
iris_emit_cmd(batch, GENX(MEDIA_CURBE_LOAD), curbe) {
@ -8976,8 +8956,7 @@ iris_upload_gpgpu_walker(struct iris_context *ice,
idd.SharedLocalMemorySize =
encode_slm_size(GFX_VER, ish->kernel_shared_size + grid->variable_shared_mem);
idd.KernelStartPointer =
KSP(shader) + brw_cs_prog_data_prog_offset(cs_prog_data,
dispatch.simd_size);
KSP(shader) + iris_cs_data_prog_offset(cs_data, dispatch.simd_size);
idd.SamplerStatePointer = shs->sampler_table.offset;
idd.BindingTablePointer =
binder->bt_offset[MESA_SHADER_COMPUTE] >> IRIS_BT_OFFSET_SHIFT;