diff --git a/src/imagination/.clang-format b/src/imagination/.clang-format index 97192ad5c81..ba2a4a65e81 100644 --- a/src/imagination/.clang-format +++ b/src/imagination/.clang-format @@ -201,6 +201,11 @@ ForEachMacros: [ 'foreach_instr', 'foreach_instr_safe', 'foreach_list_typed', + 'foreach_list_typed_from', + 'foreach_list_typed_from_reverse', + 'foreach_list_typed_reverse', + 'foreach_list_typed_reverse_safe', + 'foreach_list_typed_safe', 'hash_table_foreach', 'hash_table_u64_foreach', 'LIST_FOR_EACH_ENTRY', @@ -243,6 +248,7 @@ ForEachMacros: [ 'pco_foreach_cf_node_in_if_then', 'pco_foreach_cf_node_in_loop', 'pco_foreach_cf_node_in_loop_epilogue', + 'pco_foreach_cf_node_in_loop_interlogue', 'pco_foreach_cf_node_in_loop_prologue', 'pco_foreach_func_in_shader', 'pco_foreach_func_in_shader_rev', @@ -265,7 +271,6 @@ ForEachMacros: [ 'pco_foreach_instr_in_block_safe', 'pco_foreach_instr_in_block_safe_rev', 'pco_foreach_instr_in_func', - 'pco_foreach_instr_in_func_after', 'pco_foreach_instr_in_func_from', 'pco_foreach_instr_in_func_from_rev', 'pco_foreach_instr_in_func_rev', @@ -276,7 +281,6 @@ ForEachMacros: [ 'pco_foreach_instr_src', 'pco_foreach_instr_src_from', 'pco_foreach_instr_src_ssa', - 'pco_foreach_instr_src_ssa_after', 'pco_foreach_instr_src_ssa_from', 'pco_foreach_loop_in_func', 'pco_foreach_loop_in_func_from', diff --git a/src/imagination/pco/pco.c b/src/imagination/pco/pco.c index c542ea8cae5..2dc03cfb8f5 100644 --- a/src/imagination/pco/pco.c +++ b/src/imagination/pco/pco.c @@ -10,6 +10,7 @@ * \brief Main compiler interface. */ +#include "compiler/list.h" #include "compiler/glsl_types.h" #include "pco.h" #include "pco_internal.h" @@ -86,7 +87,7 @@ pco_shader *pco_shader_create(pco_ctx *ctx, nir_shader *nir, void *mem_ctx) shader->name = ralloc_strdup(shader, nir->info.name); shader->is_internal = nir->info.internal; shader->is_grouped = false; - list_inithead(&shader->funcs); + exec_list_make_empty(&shader->funcs); return shader; } @@ -124,12 +125,15 @@ pco_func *pco_func_create(pco_shader *shader, */ if (type == PCO_FUNC_TYPE_PREAMBLE) { assert(!preamble); - list_add(&func->link, &shader->funcs); + exec_list_push_head(&shader->funcs, &func->node); } else if (type == PCO_FUNC_TYPE_ENTRYPOINT) { assert(!pco_entrypoint(shader)); - list_add(&func->link, !preamble ? &shader->funcs : &preamble->link); + if (!preamble) + exec_list_push_head(&shader->funcs, &func->node); + else + exec_node_insert_after(&func->node, &preamble->node); } else { - list_addtail(&func->link, &shader->funcs); + exec_list_push_tail(&shader->funcs, &func->node); } init_cf_node(&func->cf_node, PCO_CF_NODE_TYPE_FUNC); @@ -137,7 +141,7 @@ pco_func *pco_func_create(pco_shader *shader, func->type = type; func->index = shader->next_func++; - list_inithead(&func->body); + exec_list_make_empty(&func->body); func->num_params = num_params; if (num_params) { @@ -164,7 +168,7 @@ pco_block *pco_block_create(pco_func *func) init_cf_node(&block->cf_node, PCO_CF_NODE_TYPE_BLOCK); block->parent_func = func; - list_inithead(&block->instrs); + exec_list_make_empty(&block->instrs); block->index = func->next_block++; return block; @@ -182,11 +186,11 @@ pco_if *pco_if_create(pco_func *func) init_cf_node(&pif->cf_node, PCO_CF_NODE_TYPE_IF); pif->parent_func = func; - list_inithead(&pif->prologue); - list_inithead(&pif->then_body); - list_inithead(&pif->interlogue); - list_inithead(&pif->else_body); - list_inithead(&pif->epilogue); + exec_list_make_empty(&pif->prologue); + exec_list_make_empty(&pif->then_body); + exec_list_make_empty(&pif->interlogue); + exec_list_make_empty(&pif->else_body); + exec_list_make_empty(&pif->epilogue); pif->index = func->next_if++; return pif; @@ -204,9 +208,10 @@ pco_loop *pco_loop_create(pco_func *func) init_cf_node(&loop->cf_node, PCO_CF_NODE_TYPE_LOOP); loop->parent_func = func; - list_inithead(&loop->prologue); - list_inithead(&loop->body); - list_inithead(&loop->epilogue); + exec_list_make_empty(&loop->prologue); + exec_list_make_empty(&loop->body); + exec_list_make_empty(&loop->interlogue); + exec_list_make_empty(&loop->epilogue); loop->index = func->next_loop++; return loop; @@ -273,7 +278,7 @@ pco_igrp *pco_igrp_create(pco_func *func) */ void pco_instr_delete(pco_instr *instr) { - list_del(&instr->link); + exec_node_remove(&instr->node); ralloc_free(instr); } diff --git a/src/imagination/pco/pco_builder.h b/src/imagination/pco/pco_builder.h index b70e455493f..d666ad4ca67 100644 --- a/src/imagination/pco/pco_builder.h +++ b/src/imagination/pco/pco_builder.h @@ -13,9 +13,9 @@ * \\brief PCO builder header. */ +#include "compiler/list.h" #include "pco.h" #include "pco_internal.h" -#include "util/list.h" #include "util/macros.h" #include @@ -374,22 +374,6 @@ static pco_builder pco_builder_create(pco_func *func, pco_cursor cursor) }; } -/** - * \brief Inserts a block at a position specified by the builder. - * - * \param[in] b The builder. - * \param[in] block The block. - */ -/* TODO: test with multiple blocks. */ -static inline void pco_builder_insert_block(pco_builder *b, pco_block *block) -{ - struct list_head *list = &pco_cursor_cf_node(b->cursor)->link; - bool before = pco_cursor_is_before(b->cursor); - - list_add(&block->cf_node.link, before ? list->prev : list); - b->cursor = pco_cursor_after_block(block); -} - /** * \brief Inserts a instruction at a position specified by the builder. * @@ -398,14 +382,24 @@ static inline void pco_builder_insert_block(pco_builder *b, pco_block *block) */ static inline void pco_builder_insert_instr(pco_builder *b, pco_instr *instr) { - pco_instr *cursor_instr = pco_cursor_instr(b->cursor); - bool before = pco_cursor_is_before(b->cursor); pco_block *block = pco_cursor_block(b->cursor); - struct list_head *list = cursor_instr ? &cursor_instr->link : &block->instrs; - instr->parent_block = block; - list_add(&instr->link, (before && cursor_instr) ? list->prev : list); + pco_instr *cursor_instr = pco_cursor_instr(b->cursor); + bool before = pco_cursor_is_before(b->cursor); + + if (cursor_instr) { + if (before) + exec_node_insert_node_before(&cursor_instr->node, &instr->node); + else + exec_node_insert_after(&cursor_instr->node, &instr->node); + } else { + if (before) + exec_list_push_head(&block->instrs, &instr->node); + else + exec_list_push_tail(&block->instrs, &instr->node); + } + b->cursor = pco_cursor_after_instr(instr); } @@ -417,14 +411,24 @@ static inline void pco_builder_insert_instr(pco_builder *b, pco_instr *instr) */ static inline void pco_builder_insert_igrp(pco_builder *b, pco_igrp *igrp) { - pco_igrp *cursor_igrp = pco_cursor_igrp(b->cursor); - bool before = pco_cursor_is_before(b->cursor); pco_block *block = pco_cursor_block(b->cursor); - struct list_head *list = cursor_igrp ? &cursor_igrp->link : &block->instrs; - igrp->parent_block = block; - list_add(&igrp->link, (before && cursor_igrp) ? list->prev : list); + pco_igrp *cursor_igrp = pco_cursor_igrp(b->cursor); + bool before = pco_cursor_is_before(b->cursor); + + if (cursor_igrp) { + if (before) + exec_node_insert_node_before(&cursor_igrp->node, &igrp->node); + else + exec_node_insert_after(&cursor_igrp->node, &igrp->node); + } else { + if (before) + exec_list_push_head(&block->instrs, &igrp->node); + else + exec_list_push_tail(&block->instrs, &igrp->node); + } + b->cursor = pco_cursor_after_igrp(igrp); } diff --git a/src/imagination/pco/pco_group_instrs.c b/src/imagination/pco/pco_group_instrs.c index 2f6b59fbedc..21d648f9a03 100644 --- a/src/imagination/pco/pco_group_instrs.c +++ b/src/imagination/pco/pco_group_instrs.c @@ -10,6 +10,7 @@ * \brief PCO instruction grouping pass. */ +#include "compiler/list.h" #include "hwdef/rogue_hw_defs.h" #include "pco.h" #include "pco_builder.h" @@ -155,8 +156,10 @@ static inline void calc_align_padding(pco_igrp *last_igrp, *offset_bytes += padding; /* Pad the size of the penultimate igrp. */ - pco_igrp *penultimate_igrp = - list_entry(last_igrp->link.prev, pco_igrp, link); + pco_igrp *penultimate_igrp = pco_prev_igrp(last_igrp); + + /* If we only have one igrp then its offset will be zero. */ + assert(penultimate_igrp); penultimate_igrp->enc.len.align_padding += padding; penultimate_igrp->enc.len.total += padding; diff --git a/src/imagination/pco/pco_internal.h b/src/imagination/pco/pco_internal.h index d71473b223e..21497d5dc30 100644 --- a/src/imagination/pco/pco_internal.h +++ b/src/imagination/pco/pco_internal.h @@ -13,6 +13,7 @@ * \brief PCO internal header. */ +#include "compiler/list.h" #include "compiler/spirv/nir_spirv.h" #include "hwdef/rogue_hw_utils.h" #include "pco.h" @@ -137,7 +138,7 @@ typedef struct _pco_phi_src { /** PCO instruction group. */ typedef struct _pco_igrp { - struct list_head link; /** Link in pco_block::instrs. */ + struct exec_node node; /** Node in pco_block::instrs. */ pco_block *parent_block; /** Basic block containing the igrp. */ pco_func *parent_func; /** Parent function. */ @@ -220,7 +221,7 @@ typedef struct _pco_igrp { typedef struct _pco_instr { union { struct { - struct list_head link; /** Link in pco_block::instrs. */ + struct exec_node node; /** Node in pco_block::instrs. */ pco_block *parent_block; /** Basic block containing the instruction. */ }; @@ -273,7 +274,7 @@ enum pco_cf_node_flag { /** PCO control-flow node. */ typedef struct _pco_cf_node { - struct list_head link; /** Link in lists of pco_cf_nodes. */ + struct exec_node node; /** Node in lists of pco_cf_nodes. */ enum pco_cf_node_type type; /** CF node type. */ struct _pco_cf_node *parent; /** Parent cf node. */ enum pco_cf_node_flag flag; /** Implementation-defined flag. */ @@ -283,7 +284,7 @@ typedef struct _pco_cf_node { typedef struct _pco_block { pco_cf_node cf_node; /** Control flow node. */ pco_func *parent_func; /** Parent function. */ - struct list_head instrs; /** Instruction/group list. */ + struct exec_list instrs; /** Instruction/group list. */ unsigned index; /** Block index. */ } pco_block; @@ -292,11 +293,11 @@ typedef struct _pco_if { pco_cf_node cf_node; /** CF node. */ pco_func *parent_func; /** Parent function. */ pco_ref cond; /** If condition. */ - struct list_head prologue; /** List of pco_cf_nodes for if prologue. */ - struct list_head then_body; /** List of pco_cf_nodes for if body. */ - struct list_head interlogue; /** List of pco_cf_nodes for if interlogue. */ - struct list_head else_body; /** List of pco_cf_nodes for else body. */ - struct list_head epilogue; /** List of pco_cf_nodes for if epilogue. */ + struct exec_list prologue; /** List of pco_cf_nodes for if prologue. */ + struct exec_list then_body; /** List of pco_cf_nodes for if body. */ + struct exec_list interlogue; /** List of pco_cf_nodes for if interlogue. */ + struct exec_list else_body; /** List of pco_cf_nodes for else body. */ + struct exec_list epilogue; /** List of pco_cf_nodes for if epilogue. */ unsigned index; /** If index. */ } pco_if; @@ -304,9 +305,10 @@ typedef struct _pco_if { typedef struct _pco_loop { pco_cf_node cf_node; /** CF node. */ pco_func *parent_func; /** Parent function. */ - struct list_head prologue; /** List of pco_cf_nodes for loop prologue. */ - struct list_head body; /** List of pco_cf_nodes for loop body. */ - struct list_head epilogue; /** List of pco_cf_nodes for loop epilogue. */ + struct exec_list prologue; /** List of pco_cf_nodes for loop prologue. */ + struct exec_list body; /** List of pco_cf_nodes for loop body. */ + struct exec_list interlogue; /** List of pco_cf_nodes for loop interlogue. */ + struct exec_list epilogue; /** List of pco_cf_nodes for loop epilogue. */ unsigned index; /** Loop index. */ } pco_loop; @@ -321,7 +323,7 @@ typedef struct _pco_vec_info { /** PCO function. */ typedef struct _pco_func { - struct list_head link; /** Link in pco_shader::funcs. */ + struct exec_node node; /** Node in pco_shader::funcs. */ pco_cf_node cf_node; /** Control flow node. */ pco_shader *parent_shader; /** Shader containing the function. */ @@ -330,7 +332,7 @@ typedef struct _pco_func { unsigned index; /** Function index. */ const char *name; /** Function name. */ - struct list_head body; /** List of pco_cf_nodes for function body. */ + struct exec_list body; /** List of pco_cf_nodes for function body. */ unsigned num_params; pco_ref *params; @@ -360,7 +362,7 @@ typedef struct _pco_shader { bool is_grouped; /** Whether the shader uses igrps. */ bool is_legalized; /** Whether the shader has been legalized. */ - struct list_head funcs; /** List of functions. */ + struct exec_list funcs; /** List of functions. */ unsigned next_func; /** Next function index. */ pco_data data; /** Shader data. */ @@ -435,7 +437,7 @@ void pco_instr_delete(pco_instr *instr); static inline out_type *name(const in_type *parent) \ { \ assert(parent && parent->type_field == type_value); \ - return list_entry(parent, out_type, field); \ + return exec_node_data(out_type, parent, field); \ } PCO_DEFINE_CAST(pco_cf_node_as_block, @@ -467,47 +469,52 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, /* Iterators. */ #define pco_foreach_func_in_shader(func, shader) \ - list_for_each_entry (pco_func, func, &(shader)->funcs, link) + foreach_list_typed (pco_func, func, node, &(shader)->funcs) #define pco_foreach_func_in_shader_rev(func, shader) \ - list_for_each_entry_rev (pco_func, func, &(shader)->funcs, link) + foreach_list_typed_reverse (pco_func, func, node, &(shader)->funcs) #define pco_foreach_cf_node_in_if_prologue(cf_node, _if) \ - list_for_each_entry (pco_cf_node, cf_node, &(_if)->prologue, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(_if)->prologue) #define pco_foreach_cf_node_in_if_then(cf_node, _if) \ - list_for_each_entry (pco_cf_node, cf_node, &(_if)->then_body, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(_if)->then_body) #define pco_foreach_cf_node_in_if_interlogue(cf_node, _if) \ - list_for_each_entry (pco_cf_node, cf_node, &(_if)->interlogue, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(_if)->interlogue) #define pco_foreach_cf_node_in_if_else(cf_node, _if) \ - list_for_each_entry (pco_cf_node, cf_node, &(_if)->else_body, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(_if)->else_body) #define pco_foreach_cf_node_in_if_epilogue(cf_node, _if) \ - list_for_each_entry (pco_cf_node, cf_node, &(_if)->epilogue, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(_if)->epilogue) #define pco_foreach_cf_node_in_loop_prologue(cf_node, loop) \ - list_for_each_entry (pco_cf_node, cf_node, &(loop)->prologue, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(loop)->prologue) #define pco_foreach_cf_node_in_loop(cf_node, loop) \ - list_for_each_entry (pco_cf_node, cf_node, &(loop)->body, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(loop)->body) + +#define pco_foreach_cf_node_in_loop_interlogue(cf_node, loop) \ + foreach_list_typed (pco_cf_node, cf_node, node, &(loop)->interlogue) #define pco_foreach_cf_node_in_loop_epilogue(cf_node, loop) \ - list_for_each_entry (pco_cf_node, cf_node, &(loop)->epilogue, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(loop)->epilogue) #define pco_foreach_cf_node_in_func(cf_node, func) \ - list_for_each_entry (pco_cf_node, cf_node, &(func)->body, link) + foreach_list_typed (pco_cf_node, cf_node, node, &(func)->body) #define pco_foreach_block_in_func(block, func) \ for (pco_block *block = pco_func_first_block(func); block != NULL; \ block = pco_next_block(block)) -#define pco_foreach_block_in_func_from(block, from) \ - for (pco_block *block = from; block != NULL; block = pco_next_block(block)) +#define pco_foreach_block_in_func_from(block, from) \ + for (pco_block *block = pco_next_block(from); block != NULL; \ + block = pco_next_block(block)) -#define pco_foreach_block_in_func_from_rev(block, from) \ - for (pco_block *block = from; block != NULL; block = pco_prev_block(block)) +#define pco_foreach_block_in_func_from_rev(block, from) \ + for (pco_block *block = pco_prev_block(from); block != NULL; \ + block = pco_prev_block(block)) #define pco_foreach_block_in_func_rev(block, func) \ for (pco_block *block = pco_func_last_block(func); block != NULL; \ @@ -528,10 +535,10 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, if (pco_ref_is_ssa(pif->cond) && pco_ref_get_bits(pif->cond) == 1) #define pco_foreach_if_in_func_from(pif, from) \ - for (pco_if *pif = from; pif != NULL; pif = pco_next_if(pif)) + for (pco_if *pif = pco_next_if(from); pif != NULL; pif = pco_next_if(pif)) #define pco_foreach_if_in_func_from_rev(pif, from) \ - for (pco_if *pif = from; pif != NULL; pif = pco_prev_if(pif)) + for (pco_if *pif = pco_prev_if(from); pif != NULL; pif = pco_prev_if(pif)) #define pco_foreach_if_in_func_rev(pif, func) \ for (pco_if *pif = pco_func_last_if(func); pif != NULL; \ @@ -541,46 +548,38 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, for (pco_loop *loop = pco_func_first_loop(func); loop != NULL; \ loop = pco_next_loop(loop)) -#define pco_foreach_loop_in_func_from(loop, from) \ - for (pco_loop *loop = from; loop != NULL; loop = pco_next_loop(loop)) +#define pco_foreach_loop_in_func_from(loop, from) \ + for (pco_loop *loop = pco_next_loop(from); loop != NULL; \ + loop = pco_next_loop(loop)) -#define pco_foreach_loop_in_func_from_rev(loop, from) \ - for (pco_loop *loop = from; loop != NULL; loop = pco_prev_loop(loop)) +#define pco_foreach_loop_in_func_from_rev(loop, from) \ + for (pco_loop *loop = pco_prev_loop(from); loop != NULL; \ + loop = pco_prev_loop(loop)) #define pco_foreach_loop_in_func_rev(loop, func) \ for (pco_loop *loop = pco_func_last_loop(func); loop != NULL; \ loop = pco_prev_loop(loop)) #define pco_foreach_instr_in_block(instr, block) \ - list_for_each_entry (pco_instr, instr, &(block)->instrs, link) + foreach_list_typed (pco_instr, instr, node, &(block)->instrs) -#define pco_foreach_instr_in_block_from(instr, block, from) \ - list_for_each_entry_from ( \ - pco_instr, \ - instr, \ - block == from->parent_block ? &from->link : (block)->instrs.next, \ - &(block)->instrs, \ - link) +#define pco_foreach_instr_in_block_from(instr, block, from) \ + foreach_list_typed_from (pco_instr, instr, node, _, &from->node) -#define pco_foreach_instr_in_block_from_rev(instr, block, from) \ - list_for_each_entry_from_rev ( \ - pco_instr, \ - instr, \ - block == from->parent_block ? &from->link : (block)->instrs.prev, \ - &(block)->instrs, \ - link) +#define pco_foreach_instr_in_block_from_rev(instr, block, from) \ + foreach_list_typed_from_reverse (pco_instr, instr, node, _, &from->node) #define pco_foreach_instr_in_block_safe(instr, block) \ - list_for_each_entry_safe (pco_instr, instr, &(block)->instrs, link) + foreach_list_typed_safe (pco_instr, instr, node, &(block)->instrs) #define pco_foreach_instr_in_block_rev(instr, block) \ - list_for_each_entry_rev (pco_instr, instr, &(block)->instrs, link) + foreach_list_typed_reverse (pco_instr, instr, node, &(block)->instrs) #define pco_foreach_instr_in_block_safe_rev(instr, block) \ - list_for_each_entry_safe_rev (pco_instr, instr, &(block)->instrs, link) + foreach_list_typed_reverse_safe (pco_instr, instr, node, &(block)->instrs) #define pco_foreach_igrp_in_block(igrp, block) \ - list_for_each_entry (pco_igrp, igrp, &(block)->instrs, link) + foreach_list_typed (pco_igrp, igrp, node, &(block)->instrs) #define pco_foreach_phi_src_in_instr(phi_src, instr) \ list_for_each_entry (pco_phi_src, phi_src, &(instr)->phi_srcs, link) @@ -589,17 +588,13 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, pco_foreach_block_in_func (block, func) \ pco_foreach_instr_in_block (instr, block) -#define pco_foreach_instr_in_func_from(instr, from) \ - pco_foreach_block_in_func_from (block, from->parent_block) \ - pco_foreach_instr_in_block_from (instr, block, from) +#define pco_foreach_instr_in_func_from(instr, from) \ + for (pco_instr *instr = pco_next_instr(from); instr != NULL; \ + instr = pco_next_instr(instr)) -#define pco_foreach_instr_in_func_after(instr, after) \ - if (pco_next_instr(after)) \ - pco_foreach_instr_in_func_from (instr, pco_next_instr(after)) - -#define pco_foreach_instr_in_func_from_rev(instr, from) \ - pco_foreach_block_in_func_from_rev (block, from->parent_block) \ - pco_foreach_instr_in_block_from_rev (instr, block, from) +#define pco_foreach_instr_in_func_from_rev(instr, from) \ + for (pco_instr *instr = pco_prev_instr(from); instr != NULL; \ + instr = pco_prev_instr(instr)) #define pco_foreach_instr_in_func_safe(instr, func) \ pco_foreach_block_in_func (block, func) \ @@ -615,7 +610,7 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, #define pco_foreach_igrp_in_func(igrp, func) \ pco_foreach_block_in_func (block, func) \ - list_for_each_entry (pco_igrp, igrp, &(block)->instrs, link) + pco_foreach_igrp_in_block (igrp, block) #define pco_foreach_instr_in_igrp(instr, igrp) \ for (pco_instr *instr = pco_igrp_first_instr(igrp); instr != NULL; \ @@ -630,8 +625,9 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, pdest < &instr->dest[instr->num_dests]; \ ++pdest) -#define pco_foreach_instr_dest_from(pdest, instr, pdest_from) \ - for (pco_ref *pdest = pdest_from; pdest < &instr->dest[instr->num_dests]; \ +#define pco_foreach_instr_dest_from(pdest, instr, pdest_from) \ + for (pco_ref *pdest = pdest_from + 1; \ + pdest < &instr->dest[instr->num_dests]; \ ++pdest) #define pco_foreach_instr_dest_ssa(pdest, instr) \ @@ -646,8 +642,9 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, for (pco_ref *psrc = &instr->src[0]; psrc < &instr->src[instr->num_srcs]; \ ++psrc) -#define pco_foreach_instr_src_from(psrc, instr, psrc_from) \ - for (pco_ref *psrc = psrc_from; psrc < &instr->src[instr->num_srcs]; ++psrc) +#define pco_foreach_instr_src_from(psrc, instr, psrc_from) \ + for (pco_ref *psrc = psrc_from + 1; psrc < &instr->src[instr->num_srcs]; \ + ++psrc) #define pco_foreach_instr_src_ssa(psrc, instr) \ pco_foreach_instr_src (psrc, instr) \ @@ -657,15 +654,12 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, pco_foreach_instr_src_from (psrc, instr, psrc_from) \ if (pco_ref_is_ssa(*psrc)) -#define pco_foreach_instr_src_ssa_after(psrc, instr, psrc_after) \ - pco_foreach_instr_src_ssa_from (psrc, instr, psrc_after + 1) - -#define pco_first_cf_node_list(body) list_first_entry(body, pco_cf_node, link) -#define pco_last_cf_node_list(body) list_last_entry(body, pco_cf_node, link) -#define pco_next_cf_node_list(cf_node) \ - list_entry((cf_node)->link.next, pco_cf_node, link) -#define pco_prev_cf_node_list(cf_node) \ - list_entry((cf_node)->link.prev, pco_cf_node, link) +#define pco_cf_node_head(list) exec_node_data_head(pco_cf_node, list, node) +#define pco_cf_node_tail(list) exec_node_data_tail(pco_cf_node, list, node) +#define pco_cf_node_next(cf_node) \ + exec_node_data_next(pco_cf_node, cf_node, node) +#define pco_cf_node_prev(cf_node) \ + exec_node_data_prev(pco_cf_node, cf_node, node) /** * \brief Returns the preamble function of a PCO shader. @@ -675,10 +669,10 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, */ static inline pco_func *pco_preamble(pco_shader *shader) { - if (list_is_empty(&shader->funcs)) + if (exec_list_is_empty(&shader->funcs)) return NULL; - pco_func *func = list_first_entry(&shader->funcs, pco_func, link); + pco_func *func = exec_node_data_head(pco_func, &shader->funcs, node); if (func->type == PCO_FUNC_TYPE_PREAMBLE) return func; @@ -693,15 +687,16 @@ static inline pco_func *pco_preamble(pco_shader *shader) */ static inline pco_func *pco_entrypoint(pco_shader *shader) { - if (list_is_empty(&shader->funcs)) + if (exec_list_is_empty(&shader->funcs)) return NULL; /* Entrypoint will either be the first or second function in the shader, * depending on whether or not there is a preamble. */ pco_func *preamble = pco_preamble(shader); - pco_func *func = !preamble ? list_first_entry(&shader->funcs, pco_func, link) - : list_entry(preamble->link.next, pco_func, link); + pco_func *func = !preamble + ? exec_node_data_head(pco_func, &shader->funcs, node) + : exec_node_data_next(pco_func, preamble, node); if (func->type == PCO_FUNC_TYPE_ENTRYPOINT) return func; @@ -747,20 +742,20 @@ static inline unsigned pco_igrp_variant(const pco_igrp *igrp, */ static inline pco_cf_node *pco_first_if_cf_node(pco_if *pif) { - if (!list_is_empty(&pif->prologue)) - return pco_first_cf_node_list(&pif->prologue); + if (!exec_list_is_empty(&pif->prologue)) + return pco_cf_node_head(&pif->prologue); - if (!list_is_empty(&pif->then_body)) - return pco_first_cf_node_list(&pif->then_body); + if (!exec_list_is_empty(&pif->then_body)) + return pco_cf_node_head(&pif->then_body); - if (!list_is_empty(&pif->interlogue)) - return pco_first_cf_node_list(&pif->interlogue); + if (!exec_list_is_empty(&pif->interlogue)) + return pco_cf_node_head(&pif->interlogue); - if (!list_is_empty(&pif->else_body)) - return pco_first_cf_node_list(&pif->else_body); + if (!exec_list_is_empty(&pif->else_body)) + return pco_cf_node_head(&pif->else_body); - if (!list_is_empty(&pif->epilogue)) - return pco_first_cf_node_list(&pif->epilogue); + if (!exec_list_is_empty(&pif->epilogue)) + return pco_cf_node_head(&pif->epilogue); UNREACHABLE("Empty if."); } @@ -773,20 +768,20 @@ static inline pco_cf_node *pco_first_if_cf_node(pco_if *pif) */ static inline pco_cf_node *pco_last_if_cf_node(pco_if *pif) { - if (!list_is_empty(&pif->epilogue)) - return pco_last_cf_node_list(&pif->epilogue); + if (!exec_list_is_empty(&pif->epilogue)) + return pco_cf_node_tail(&pif->epilogue); - if (!list_is_empty(&pif->else_body)) - return pco_last_cf_node_list(&pif->else_body); + if (!exec_list_is_empty(&pif->else_body)) + return pco_cf_node_tail(&pif->else_body); - if (!list_is_empty(&pif->interlogue)) - return pco_last_cf_node_list(&pif->interlogue); + if (!exec_list_is_empty(&pif->interlogue)) + return pco_cf_node_tail(&pif->interlogue); - if (!list_is_empty(&pif->then_body)) - return pco_last_cf_node_list(&pif->then_body); + if (!exec_list_is_empty(&pif->then_body)) + return pco_cf_node_tail(&pif->then_body); - if (!list_is_empty(&pif->prologue)) - return pco_last_cf_node_list(&pif->prologue); + if (!exec_list_is_empty(&pif->prologue)) + return pco_cf_node_tail(&pif->prologue); UNREACHABLE("Empty if."); } @@ -803,26 +798,26 @@ static inline pco_cf_node *pco_next_if_cf_node(pco_cf_node *cf_node) switch (cf_node->flag) { case PCO_CF_NODE_FLAG_PROLOGUE: - if (!list_is_empty(&pif->then_body)) - return pco_first_cf_node_list(&pif->then_body); + if (!exec_list_is_empty(&pif->then_body)) + return pco_cf_node_head(&pif->then_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_THEN: - if (!list_is_empty(&pif->interlogue)) - return pco_first_cf_node_list(&pif->interlogue); + if (!exec_list_is_empty(&pif->interlogue)) + return pco_cf_node_head(&pif->interlogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_INTERLOGUE: - if (!list_is_empty(&pif->else_body)) - return pco_first_cf_node_list(&pif->else_body); + if (!exec_list_is_empty(&pif->else_body)) + return pco_cf_node_head(&pif->else_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_ELSE: - if (!list_is_empty(&pif->epilogue)) - return pco_first_cf_node_list(&pif->epilogue); + if (!exec_list_is_empty(&pif->epilogue)) + return pco_cf_node_head(&pif->epilogue); FALLTHROUGH; @@ -848,26 +843,26 @@ static inline pco_cf_node *pco_prev_if_cf_node(pco_cf_node *cf_node) switch (cf_node->flag) { case PCO_CF_NODE_FLAG_EPILOGUE: - if (!list_is_empty(&pif->else_body)) - return pco_last_cf_node_list(&pif->else_body); + if (!exec_list_is_empty(&pif->else_body)) + return pco_cf_node_tail(&pif->else_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_ELSE: - if (!list_is_empty(&pif->interlogue)) - return pco_last_cf_node_list(&pif->interlogue); + if (!exec_list_is_empty(&pif->interlogue)) + return pco_cf_node_tail(&pif->interlogue); FALLTHROUGH; case PCO_CF_NODE_FLAG_INTERLOGUE: - if (!list_is_empty(&pif->then_body)) - return pco_last_cf_node_list(&pif->then_body); + if (!exec_list_is_empty(&pif->then_body)) + return pco_cf_node_tail(&pif->then_body); FALLTHROUGH; case PCO_CF_NODE_FLAG_IF_THEN: - if (!list_is_empty(&pif->prologue)) - return pco_last_cf_node_list(&pif->prologue); + if (!exec_list_is_empty(&pif->prologue)) + return pco_cf_node_tail(&pif->prologue); FALLTHROUGH; @@ -889,14 +884,17 @@ static inline pco_cf_node *pco_prev_if_cf_node(pco_cf_node *cf_node) */ static inline pco_cf_node *pco_first_loop_cf_node(pco_loop *loop) { - if (!list_is_empty(&loop->prologue)) - return pco_first_cf_node_list(&loop->prologue); + if (!exec_list_is_empty(&loop->prologue)) + return pco_cf_node_head(&loop->prologue); - if (!list_is_empty(&loop->body)) - return pco_first_cf_node_list(&loop->body); + if (!exec_list_is_empty(&loop->body)) + return pco_cf_node_head(&loop->body); - if (!list_is_empty(&loop->epilogue)) - return pco_first_cf_node_list(&loop->epilogue); + if (!exec_list_is_empty(&loop->interlogue)) + return pco_cf_node_head(&loop->interlogue); + + if (!exec_list_is_empty(&loop->epilogue)) + return pco_cf_node_head(&loop->epilogue); UNREACHABLE("Empty loop."); } @@ -909,14 +907,17 @@ static inline pco_cf_node *pco_first_loop_cf_node(pco_loop *loop) */ static inline pco_cf_node *pco_last_loop_cf_node(pco_loop *loop) { - if (!list_is_empty(&loop->epilogue)) - return pco_last_cf_node_list(&loop->epilogue); + if (!exec_list_is_empty(&loop->epilogue)) + return pco_cf_node_tail(&loop->epilogue); - if (!list_is_empty(&loop->body)) - return pco_last_cf_node_list(&loop->body); + if (!exec_list_is_empty(&loop->interlogue)) + return pco_cf_node_tail(&loop->interlogue); - if (!list_is_empty(&loop->prologue)) - return pco_last_cf_node_list(&loop->prologue); + if (!exec_list_is_empty(&loop->body)) + return pco_cf_node_tail(&loop->body); + + if (!exec_list_is_empty(&loop->prologue)) + return pco_cf_node_tail(&loop->prologue); UNREACHABLE("Empty loop."); } @@ -933,14 +934,20 @@ static inline pco_cf_node *pco_next_loop_cf_node(pco_cf_node *cf_node) switch (cf_node->flag) { case PCO_CF_NODE_FLAG_PROLOGUE: - if (!list_is_empty(&loop->body)) - return pco_first_cf_node_list(&loop->body); + if (!exec_list_is_empty(&loop->body)) + return pco_cf_node_head(&loop->body); FALLTHROUGH; case PCO_CF_NODE_FLAG_BODY: - if (!list_is_empty(&loop->epilogue)) - return pco_first_cf_node_list(&loop->epilogue); + if (!exec_list_is_empty(&loop->interlogue)) + return pco_cf_node_head(&loop->interlogue); + + FALLTHROUGH; + + case PCO_CF_NODE_FLAG_INTERLOGUE: + if (!exec_list_is_empty(&loop->epilogue)) + return pco_cf_node_head(&loop->epilogue); FALLTHROUGH; @@ -966,14 +973,20 @@ static inline pco_cf_node *pco_prev_loop_cf_node(pco_cf_node *cf_node) switch (cf_node->flag) { case PCO_CF_NODE_FLAG_EPILOGUE: - if (!list_is_empty(&loop->body)) - return pco_last_cf_node_list(&loop->body); + if (!exec_list_is_empty(&loop->interlogue)) + return pco_cf_node_tail(&loop->interlogue); + + FALLTHROUGH; + + case PCO_CF_NODE_FLAG_INTERLOGUE: + if (!exec_list_is_empty(&loop->body)) + return pco_cf_node_tail(&loop->body); FALLTHROUGH; case PCO_CF_NODE_FLAG_BODY: - if (!list_is_empty(&loop->prologue)) - return pco_last_cf_node_list(&loop->prologue); + if (!exec_list_is_empty(&loop->prologue)) + return pco_cf_node_tail(&loop->prologue); FALLTHROUGH; @@ -987,61 +1000,6 @@ static inline pco_cf_node *pco_prev_loop_cf_node(pco_cf_node *cf_node) UNREACHABLE(""); } -/** - * \brief Returns the first CF node from the body containing the current CF - * node. - * - * \param[in] cf_node The current CF node. - * \return The first CF node. - */ -static inline pco_cf_node *pco_first_cf_node(pco_cf_node *cf_node) -{ - pco_cf_node *parent_cf_node = cf_node->parent; - - switch (parent_cf_node->type) { - case PCO_CF_NODE_TYPE_IF: - return pco_first_if_cf_node(pco_cf_node_as_if(parent_cf_node)); - - case PCO_CF_NODE_TYPE_LOOP: - return pco_first_loop_cf_node(pco_cf_node_as_loop(parent_cf_node)); - - case PCO_CF_NODE_TYPE_FUNC: - return pco_first_cf_node_list(&pco_cf_node_as_func(parent_cf_node)->body); - - default: - break; - } - - UNREACHABLE(""); -} - -/** - * \brief Returns the last CF node from the body containing the current CF node. - * - * \param[in] cf_node The current CF node. - * \return The last CF node. - */ -static inline pco_cf_node *pco_last_cf_node(pco_cf_node *cf_node) -{ - pco_cf_node *parent_cf_node = cf_node->parent; - - switch (parent_cf_node->type) { - case PCO_CF_NODE_TYPE_IF: - return pco_last_if_cf_node(pco_cf_node_as_if(parent_cf_node)); - - case PCO_CF_NODE_TYPE_LOOP: - return pco_last_loop_cf_node(pco_cf_node_as_loop(parent_cf_node)); - - case PCO_CF_NODE_TYPE_FUNC: - return pco_last_cf_node_list(&pco_cf_node_as_func(parent_cf_node)->body); - - default: - break; - } - - UNREACHABLE(""); -} - /** * \brief Returns the next CF node. * @@ -1054,11 +1012,15 @@ static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node) return NULL; switch (cf_node->type) { - case PCO_CF_NODE_TYPE_BLOCK: - /* Not yet reached the end of the body, return the next cf node. */ - if (cf_node != pco_last_cf_node(cf_node)) - return pco_next_cf_node_list(cf_node); + case PCO_CF_NODE_TYPE_BLOCK: { + pco_cf_node *next_cf_node = pco_cf_node_next(cf_node); + + /* Not yet reached the end of the list, return the next cf node. */ + if (next_cf_node) + return next_cf_node; + break; + } case PCO_CF_NODE_TYPE_IF: return pco_first_if_cf_node(pco_cf_node_as_if(cf_node)); @@ -1075,14 +1037,18 @@ static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node) switch (parent_cf_node->type) { case PCO_CF_NODE_TYPE_IF: { pco_cf_node *next_cf_node = pco_next_if_cf_node(cf_node); - return next_cf_node ? next_cf_node - : pco_next_cf_node_list(parent_cf_node); + if (next_cf_node) + return next_cf_node; + + break; } case PCO_CF_NODE_TYPE_LOOP: { pco_cf_node *next_cf_node = pco_next_loop_cf_node(cf_node); - return next_cf_node ? next_cf_node - : pco_next_cf_node_list(parent_cf_node); + if (next_cf_node) + return next_cf_node; + + break; } /* End of the function; return NULL. */ @@ -1090,10 +1056,10 @@ static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node) return NULL; default: - break; + UNREACHABLE(""); } - UNREACHABLE(""); + return pco_cf_node_next(parent_cf_node); } /** @@ -1108,11 +1074,15 @@ static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node) return NULL; switch (cf_node->type) { - case PCO_CF_NODE_TYPE_BLOCK: - /* Not yet reached the start of the body, return the previous cf node. */ - if (cf_node != pco_first_cf_node(cf_node)) - return pco_prev_cf_node_list(cf_node); + case PCO_CF_NODE_TYPE_BLOCK: { + pco_cf_node *prev_cf_node = pco_cf_node_prev(cf_node); + + /* Not yet reached the start of the list, return the previous cf node. */ + if (prev_cf_node) + return prev_cf_node; + break; + } case PCO_CF_NODE_TYPE_IF: return pco_last_if_cf_node(pco_cf_node_as_if(cf_node)); @@ -1129,14 +1099,18 @@ static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node) switch (parent_cf_node->type) { case PCO_CF_NODE_TYPE_IF: { pco_cf_node *prev_cf_node = pco_prev_if_cf_node(cf_node); - return prev_cf_node ? prev_cf_node - : pco_prev_cf_node_list(parent_cf_node); + if (prev_cf_node) + return prev_cf_node; + + break; } case PCO_CF_NODE_TYPE_LOOP: { pco_cf_node *prev_cf_node = pco_prev_loop_cf_node(cf_node); - return prev_cf_node ? prev_cf_node - : pco_prev_cf_node_list(parent_cf_node); + if (prev_cf_node) + return prev_cf_node; + + break; } /* Start of the function; return NULL. */ @@ -1144,10 +1118,10 @@ static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node) return NULL; default: - break; + UNREACHABLE(""); } - UNREACHABLE(""); + return pco_cf_node_prev(parent_cf_node); } /** @@ -1177,9 +1151,9 @@ static inline pco_cf_node *pco_next_cf_node_type(pco_cf_node *cf_node, static inline pco_cf_node * pco_func_first_cf_node_type(pco_func *func, enum pco_cf_node_type type) { - assert(!list_is_empty(&func->body)); + assert(!exec_list_is_empty(&func->body)); - pco_cf_node *cf_node = pco_first_cf_node_list(&func->body); + pco_cf_node *cf_node = pco_cf_node_head(&func->body); if (cf_node->type == type) return cf_node; @@ -1213,9 +1187,9 @@ static inline pco_cf_node *pco_prev_cf_node_type(pco_cf_node *cf_node, static inline pco_cf_node * pco_func_last_cf_node_type(pco_func *func, enum pco_cf_node_type type) { - assert(!list_is_empty(&func->body)); + assert(!exec_list_is_empty(&func->body)); - pco_cf_node *cf_node = pco_last_cf_node_list(&func->body); + pco_cf_node *cf_node = pco_cf_node_tail(&func->body); if (cf_node->type == type) return cf_node; @@ -1228,26 +1202,26 @@ pco_func_last_cf_node_type(pco_func *func, enum pco_cf_node_type type) { \ pco_cf_node *cf_node = \ pco_next_cf_node_type(¤t_##type->cf_node, cf_node_type); \ - return cf_node == NULL ? NULL : pco_cf_node_as_##type(cf_node); \ + return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } \ \ static inline pco_##type *pco_func_first_##type(pco_func *func) \ { \ pco_cf_node *cf_node = pco_func_first_cf_node_type(func, cf_node_type); \ - return cf_node == NULL ? NULL : pco_cf_node_as_##type(cf_node); \ + return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } \ \ static inline pco_##type *pco_prev_##type(pco_##type *current_##type) \ { \ pco_cf_node *cf_node = \ pco_prev_cf_node_type(¤t_##type->cf_node, cf_node_type); \ - return cf_node == NULL ? NULL : pco_cf_node_as_##type(cf_node); \ + return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } \ \ static inline pco_##type *pco_func_last_##type(pco_func *func) \ { \ pco_cf_node *cf_node = pco_func_last_cf_node_type(func, cf_node_type); \ - return cf_node == NULL ? NULL : pco_cf_node_as_##type(cf_node); \ + return !cf_node ? NULL : pco_cf_node_as_##type(cf_node); \ } PCO_DEFINE_CF_ITER(block, PCO_CF_NODE_TYPE_BLOCK) @@ -1264,11 +1238,7 @@ PCO_DEFINE_CF_ITER(loop, PCO_CF_NODE_TYPE_LOOP) */ static inline pco_instr *pco_first_instr(pco_block *block) { - assert(!block->parent_func->parent_shader->is_grouped); - if (list_is_empty(&block->instrs)) - return NULL; - - return list_first_entry(&block->instrs, pco_instr, link); + return exec_node_data_head(pco_instr, &block->instrs, node); } /** @@ -1279,11 +1249,7 @@ static inline pco_instr *pco_first_instr(pco_block *block) */ static inline pco_instr *pco_last_instr(pco_block *block) { - assert(!block->parent_func->parent_shader->is_grouped); - if (list_is_empty(&block->instrs)) - return NULL; - - return list_last_entry(&block->instrs, pco_instr, link); + return exec_node_data_tail(pco_instr, &block->instrs, node); } /** @@ -1295,20 +1261,15 @@ static inline pco_instr *pco_last_instr(pco_block *block) */ static inline pco_instr *pco_next_instr(pco_instr *instr) { - assert(!instr->parent_func->parent_shader->is_grouped); if (!instr) return NULL; - if (instr != pco_last_instr(instr->parent_block)) - return list_entry(instr->link.next, pco_instr, link); + pco_instr *next_instr = exec_node_data_next(pco_instr, instr, node); + if (next_instr) + return next_instr; pco_block *block = pco_next_block(instr->parent_block); - - /* End of the function. */ - if (!block) - return NULL; - - return pco_first_instr(block); + return !block ? NULL : pco_first_instr(block); } /** @@ -1320,20 +1281,15 @@ static inline pco_instr *pco_next_instr(pco_instr *instr) */ static inline pco_instr *pco_prev_instr(pco_instr *instr) { - assert(!instr->parent_func->parent_shader->is_grouped); if (!instr) return NULL; - if (instr != pco_first_instr(instr->parent_block)) - return list_entry(instr->link.prev, pco_instr, link); + pco_instr *prev_instr = exec_node_data_prev(pco_instr, instr, node); + if (prev_instr) + return prev_instr; pco_block *block = pco_prev_block(instr->parent_block); - - /* Start of the function. */ - if (!block) - return NULL; - - return pco_last_instr(block); + return !block ? NULL : pco_last_instr(block); } /** @@ -1344,11 +1300,7 @@ static inline pco_instr *pco_prev_instr(pco_instr *instr) */ static inline pco_igrp *pco_first_igrp(pco_block *block) { - assert(block->parent_func->parent_shader->is_grouped); - if (list_is_empty(&block->instrs)) - return NULL; - - return list_first_entry(&block->instrs, pco_igrp, link); + return exec_node_data_head(pco_igrp, &block->instrs, node); } /** @@ -1359,11 +1311,7 @@ static inline pco_igrp *pco_first_igrp(pco_block *block) */ static inline pco_igrp *pco_last_igrp(pco_block *block) { - assert(block->parent_func->parent_shader->is_grouped); - if (list_is_empty(&block->instrs)) - return NULL; - - return list_last_entry(&block->instrs, pco_igrp, link); + return exec_node_data_tail(pco_igrp, &block->instrs, node); } /** @@ -1371,24 +1319,19 @@ static inline pco_igrp *pco_last_igrp(pco_block *block) * * \param[in] igrp The current instruction group. * \return The next instruction group, or NULL if the end of the function has - * been reached. + * been reached. */ static inline pco_igrp *pco_next_igrp(pco_igrp *igrp) { - assert(igrp->parent_func->parent_shader->is_grouped); if (!igrp) return NULL; - if (igrp != pco_last_igrp(igrp->parent_block)) - return list_entry(igrp->link.next, pco_igrp, link); + pco_igrp *next_igrp = exec_node_data_next(pco_igrp, igrp, node); + if (next_igrp) + return next_igrp; pco_block *block = pco_next_block(igrp->parent_block); - - /* End of the function. */ - if (!block) - return NULL; - - return pco_first_igrp(block); + return !block ? NULL : pco_first_igrp(block); } /** @@ -1396,24 +1339,19 @@ static inline pco_igrp *pco_next_igrp(pco_igrp *igrp) * * \param[in] igrp The current instruction group. * \return The previous instruction group, or NULL if the start of the function - * has been reached. + * has been reached. */ static inline pco_igrp *pco_prev_igrp(pco_igrp *igrp) { - assert(igrp->parent_func->parent_shader->is_grouped); if (!igrp) return NULL; - if (igrp == pco_first_igrp(igrp->parent_block)) - return list_entry(igrp->link.prev, pco_igrp, link); + pco_igrp *prev_igrp = exec_node_data_prev(pco_igrp, igrp, node); + if (prev_igrp) + return prev_igrp; pco_block *block = pco_prev_block(igrp->parent_block); - - /* Start of the function. */ - if (!block) - return NULL; - - return pco_last_igrp(block); + return !block ? NULL : pco_last_igrp(block); } /** @@ -2553,11 +2491,16 @@ static inline bool pco_igrp_dests_unset(pco_igrp *igrp) * \brief Iterates backwards to find the parent instruction of a source. * * \param[in] src The source whose parent is to be found. - * \param[in] from The instruction to start iterating back from. + * \param[in] from The instruction to start iterating back from (inclusive). * \return The parent instruction if found, else NULL. */ static inline pco_instr *find_parent_instr_from(pco_ref src, pco_instr *from) { + pco_foreach_instr_dest_ssa (pdest, from) { + if (pco_refs_are_equal(*pdest, src)) + return from; + } + pco_foreach_instr_in_func_from_rev (instr, from) { pco_foreach_instr_dest_ssa (pdest, instr) { if (pco_refs_are_equal(*pdest, src)) diff --git a/src/imagination/pco/pco_map.py b/src/imagination/pco/pco_map.py index ec324e0ea41..2eff31319a6 100644 --- a/src/imagination/pco/pco_map.py +++ b/src/imagination/pco/pco_map.py @@ -541,7 +541,7 @@ def group_map(op, hdr, enc_ops, srcs=[], iss=[], dests=[]): assert _enc_op.bname in encode_maps.keys() assert phase in [phase_map for phase_map, *_ in encode_maps[_enc_op.bname].op_ref_maps] - op_mappings.append(f'list_del(&{{1}}->link);') + op_mappings.append(f'exec_node_remove(&{{1}}->node);') op_mappings.append(f'{{}}->instrs[{phase}] = {{}};') op_mappings.append(f'{{0}}->instrs[{phase}]->phase = {phase};') op_mappings.append(f'{{0}}->instrs[{phase}]->parent_igrp = {{0}};') diff --git a/src/imagination/pco/pco_opt.c b/src/imagination/pco/pco_opt.c index 54f11db1ec6..267947e8b74 100644 --- a/src/imagination/pco/pco_opt.c +++ b/src/imagination/pco/pco_opt.c @@ -61,7 +61,7 @@ static inline bool pco_opt_prep_mods(pco_shader *shader, mod->op != PCO_OP_FLR) continue; - pco_foreach_instr_in_func_after (instr, mod) { + pco_foreach_instr_in_func_from (instr, mod) { if (instr->op != PCO_OP_FADD && instr->op != PCO_OP_FMUL) continue; @@ -436,7 +436,7 @@ static inline bool try_prop_hw_comp(pco_ref src, pco_ref repl, pco_instr *from) { bool progress = false; - pco_foreach_instr_in_func_after (instr, from) { + pco_foreach_instr_in_func_from (instr, from) { pco_foreach_instr_src_ssa (psrc, instr) { if (psrc->val != src.val) continue; diff --git a/src/imagination/pco/pco_print.c b/src/imagination/pco/pco_print.c index ce40a3cb4db..f647238742c 100644 --- a/src/imagination/pco/pco_print.c +++ b/src/imagination/pco/pco_print.c @@ -448,11 +448,11 @@ static void _pco_print_instr(pco_print_state *state, pco_instr *instr) bool printed = false; /* Destinations. */ - for (unsigned d = 0; d < instr->num_dests; ++d) { + pco_foreach_instr_dest (pdest, instr) { if (printed) pco_printf(state, ","); pco_printf(state, " "); - _pco_print_ref(state, instr->dest[d]); + _pco_print_ref(state, *pdest); printed = true; } @@ -492,28 +492,39 @@ static void _pco_print_instr(pco_print_state *state, pco_instr *instr) } /* Sources. */ - for (unsigned s = 0; s < instr->num_srcs; ++s) { + pco_foreach_instr_src (psrc, instr) { if (printed) pco_printf(state, ","); pco_printf(state, " "); - _pco_print_ref(state, instr->src[s]); + _pco_print_ref(state, *psrc); printed = true; } pco_printf(state, ";"); /* Spec for destinations. */ - if (state->verbose && !state->is_grouped && instr->num_dests) { + bool no_dest_specs = true; + pco_foreach_instr_dest (pdest, instr) { + if (!pco_ref_is_null(*pdest)) { + no_dest_specs = false; + break; + } + } + + if (state->verbose && !state->is_grouped && !no_dest_specs) { pco_printf(state, " /*"); printed = false; - for (unsigned d = 0; d < instr->num_dests; ++d) { + pco_foreach_instr_dest (pdest, instr) { + if (pco_ref_is_null(*pdest)) + continue; + if (printed) pco_printf(state, ","); pco_printf(state, " "); - _pco_print_ref(state, instr->dest[d]); + _pco_print_ref(state, *pdest); pco_printf(state, ":"); - pco_print_ref_spec(state, instr->dest[d]); + pco_print_ref_spec(state, *pdest); printed = true; } @@ -895,11 +906,16 @@ static void pco_print_if(pco_print_state *state, pco_if *pif) { pco_printfi(state, "if "); pco_print_if_name(state, pif); - pco_printf(state, " ("); - _pco_print_ref(state, pif->cond); - pco_printf(state, ") {\n"); - if (!list_is_empty(&pif->prologue)) { + if (!pco_ref_is_null(pif->cond)) { + pco_printf(state, " ("); + _pco_print_ref(state, pif->cond); + pco_printf(state, ")"); + } + + pco_printf(state, " {\n"); + + if (!exec_list_is_empty(&pif->prologue)) { pco_printfi(state, "/* prologue start */\n"); ++state->indent; @@ -911,7 +927,7 @@ static void pco_print_if(pco_print_state *state, pco_if *pif) pco_printfi(state, "/* prologue end */\n"); } - if (!list_is_empty(&pif->then_body)) { + if (!exec_list_is_empty(&pif->then_body)) { ++state->indent; pco_foreach_cf_node_in_if_then (cf_node, pif) { _pco_print_cf_node(state, cf_node); @@ -919,10 +935,10 @@ static void pco_print_if(pco_print_state *state, pco_if *pif) --state->indent; } - if (!list_is_empty(&pif->else_body)) { + if (!exec_list_is_empty(&pif->else_body)) { pco_printfi(state, "} else {\n"); - if (!list_is_empty(&pif->interlogue)) { + if (!exec_list_is_empty(&pif->interlogue)) { pco_printfi(state, "/* interlogue start */\n"); ++state->indent; @@ -940,10 +956,10 @@ static void pco_print_if(pco_print_state *state, pco_if *pif) } --state->indent; } else { - assert(list_is_empty(&pif->interlogue)); + assert(exec_list_is_empty(&pif->interlogue)); } - if (!list_is_empty(&pif->epilogue)) { + if (!exec_list_is_empty(&pif->epilogue)) { pco_printfi(state, "/* epilogue start */\n"); ++state->indent; @@ -981,7 +997,7 @@ static void pco_print_loop(pco_print_state *state, pco_loop *loop) pco_print_loop_name(state, loop); pco_printf(state, " {\n"); - if (!list_is_empty(&loop->prologue)) { + if (!exec_list_is_empty(&loop->prologue)) { pco_printfi(state, "/* prologue start */\n"); ++state->indent; @@ -993,7 +1009,7 @@ static void pco_print_loop(pco_print_state *state, pco_loop *loop) pco_printfi(state, "/* prologue end */\n"); } - if (!list_is_empty(&loop->body)) { + if (!exec_list_is_empty(&loop->body)) { ++state->indent; pco_foreach_cf_node_in_loop (cf_node, loop) { _pco_print_cf_node(state, cf_node); @@ -1001,7 +1017,23 @@ static void pco_print_loop(pco_print_state *state, pco_loop *loop) --state->indent; } - if (!list_is_empty(&loop->epilogue)) { + if (!exec_list_is_empty(&loop->interlogue)) { + pco_printfi(state, "/* loop "); + pco_print_loop_name(state, loop); + pco_printf(state, " interlogue start */\n"); + + ++state->indent; + pco_foreach_cf_node_in_loop_interlogue (cf_node, loop) { + _pco_print_cf_node(state, cf_node); + } + --state->indent; + + pco_printfi(state, "/* loop "); + pco_print_loop_name(state, loop); + pco_printf(state, " interlogue end */\n"); + } + + if (!exec_list_is_empty(&loop->epilogue)) { pco_printfi(state, "/* epilogue start */\n"); ++state->indent; diff --git a/src/imagination/pco/pco_ra.c b/src/imagination/pco/pco_ra.c index 01efd6b364e..320e55562f8 100644 --- a/src/imagination/pco/pco_ra.c +++ b/src/imagination/pco/pco_ra.c @@ -49,7 +49,7 @@ static bool vec_has_repeated_ssas(pco_instr *vec) assert(vec->op == PCO_OP_VEC); pco_foreach_instr_src_ssa (psrc, vec) { - pco_foreach_instr_src_ssa_after (psrc_inner, vec, psrc) { + pco_foreach_instr_src_ssa_from (psrc_inner, vec, psrc) { if (psrc_inner->val == psrc->val) return true; } diff --git a/src/imagination/pco/pco_trans_nir.c b/src/imagination/pco/pco_trans_nir.c index a537dbd0fc8..f30486eb522 100644 --- a/src/imagination/pco/pco_trans_nir.c +++ b/src/imagination/pco/pco_trans_nir.c @@ -17,7 +17,6 @@ #include "pco_builder.h" #include "pco_internal.h" #include "util/bitset.h" -#include "util/list.h" #include "util/macros.h" #include "util/ralloc.h" @@ -39,7 +38,7 @@ typedef struct _trans_ctx { /* Forward declarations. */ static pco_block *trans_cf_nodes(trans_ctx *tctx, pco_cf_node *parent_cf_node, - struct list_head *cf_node_list, + struct exec_list *cf_node_list, struct exec_list *nir_cf_node_list); /** @@ -1344,12 +1343,20 @@ static pco_instr *trans_instr(trans_ctx *tctx, nir_instr *ninstr) * \brief Translates a NIR block into PCO. * * \param[in] tctx Translation context. + * \param[in] parent_cf_node The parent cf node. + * \param[in] cf_node_list The PCO cf node list. * \param[in] nblock The nir block. * \return The PCO block. */ -static pco_block *trans_block(trans_ctx *tctx, nir_block *nblock) +static pco_block *trans_block(trans_ctx *tctx, + pco_cf_node *parent_cf_node, + struct exec_list *cf_node_list, + nir_block *nblock) { pco_block *block = pco_block_create(tctx->func); + block->cf_node.parent = parent_cf_node; + exec_list_push_tail(cf_node_list, &block->cf_node.node); + tctx->b = pco_builder_create(tctx->func, pco_cursor_after_block(block)); nir_foreach_instr (ninstr, nblock) { @@ -1363,32 +1370,40 @@ static pco_block *trans_block(trans_ctx *tctx, nir_block *nblock) * \brief Translates a NIR if into PCO. * * \param[in] tctx Translation context. + * \param[in] parent_cf_node The parent cf node. + * \param[in] cf_node_list The PCO cf node list. * \param[in] nif The nir if. - * \return The PCO if. */ -static pco_if *trans_if(trans_ctx *tctx, nir_if *nif) +static void trans_if(trans_ctx *tctx, + pco_cf_node *parent_cf_node, + struct exec_list *cf_node_list, + nir_if *nif) { pco_if *pif = pco_if_create(tctx->func); UNREACHABLE("finishme: trans_if"); - - return pif; + pif->cf_node.parent = parent_cf_node; + exec_list_push_tail(cf_node_list, &pif->cf_node.node); } /** * \brief Translates a NIR loop into PCO. * * \param[in] tctx Translation context. + * \param[in] parent_cf_node The parent cf node. + * \param[in] cf_node_list The PCO cf node list. * \param[in] nloop The nir loop. - * \return The PCO loop. */ -static pco_loop *trans_loop(trans_ctx *tctx, nir_loop *nloop) +static void trans_loop(trans_ctx *tctx, + pco_cf_node *parent_cf_node, + struct exec_list *cf_node_list, + nir_loop *nloop) { pco_loop *loop = pco_loop_create(tctx->func); + loop->cf_node.parent = parent_cf_node; + exec_list_push_tail(cf_node_list, &loop->cf_node.node); UNREACHABLE("finishme: trans_loop"); - - return loop; } /** @@ -1443,41 +1458,42 @@ static pco_func *trans_func(trans_ctx *tctx, nir_function_impl *impl) */ static pco_block *trans_cf_nodes(trans_ctx *tctx, pco_cf_node *parent_cf_node, - struct list_head *cf_node_list, + struct exec_list *cf_node_list, struct exec_list *nir_cf_node_list) { pco_block *start_block = NULL; - pco_cf_node *cf_node; foreach_list_typed (nir_cf_node, ncf_node, node, nir_cf_node_list) { switch (ncf_node->type) { case nir_cf_node_block: { - pco_block *block = trans_block(tctx, nir_cf_node_as_block(ncf_node)); - cf_node = &block->cf_node; - + pco_block *block = trans_block(tctx, + parent_cf_node, + cf_node_list, + nir_cf_node_as_block(ncf_node)); if (!start_block) start_block = block; break; } case nir_cf_node_if: { - pco_if *pif = trans_if(tctx, nir_cf_node_as_if(ncf_node)); - cf_node = &pif->cf_node; + trans_if(tctx, + parent_cf_node, + cf_node_list, + nir_cf_node_as_if(ncf_node)); break; } case nir_cf_node_loop: { - pco_loop *loop = trans_loop(tctx, nir_cf_node_as_loop(ncf_node)); - cf_node = &loop->cf_node; + trans_loop(tctx, + parent_cf_node, + cf_node_list, + nir_cf_node_as_loop(ncf_node)); break; } default: UNREACHABLE(""); } - - cf_node->parent = parent_cf_node; - list_addtail(&cf_node->link, cf_node_list); } return start_block;