From 6036f4900cd68a0fcd952aa5ea7668744d8e021c Mon Sep 17 00:00:00 2001 From: Simon Perretta Date: Fri, 24 Jan 2025 17:56:54 +0000 Subject: [PATCH] pco: add support for loops and ifs using predicated execution Signed-off-by: Simon Perretta Acked-by: Erik Faye-Lund Part-of: --- docs/envvars.rst | 3 + src/imagination/.clang-format | 1 + src/imagination/pco/pco_builder.h | 25 +- src/imagination/pco/pco_cf.c | 388 ++++++++++++++++++++++++---- src/imagination/pco/pco_debug.c | 1 + src/imagination/pco/pco_end.c | 2 +- src/imagination/pco/pco_internal.h | 70 ++++- src/imagination/pco/pco_ir.c | 3 + src/imagination/pco/pco_legalize.c | 6 +- src/imagination/pco/pco_map.py | 85 ++++++ src/imagination/pco/pco_ops.py | 7 +- src/imagination/pco/pco_ra.c | 24 ++ src/imagination/pco/pco_trans_nir.c | 36 ++- 13 files changed, 597 insertions(+), 54 deletions(-) diff --git a/docs/envvars.rst b/docs/envvars.rst index 1ecac8ed2c5..5d8873d4897 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -2080,6 +2080,9 @@ PowerVR driver environment variables ``reindex`` Reindex IR at the end of each pass. + ``no_pred_cf`` + No predicated execution in CF. + .. envvar:: PCO_SKIP_PASSES A comma-separated list of passes to skip. diff --git a/src/imagination/.clang-format b/src/imagination/.clang-format index abb996fd48c..25469e41331 100644 --- a/src/imagination/.clang-format +++ b/src/imagination/.clang-format @@ -244,6 +244,7 @@ ForEachMacros: [ 'pco_foreach_block_in_func_from_rev', 'pco_foreach_block_in_func_rev', 'pco_foreach_cf_node_in_func', + 'pco_foreach_cf_node_in_func_structured', 'pco_foreach_cf_node_in_if_else', 'pco_foreach_cf_node_in_if_epilogue', 'pco_foreach_cf_node_in_if_interlogue', diff --git a/src/imagination/pco/pco_builder.h b/src/imagination/pco/pco_builder.h index d666ad4ca67..6339fde44b7 100644 --- a/src/imagination/pco/pco_builder.h +++ b/src/imagination/pco/pco_builder.h @@ -99,6 +99,29 @@ static inline pco_cursor pco_cursor_after_block(pco_block *block) return pco_cursor_after_cf_node(&block->cf_node); } +/** + * \brief Returns a cursor set to before an exec_list. + * + * \param[in] exec_list The exec_list. + * \return The cursor. + */ +static inline pco_cursor +pco_cursor_before_exec_list(struct exec_list *exec_list) +{ + return pco_cursor_before_cf_node(pco_cf_node_head(exec_list)); +} + +/** + * \brief Returns a cursor set to after an exec_list. + * + * \param[in] block The exec_list. + * \return The exec_list. + */ +static inline pco_cursor pco_cursor_after_exec_list(struct exec_list *exec_list) +{ + return pco_cursor_after_cf_node(pco_cf_node_tail(exec_list)); +} + /** * \brief Returns a cursor set to before an instruction. * @@ -441,7 +464,7 @@ static inline void pco_builder_insert_igrp(pco_builder *b, pco_igrp *igrp) * \param[in] instr The instruction. * \return True if the instruction has the default execution condition. */ -static inline bool pco_instr_default_exec(pco_instr *instr) +static inline bool pco_instr_has_default_exec(pco_instr *instr) { if (!pco_instr_has_exec_cnd(instr)) return true; diff --git a/src/imagination/pco/pco_cf.c b/src/imagination/pco/pco_cf.c index a22b981a297..d30ac378475 100644 --- a/src/imagination/pco/pco_cf.c +++ b/src/imagination/pco/pco_cf.c @@ -40,18 +40,6 @@ static pco_ref emc_ref(pco_func *func, pco_builder *b) return func->emc; } -static inline bool can_pred_exec(pco_if *pif) -{ - /* TODO */ - return false; -} - -static inline void -lower_if_pred_exec(pco_if *pif, bool has_else, bool invert_cond) -{ - /* TODO */ -} - static inline pco_block *cf_section_create(pco_func *func, pco_cf_node *parent_cf_node, struct exec_list *cf_node_list, @@ -69,10 +57,103 @@ static inline pco_block *cf_section_create(pco_func *func, return block; } -static inline void lower_if(pco_if *pif, bool has_else, bool invert_cond) +static inline bool body_has_non_preds(struct exec_list *body) { - pco_func *func = pif->parent_func; + if (exec_list_is_empty(body)) + return false; + pco_block *block = pco_cf_node_as_block(pco_cf_node_head(body)); + pco_instr *last_instr = NULL; + pco_foreach_instr_in_block (instr, block) { + /* Make sure there are no instructions that read/write predicates. */ + pco_foreach_instr_src (psrc, instr) { + if (pco_ref_is_pred(*psrc)) + return true; + } + + pco_foreach_instr_dest (pdest, instr) { + if (pco_ref_is_pred(*pdest)) + return true; + } + + if (!pco_instr_has_exec_cnd(instr)) + return true; + + if (!pco_instr_has_default_exec(instr)) + return true; + + last_instr = instr; + } + + assert(last_instr); + return last_instr->op == PCO_OP_BR; +} + +static inline bool can_pred_exec(pco_if *pif) +{ + if (PCO_DEBUG(NO_PRED_CF)) + return false; + + /* Skip if there is any nesting. */ + if (exec_list_length(&pif->then_body) > 1 || + exec_list_length(&pif->else_body) > 1) { + return false; + } + + /* Skip if then/else blocks end with a branch, or contain non-predicatable + * instructions. + * Note: WDFs can't be predicated and won't be inserted until the scheduling + * pass which comes after this one, but we don't have to worry about it as if + * there are no outstanding data fences it'll simply NOP. + */ + if (body_has_non_preds(&pif->then_body) || + body_has_non_preds(&pif->else_body)) + return false; + + return true; +} + +static inline void set_body_exec_cnd(struct exec_list *body, + enum pco_exec_cnd exec_cnd) +{ + assert(!exec_list_is_empty(body)); + + pco_block *block = pco_cf_node_as_block(pco_cf_node_head(body)); + pco_foreach_instr_in_block (instr, block) { + pco_instr_set_exec_cnd(instr, exec_cnd); + } +} + +static inline void +lower_if_pred_exec(pco_if *pif, pco_func *func, bool has_else, bool invert_cond) +{ + pco_block *prologue = cf_section_create(func, + &pif->cf_node, + &pif->prologue, + PCO_CF_NODE_FLAG_PROLOGUE); + + /* Setup the prologue. */ + pco_builder b = pco_builder_create(func, pco_cursor_after_block(prologue)); + + /* TODO: see if the cond producer can set p0 directly. */ + pco_tstz(&b, + pco_ref_null(), + pco_ref_pred(PCO_PRED_P0), + pif->cond, + .tst_type_main = PCO_TST_TYPE_MAIN_U32); + + set_body_exec_cnd(&pif->then_body, + invert_cond ? PCO_EXEC_CND_E1_Z1 : PCO_EXEC_CND_E1_Z0); + + if (has_else) { + set_body_exec_cnd(&pif->else_body, + invert_cond ? PCO_EXEC_CND_E1_Z0 : PCO_EXEC_CND_E1_Z1); + } +} + +static inline void +lower_if_cond_exec(pco_if *pif, pco_func *func, bool has_else, bool invert_cond) +{ pco_block *prologue = cf_section_create(func, &pif->cf_node, &pif->prologue, @@ -137,57 +218,276 @@ static inline void lower_if(pco_if *pif, bool has_else, bool invert_cond) emc, pco_ref_imm8(1), .exec_cnd = PCO_EXEC_CND_EX_ZX); +} + +static inline void lower_if(pco_if *pif, pco_func *func) +{ + assert(!pco_ref_is_null(pif->cond)); + assert(exec_list_is_empty(&pif->prologue)); + assert(exec_list_is_empty(&pif->interlogue)); + assert(exec_list_is_empty(&pif->epilogue)); + + bool has_then = !exec_list_is_empty(&pif->then_body); + bool has_else = !exec_list_is_empty(&pif->else_body); + assert(has_then || has_else); + + /* If we only have an else body, invert the condition and bodies. */ + bool invert_cond = false; + if (!has_then && has_else) { + struct exec_list temp; + memcpy(&temp, &pif->then_body, sizeof(pif->then_body)); + memcpy(&pif->then_body, &pif->else_body, sizeof(pif->else_body)); + memcpy(&pif->else_body, &pif->then_body, sizeof(pif->then_body)); + invert_cond = true; + + has_then = true; + has_else = false; + } + + assert(has_then); + + if (pif->pred_exec) + lower_if_pred_exec(pif, func, has_else, invert_cond); + else + lower_if_cond_exec(pif, func, has_else, invert_cond); pif->cond = pco_ref_null(); } -static inline bool lower_ifs(pco_func *func) +static inline void lower_loop(pco_loop *loop, pco_func *func) { - bool progress = false; + assert(exec_list_is_empty(&loop->prologue)); + assert(exec_list_is_empty(&loop->interlogue)); + assert(exec_list_is_empty(&loop->epilogue)); - pco_foreach_if_in_func (pif, func) { - assert(!pco_ref_is_null(pif->cond)); + pco_block *prologue = cf_section_create(func, + &loop->cf_node, + &loop->prologue, + PCO_CF_NODE_FLAG_PROLOGUE); - assert(exec_list_is_empty(&pif->prologue)); - assert(exec_list_is_empty(&pif->interlogue)); - assert(exec_list_is_empty(&pif->epilogue)); + pco_block *interlogue = cf_section_create(func, + &loop->cf_node, + &loop->interlogue, + PCO_CF_NODE_FLAG_INTERLOGUE); - bool has_then = !exec_list_is_empty(&pif->then_body); - bool has_else = !exec_list_is_empty(&pif->else_body); - assert(has_then || has_else); + pco_block *epilogue = cf_section_create(func, + &loop->cf_node, + &loop->epilogue, + PCO_CF_NODE_FLAG_EPILOGUE); - /* If we only have an else body, invert the condition and bodies. */ - bool invert_cond = false; - if (!has_then && has_else) { - struct exec_list temp; - memcpy(&temp, &pif->then_body, sizeof(pif->then_body)); - memcpy(&pif->then_body, &pif->else_body, sizeof(pif->else_body)); - memcpy(&pif->else_body, &pif->then_body, sizeof(pif->then_body)); - invert_cond = true; + /* Setup the prologue. */ + pco_builder b = pco_builder_create(func, pco_cursor_after_block(prologue)); + pco_ref emc = emc_ref(func, &b); - has_then = true; - has_else = false; - } + pco_cndst(&b, + pco_ref_pred(PCO_PRED_PE), + emc, + emc, + pco_ref_imm8(2), /* TODO: make this a define */ + .exec_cnd = PCO_EXEC_CND_EX_ZX, + .cnd = PCO_CND_ALWAYS); - assert(has_then); + pco_br(&b, &epilogue->cf_node, .branch_cnd = PCO_BRANCH_CND_ALLINST); - if (can_pred_exec(pif)) - lower_if_pred_exec(pif, has_else, invert_cond); - else - lower_if(pif, has_else, invert_cond); + /* Setup the interlogue. */ + b.cursor = pco_cursor_after_block(interlogue); - progress = true; + pco_cndend(&b, + pco_ref_pred(PCO_PRED_PE), + emc, + emc, + pco_ref_imm8(1), /* TODO: make this a define */ + .exec_cnd = PCO_EXEC_CND_EX_ZX); + + pco_cndst(&b, + pco_ref_pred(PCO_PRED_PE), + emc, + emc, + pco_ref_imm8(1), /* TODO: make this a define */ + .exec_cnd = PCO_EXEC_CND_EX_ZX, + .cnd = PCO_CND_ALWAYS); + + /* Setup the epilogue. */ + b.cursor = pco_cursor_after_block(epilogue); + + pco_cndlt(&b, + pco_ref_pred(PCO_PRED_PE), + emc, + pco_ref_pred(PCO_PRED_P0), + emc, + pco_ref_imm8(2), /* TODO: make this a define */ + .exec_cnd = PCO_EXEC_CND_EX_ZX, + .cnd = PCO_CND_ALWAYS); + + pco_br(&b, + pco_cf_node_head(&loop->body), + .exec_cnd = PCO_EXEC_CND_E1_Z1, + .branch_cnd = PCO_BRANCH_CND_EXEC_COND); +} + +static inline void lower_break_continue(pco_instr *instr, + pco_func *func, + pco_if *pif, + pco_loop *loop, + unsigned loop_nestings, + bool is_continue) +{ + pco_builder b = pco_builder_create(func, pco_cursor_before_instr(instr)); + pco_ref emc = emc_ref(func, &b); + enum pco_exec_cnd exec_cnd = pco_instr_get_exec_cnd(instr); + + pco_ref val = pco_ref_new_ssa32(func); + pco_movi32(&b, + val, + pco_ref_imm32(loop_nestings + (is_continue ? 1 : 2)), + .exec_cnd = exec_cnd); + + enum pco_cnd cnd; + switch (exec_cnd) { + case PCO_CC_E1_ZX: + assert(!pif->pred_exec); + cnd = PCO_CND_ALWAYS; + break; + + case PCO_CC_E1_Z1: + assert(pif->pred_exec); + cnd = PCO_CND_P0_TRUE; + break; + + case PCO_CC_E1_Z0: + assert(pif->pred_exec); + cnd = PCO_CND_P0_FALSE; + break; + + default: + UNREACHABLE(""); } - return progress; + pco_cndsm(&b, + pco_ref_pred(PCO_PRED_PE), + emc, + emc, + val, + .exec_cnd = PCO_EXEC_CND_EX_ZX, + .cnd = cnd); + + pco_instr_delete(instr); } static inline bool pco_lower_cf(pco_func *func) { bool progress = false; - progress |= lower_ifs(func); - /* TODO: lower_loops(func); */ + void *mem_ctx = ralloc_context(NULL); + unsigned loop_nestings = 0; + struct util_dynarray loop_nestings_stack; + struct util_dynarray pif_stack; + struct util_dynarray loop_stack; + + util_dynarray_init(&loop_nestings_stack, mem_ctx); + util_dynarray_init(&pif_stack, mem_ctx); + util_dynarray_init(&loop_stack, mem_ctx); + + pco_foreach_cf_node_in_func_structured (cf_node, cf_node_completed, func) { + /* Handle the end of an if/loop. */ + if (cf_node_completed) { + switch (cf_node_completed->type) { + case PCO_CF_NODE_TYPE_IF: { + pco_if *pif = pco_cf_node_as_if(cf_node_completed); + + ASSERTED pco_if *last_pif = util_dynarray_pop(&pif_stack, pco_if *); + assert(pif == last_pif); + + if (!pif->pred_exec) + --loop_nestings; + + break; + } + + case PCO_CF_NODE_TYPE_LOOP: { + ASSERTED pco_loop *loop = pco_cf_node_as_loop(cf_node_completed); + ASSERTED pco_loop *last_loop = + util_dynarray_pop(&loop_stack, pco_loop *); + assert(loop == last_loop); + + assert(loop_nestings == 0); + loop_nestings = util_dynarray_pop(&loop_nestings_stack, unsigned); + break; + } + + default: + break; + } + } + + /* Handle the start of an if/loop, or lower break/continue for blocks. */ + switch (cf_node->type) { + case PCO_CF_NODE_TYPE_IF: { + pco_if *pif = pco_cf_node_as_if(cf_node); + pif->pred_exec = can_pred_exec(pif); + util_dynarray_append(&pif_stack, pco_if *, pif); + + if (!pif->pred_exec) + ++loop_nestings; + + lower_if(pif, func); + progress = true; + + break; + } + + case PCO_CF_NODE_TYPE_LOOP: { + util_dynarray_append(&loop_nestings_stack, unsigned, loop_nestings); + loop_nestings = 0; + + pco_loop *loop = pco_cf_node_as_loop(cf_node); + util_dynarray_append(&loop_stack, pco_loop *, loop); + + lower_loop(loop, func); + progress = true; + + break; + } + + case PCO_CF_NODE_TYPE_BLOCK: { + pco_block *block = pco_cf_node_as_block(cf_node); + pco_foreach_instr_in_block_safe (instr, block) { + if (instr->op != PCO_OP_BREAK && instr->op != PCO_OP_CONTINUE) + continue; + + /* This has to be the last instruction in the block. */ + assert(instr == pco_last_instr(block)); + + pco_if *current_pif = util_dynarray_top(&pif_stack, pco_if *); + pco_loop *current_loop = util_dynarray_top(&loop_stack, pco_loop *); + + assert(current_pif == + pco_cf_node_as_if(instr->parent_block->cf_node.parent)); + + lower_break_continue(instr, + func, + current_pif, + current_loop, + loop_nestings, + instr->op == PCO_OP_CONTINUE); + + progress = true; + } + + break; + } + + default: + break; + } + } + + assert(!util_dynarray_num_elements(&loop_stack, pco_loop *)); + assert(!util_dynarray_num_elements(&pif_stack, pco_if *)); + assert(!util_dynarray_num_elements(&loop_nestings_stack, unsigned)); + assert(loop_nestings == 0); + + ralloc_free(mem_ctx); return progress; } diff --git a/src/imagination/pco/pco_debug.c b/src/imagination/pco/pco_debug.c index 4b7140006fd..881ed0c60b5 100644 --- a/src/imagination/pco/pco_debug.c +++ b/src/imagination/pco/pco_debug.c @@ -25,6 +25,7 @@ static const struct debug_named_value pco_debug_options[] = { { "val_skip", PCO_DEBUG_VAL_SKIP, "Skip IR validation." }, { "reindex", PCO_DEBUG_REINDEX, "Reindex IR at the end of each pass." }, + { "no_pred_cf", PCO_DEBUG_NO_PRED_CF, "No predicated execution in CF." }, DEBUG_NAMED_VALUE_END, }; diff --git a/src/imagination/pco/pco_end.c b/src/imagination/pco/pco_end.c index e131619a509..5c2e0f5d8c6 100644 --- a/src/imagination/pco/pco_end.c +++ b/src/imagination/pco/pco_end.c @@ -35,7 +35,7 @@ bool pco_end(pco_shader *shader) if (shader->stage == MESA_SHADER_VERTEX) { if (last_instr && last_instr->op == PCO_OP_UVSW_WRITE && - pco_instr_default_exec(last_instr) && + pco_instr_has_default_exec(last_instr) && pco_instr_get_rpt(last_instr) == 1) { pco_instr *new_last_instr = pco_uvsw_write_emit_endtask(&b, diff --git a/src/imagination/pco/pco_internal.h b/src/imagination/pco/pco_internal.h index a54e22e808f..c8ae7be8c3c 100644 --- a/src/imagination/pco/pco_internal.h +++ b/src/imagination/pco/pco_internal.h @@ -57,6 +57,7 @@ void pco_setup_nir_options(const struct pvr_device_info *dev_info, enum pco_debug { PCO_DEBUG_VAL_SKIP = BITFIELD64_BIT(0), PCO_DEBUG_REINDEX = BITFIELD64_BIT(1), + PCO_DEBUG_NO_PRED_CF = BITFIELD64_BIT(2), }; extern uint64_t pco_debug; @@ -296,6 +297,7 @@ typedef struct _pco_if { pco_cf_node cf_node; /** CF node. */ pco_func *parent_func; /** Parent function. */ pco_ref cond; /** If condition. */ + bool pred_exec; /** Whether this if construct uses predicated execution. */ struct exec_list prologue; /** List of pco_cf_nodes for if prologue. */ struct exec_list then_body; /** List of pco_cf_nodes for if body. */ struct exec_list interlogue; /** List of pco_cf_nodes for if interlogue. */ @@ -510,6 +512,14 @@ PCO_DEFINE_CAST(pco_cf_node_as_func, #define pco_foreach_cf_node_in_func(cf_node, func) \ foreach_list_typed (pco_cf_node, cf_node, node, &(func)->body) +#define pco_foreach_cf_node_in_func_structured(cf_node, \ + cf_node_completed, \ + func) \ + for (pco_cf_node *cf_node = pco_first_func_cf_node(func), \ + *cf_node_completed = NULL; \ + cf_node != NULL; \ + cf_node = pco_next_cf_node(cf_node, &cf_node_completed)) + #define pco_foreach_block_in_func(block, func) \ for (pco_block *block = pco_func_first_block(func); block != NULL; \ block = pco_next_block(block)) @@ -785,6 +795,34 @@ static inline unsigned pco_igrp_variant(const pco_igrp *igrp, /* Motions. */ +/** + * \brief Returns the first CF node in a PCO function. + * + * \param[in] func PCO function. + * \return The first CF node. + */ +static inline pco_cf_node *pco_first_func_cf_node(pco_func *func) +{ + if (!exec_list_is_empty(&func->body)) + return pco_cf_node_head(&func->body); + + UNREACHABLE("Empty function."); +} + +/** + * \brief Returns the last CF node in a PCO function. + * + * \param[in] func PCO function. + * \return The last CF node. + */ +static inline pco_cf_node *pco_last_func_cf_node(pco_func *func) +{ + if (!exec_list_is_empty(&func->body)) + return pco_cf_node_tail(&func->body); + + UNREACHABLE("Empty function."); +} + /** * \brief Returns the first CF node in a PCO if. * @@ -1055,10 +1093,16 @@ static inline pco_cf_node *pco_prev_loop_cf_node(pco_cf_node *cf_node) * \brief Returns the next CF node. * * \param[in] cf_node The current CF node. + * \param[out] cf_node_completed The last parent CF node (function, if, loop) + * completed, or NULL if none/not a parent. * \return The next CF node. */ -static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node) +static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node, + pco_cf_node **cf_node_completed) { + if (cf_node_completed) + *cf_node_completed = NULL; + if (!cf_node) return NULL; @@ -1104,12 +1148,18 @@ static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node) /* End of the function; return NULL. */ case PCO_CF_NODE_TYPE_FUNC: + if (cf_node_completed) + *cf_node_completed = parent_cf_node; + return NULL; default: UNREACHABLE(""); } + if (cf_node_completed) + *cf_node_completed = parent_cf_node; + return pco_cf_node_next(parent_cf_node); } @@ -1117,10 +1167,16 @@ static inline pco_cf_node *pco_next_cf_node(pco_cf_node *cf_node) * \brief Returns the previous CF node. * * \param[in] cf_node The current CF node. + * \param[out] cf_node_completed The last parent CF node (function, if, loop) + * completed, or NULL if none/not a parent. * \return The previous CF node. */ -static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node) +static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node, + pco_cf_node **cf_node_completed) { + if (cf_node_completed) + *cf_node_completed = NULL; + if (!cf_node) return NULL; @@ -1166,12 +1222,18 @@ static inline pco_cf_node *pco_prev_cf_node(pco_cf_node *cf_node) /* Start of the function; return NULL. */ case PCO_CF_NODE_TYPE_FUNC: + if (cf_node_completed) + *cf_node_completed = parent_cf_node; + return NULL; default: UNREACHABLE(""); } + if (cf_node_completed) + *cf_node_completed = parent_cf_node; + return pco_cf_node_prev(parent_cf_node); } @@ -1186,7 +1248,7 @@ static inline pco_cf_node *pco_next_cf_node_type(pco_cf_node *cf_node, enum pco_cf_node_type type) { do { - cf_node = pco_next_cf_node(cf_node); + cf_node = pco_next_cf_node(cf_node, NULL); } while (cf_node != NULL && cf_node->type != type); return cf_node; @@ -1222,7 +1284,7 @@ static inline pco_cf_node *pco_prev_cf_node_type(pco_cf_node *cf_node, enum pco_cf_node_type type) { do { - cf_node = pco_prev_cf_node(cf_node); + cf_node = pco_prev_cf_node(cf_node, NULL); } while (cf_node != NULL && cf_node->type != type); return cf_node; diff --git a/src/imagination/pco/pco_ir.c b/src/imagination/pco/pco_ir.c index 64abd9657e1..fe5b17944fa 100644 --- a/src/imagination/pco/pco_ir.c +++ b/src/imagination/pco/pco_ir.c @@ -40,6 +40,9 @@ void pco_process_ir(pco_ctx *ctx, pco_shader *shader) PCO_PASS(_, shader, pco_shrink_vecs); + PCO_PASS(_, shader, pco_const_imms); + PCO_PASS(_, shader, pco_opt); + do { progress = false; PCO_PASS(progress, shader, pco_dce); diff --git a/src/imagination/pco/pco_legalize.c b/src/imagination/pco/pco_legalize.c index ec787f1aeed..69e46790663 100644 --- a/src/imagination/pco/pco_legalize.c +++ b/src/imagination/pco/pco_legalize.c @@ -38,10 +38,12 @@ static void insert_mov_ref(pco_instr *instr, pco_ref *ref, bool needs_s124) pco_builder b = pco_builder_create(instr->parent_func, pco_cursor_before_instr(instr)); + enum pco_exec_cnd exec_cnd = pco_instr_get_exec_cnd(instr); + pco_instr *mov_instr; if (needs_s124) - pco_movs1(&b, new_ref, *ref); + mov_instr = pco_movs1(&b, new_ref, *ref, .exec_cnd = exec_cnd); else - pco_mbyp(&b, new_ref, *ref); + mov_instr = pco_mbyp(&b, new_ref, *ref, .exec_cnd = exec_cnd); *ref = new_ref; } diff --git a/src/imagination/pco/pco_map.py b/src/imagination/pco/pco_map.py index b19c4d6b6d9..9ec20e930a9 100644 --- a/src/imagination/pco/pco_map.py +++ b/src/imagination/pco/pco_map.py @@ -1596,6 +1596,39 @@ encode_map(O_CNDST, op_ref_maps=[('ctrl', ['pe', 'w0'], ['s0', 'imm'])] ) +encode_map(O_CNDEF, + encodings=[ + (I_CND, [ + ('adjust', ('pco_ref_get_imm', SRC(1))), + ('pcnd', OM_CND), + ('cndinst', 'ef') + ]) + ], + op_ref_maps=[('ctrl', ['pe', 'w0'], ['s0', 'imm'])] +) + +encode_map(O_CNDSM, + encodings=[ + (I_CND, [ + ('adjust', 0), + ('pcnd', OM_CND), + ('cndinst', 'sm') + ]) + ], + op_ref_maps=[('ctrl', ['pe', 'w0'], ['s0', 's2'])] +) + +encode_map(O_CNDLT, + encodings=[ + (I_CND, [ + ('adjust', ('pco_ref_get_imm', SRC(1))), + ('pcnd', OM_CND), + ('cndinst', 'lt') + ]) + ], + op_ref_maps=[('ctrl', ['pe', 'w0', 'p0'], ['s0', 'imm'])] +) + encode_map(O_CNDEND, encodings=[ (I_CND, [ @@ -2985,6 +3018,58 @@ group_map(O_CNDST, dests=[('w[0]', ('ctrl', DEST(1)), 'w0')] ) +group_map(O_CNDEF, + hdr=(I_IGRP_HDR_CONTROL, [ + ('olchk', False), + ('w1p', False), + ('w0p', True), + ('cc', OM_EXEC_CND), + ('miscctl', False), + ('ctrlop', 'cnd') + ]), + enc_ops=[('ctrl', O_CNDEF)], + srcs=[ + ('s[0]', ('ctrl', SRC(0)), 's0'), + ('s[3]', 'pco_zero') + ], + dests=[('w[0]', ('ctrl', DEST(1)), 'w0')] +) + +group_map(O_CNDSM, + hdr=(I_IGRP_HDR_CONTROL, [ + ('olchk', False), + ('w1p', False), + ('w0p', True), + ('cc', OM_EXEC_CND), + ('miscctl', False), + ('ctrlop', 'cnd') + ]), + enc_ops=[('ctrl', O_CNDSM)], + srcs=[ + ('s[0]', ('ctrl', SRC(0)), 's0'), + ('s[2]', ('ctrl', SRC(1)), 's2'), + ('s[3]', 'pco_zero') + ], + dests=[('w[0]', ('ctrl', DEST(1)), 'w0')] +) + +group_map(O_CNDLT, + hdr=(I_IGRP_HDR_CONTROL, [ + ('olchk', False), + ('w1p', False), + ('w0p', True), + ('cc', OM_EXEC_CND), + ('miscctl', False), + ('ctrlop', 'cnd') + ]), + enc_ops=[('ctrl', O_CNDLT)], + srcs=[ + ('s[0]', ('ctrl', SRC(0)), 's0'), + ('s[3]', 'pco_zero') + ], + dests=[('w[0]', ('ctrl', DEST(1)), 'w0')] +) + group_map(O_CNDEND, hdr=(I_IGRP_HDR_CONTROL, [ ('olchk', False), diff --git a/src/imagination/pco/pco_ops.py b/src/imagination/pco/pco_ops.py index 97f69661424..3cfc309ec06 100644 --- a/src/imagination/pco/pco_ops.py +++ b/src/imagination/pco/pco_ops.py @@ -410,6 +410,8 @@ O_DITRP_READ = hw_op('ditrp.read', [OM_EXEC_CND, OM_ITR_MODE, OM_SAT, OM_SCHED, O_CNDST = hw_op('cndst', [OM_EXEC_CND, OM_CND], 2, 2) O_CNDEF = hw_op('cndef', [OM_EXEC_CND, OM_CND], 2, 2) +O_CNDSM = hw_op('cndsm', [OM_EXEC_CND, OM_CND], 2, 2) +O_CNDLT = hw_op('cndlt', [OM_EXEC_CND, OM_CND], 3, 2) O_CNDEND = hw_op('cndend', [OM_EXEC_CND], 2, 2) O_BR = hw_op('br', [OM_EXEC_CND, OM_BRANCH_CND, OM_LINK], has_target_cf_node=True) @@ -437,5 +439,8 @@ O_FNEG = pseudo_op('fneg', OM_ALU, 1, 1) O_FABS = pseudo_op('fabs', OM_ALU, 1, 1) O_FFLR = pseudo_op('fflr', OM_ALU, 1, 1) O_MOV = pseudo_op('mov', OM_ALU, 1, 1) -O_VEC = pseudo_op('vec', [], 1, VARIABLE, [], [[RM_ABS, RM_NEG]]) +O_VEC = pseudo_op('vec', [OM_EXEC_CND], 1, VARIABLE, [], [[RM_ABS, RM_NEG]]) O_COMP = pseudo_op('comp', [], 1, 2) + +O_BREAK = pseudo_op('break', [OM_EXEC_CND]) +O_CONTINUE = pseudo_op('continue', [OM_EXEC_CND]) diff --git a/src/imagination/pco/pco_ra.c b/src/imagination/pco/pco_ra.c index c13975b6490..205cb5ad4de 100644 --- a/src/imagination/pco/pco_ra.c +++ b/src/imagination/pco/pco_ra.c @@ -349,6 +349,30 @@ static bool pco_ra_func(pco_func *func, pco_extend_live_range(src_vec, dest, instr, overrides, live_ranges); } + /* Extend lifetimes of vars in loops. */ + pco_foreach_loop_in_func (loop, func) { + pco_block *prologue_block = + pco_cf_node_as_block(pco_cf_node_head(&loop->prologue)); + + pco_block *epilogue_block = + pco_cf_node_as_block(pco_cf_node_tail(&loop->epilogue)); + + unsigned loop_start_index = pco_first_instr(prologue_block)->index; + unsigned loop_end_index = pco_last_instr(epilogue_block)->index; + + /* If a var is defined before a loop and stops being used during it, + * extend its lifetime to the end of the loop. + */ + + for (unsigned var = 0; var < num_vars; ++var) { + if (live_ranges[var].start < loop_start_index && + live_ranges[var].end > loop_start_index && + live_ranges[var].end < loop_end_index) { + live_ranges[var].end = loop_end_index; + } + } + } + /* Build interference graph from overlapping live ranges. */ for (unsigned var0 = 0; var0 < num_vars; ++var0) { for (unsigned var1 = var0 + 1; var1 < num_vars; ++var1) { diff --git a/src/imagination/pco/pco_trans_nir.c b/src/imagination/pco/pco_trans_nir.c index cab4f7d5e3b..088d2085868 100644 --- a/src/imagination/pco/pco_trans_nir.c +++ b/src/imagination/pco/pco_trans_nir.c @@ -2017,6 +2017,29 @@ static pco_instr *trans_const(trans_ctx *tctx, nir_load_const_instr *nconst) return instr; } +/** + * \brief Translates a NIR jump instruction into PCO. + * + * \param[in] tctx Translation context. + * \param[in] jump The NIR jump instruction. + * \return The PCO instruction. + */ +static pco_instr *trans_jump(trans_ctx *tctx, nir_jump_instr *jump) +{ + switch (jump->type) { + case nir_jump_break: + return pco_break(&tctx->b); + + case nir_jump_continue: + return pco_continue(&tctx->b); + + default: + break; + } + + UNREACHABLE(""); +} + /** * \brief Translates a NIR instruction into PCO. * @@ -2036,6 +2059,9 @@ static pco_instr *trans_instr(trans_ctx *tctx, nir_instr *ninstr) case nir_instr_type_alu: return trans_alu(tctx, nir_instr_as_alu(ninstr)); + case nir_instr_type_jump: + return trans_jump(tctx, nir_instr_as_jump(ninstr)); + default: break; } @@ -2131,7 +2157,15 @@ static void trans_loop(trans_ctx *tctx, loop->cf_node.parent = parent_cf_node; exec_list_push_tail(cf_node_list, &loop->cf_node.node); - UNREACHABLE("finishme: trans_loop"); + assert(!nir_cf_list_is_empty_block(&nloop->body)); + assert(!nir_loop_has_continue_construct(nloop)); + + enum pco_cf_node_flag flag = tctx->flag; + tctx->flag = PCO_CF_NODE_FLAG_BODY; + + trans_cf_nodes(tctx, &loop->cf_node, &loop->body, &nloop->body); + + tctx->flag = flag; } /**