diff --git a/src/imagination/.clang-format b/src/imagination/.clang-format index 17106e7baaa..abb446037e2 100644 --- a/src/imagination/.clang-format +++ b/src/imagination/.clang-format @@ -218,6 +218,7 @@ ForEachMacros: [ 'pvr_csb_pack', 'nir_foreach_block', 'nir_foreach_block_safe', + 'nir_foreach_block_unstructured', 'nir_foreach_function', 'nir_foreach_instr', 'nir_foreach_instr_safe', @@ -258,6 +259,12 @@ ForEachMacros: [ 'rogue_foreach_reg_write_safe', 'rogue_foreach_regarray', 'rogue_foreach_regarray_safe', + 'rogue_foreach_regarray_use', + 'rogue_foreach_regarray_use_safe', + 'rogue_foreach_regarray_write', + 'rogue_foreach_regarray_write_safe', + 'rogue_foreach_subarray', + 'rogue_foreach_subarray_safe', 'rb_tree_foreach', 'rb_tree_foreach_safe', 'u_foreach_bit', diff --git a/src/imagination/rogue/passes/rogue_regalloc.c b/src/imagination/rogue/passes/rogue_regalloc.c index 75493a2700d..d0d7cbeae9c 100644 --- a/src/imagination/rogue/passes/rogue_regalloc.c +++ b/src/imagination/rogue/passes/rogue_regalloc.c @@ -23,8 +23,8 @@ #include "rogue.h" #include "util/macros.h" +#include "util/ralloc.h" #include "util/register_allocate.h" -#include "util/sparse_array.h" #include @@ -34,9 +34,6 @@ * \brief Contains the rogue_regalloc pass. */ -/* TODO: Tweak this value. */ -#define ROGUE_SSA_LIVE_RANGE_NODE_SIZE 512 - /* TODO: Internal register support for high register pressure regs. */ typedef struct rogue_live_range { @@ -56,6 +53,11 @@ bool rogue_regalloc(rogue_shader *shader) if (!num_ssa_regs) return false; + /* Ensure that ssa regs are continuous from zero, and have no gaps. */ + rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_SSA) { + assert(reg->index < num_ssa_regs); + } + /* If we already have some temps in use in the shader, we'll skip using them * for allocation. */ unsigned num_temp_regs = list_length(&shader->regs[ROGUE_REG_CLASS_TEMP]); @@ -76,23 +78,56 @@ bool rogue_regalloc(rogue_shader *shader) ra_set_finalize(ra_regs, NULL); - struct util_sparse_array ssa_live_range; - util_sparse_array_init(&ssa_live_range, - sizeof(rogue_live_range), - ROGUE_SSA_LIVE_RANGE_NODE_SIZE); + rogue_live_range *ssa_live_range = + rzalloc_array_size(shader, sizeof(*ssa_live_range), num_ssa_regs); + for (unsigned u = 0; u < num_ssa_regs; ++u) + ssa_live_range[u].start = ~0U; - /* Populate live ranges. */ + /* Populate live ranges for register arrays. */ + rogue_foreach_regarray (regarray, shader) { + enum rogue_reg_class class = regarray->regs[0]->class; + if (class != ROGUE_REG_CLASS_SSA) + continue; + + for (unsigned u = 0; u < regarray->size; ++u) { + rogue_reg *reg = regarray->regs[u]; + rogue_live_range *live_range = &ssa_live_range[reg->index]; + + assert(list_is_singular(®array->writes) || + list_is_empty(®array->writes)); + if (!list_is_empty(®array->writes)) { + rogue_regarray_write *write = + list_first_entry(®array->writes, rogue_regarray_write, link); + live_range->start = MIN2(live_range->start, write->instr->index); + } + + rogue_foreach_regarray_use (use, regarray) { + live_range->end = MAX2(live_range->end, use->instr->index); + } + + /* Here dirty represents whether the register has been added to the + * regset yet or not. */ + reg->dirty = false; + } + } + + /* Populate live ranges for registers. */ rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_SSA) { - rogue_live_range *live_range = - util_sparse_array_get(&ssa_live_range, reg->index); - rogue_reg_write *write = - list_first_entry(®->writes, rogue_reg_write, link); + if (reg->regarray) + continue; - live_range->start = write->instr->index; - live_range->end = live_range->start; + rogue_live_range *live_range = &ssa_live_range[reg->index]; - rogue_foreach_reg_use (use, reg) + assert(list_is_singular(®->writes) || list_is_empty(®->writes)); + if (!list_is_empty(®->writes)) { + rogue_reg_write *write = + list_first_entry(®->writes, rogue_reg_write, link); + live_range->start = MIN2(live_range->start, write->instr->index); + } + + rogue_foreach_reg_use (use, reg) { live_range->end = MAX2(live_range->end, use->instr->index); + } /* Here dirty represents whether the register has been added to the regset * yet or not. */ @@ -112,13 +147,18 @@ bool rogue_regalloc(rogue_shader *shader) if (regarray->parent) continue; - if (regarray->size != 4) + enum rogue_regalloc_class raclass; + + if (regarray->size == 2) + raclass = ROGUE_REGALLOC_CLASS_TEMP_2; + else if (regarray->size == 4) + raclass = ROGUE_REGALLOC_CLASS_TEMP_4; + else unreachable("Unsupported regarray size."); ra_set_node_class(ra_graph, regarray->regs[0]->index, - ra_get_class_from_index(ra_regs, - ROGUE_REGALLOC_CLASS_TEMP_4)); + ra_get_class_from_index(ra_regs, raclass)); for (unsigned u = 0; u < regarray->size; ++u) regarray->regs[u]->dirty = true; @@ -138,15 +178,13 @@ bool rogue_regalloc(rogue_shader *shader) /* Build interference graph from overlapping live ranges. */ for (unsigned index0 = 0; index0 < num_ssa_regs; ++index0) { - rogue_live_range *live_range0 = - util_sparse_array_get(&ssa_live_range, index0); + rogue_live_range *live_range0 = &ssa_live_range[index0]; for (unsigned index1 = 0; index1 < num_ssa_regs; ++index1) { if (index0 == index1) continue; - rogue_live_range *live_range1 = - util_sparse_array_get(&ssa_live_range, index1); + rogue_live_range *live_range1 = &ssa_live_range[index1]; /* If the live ranges overlap, those register nodes interfere. */ if (!(live_range0->start >= live_range1->end || @@ -155,82 +193,21 @@ bool rogue_regalloc(rogue_shader *shader) } } - /* Same src/dst interferences are disabled for the moment. - * This may need to be re-enabled in the future as certain instructions have - * restrictions on this. - */ -#if 0 - /* Add node interferences such that the same register can't be used for - * both an instruction's source and destination. - */ - rogue_foreach_instr_in_shader (instr, shader) { - switch (instr->type) { - case ROGUE_INSTR_TYPE_ALU: - { - const rogue_alu_instr *alu = rogue_instr_as_alu(instr); - const rogue_alu_op_info *info = &rogue_alu_op_infos[alu->op]; - - if (rogue_ref_get_reg_class(&alu->dst.ref) != ROGUE_REG_CLASS_SSA) - continue; - - for (unsigned s = 0; s < info->num_srcs; ++s) { - if (!rogue_ref_is_reg(&alu->src[s].ref)) - continue; - - if (rogue_ref_get_reg_class(&alu->src[s].ref) != ROGUE_REG_CLASS_SSA) - continue; - - ra_add_node_interference(ra_graph, rogue_ref_get_reg_index(&alu->dst.ref), rogue_ref_get_reg_index(&alu->src[s].ref)); - } - - break; - } - - case ROGUE_INSTR_TYPE_BACKEND: - { - const rogue_backend_instr *backend = rogue_instr_as_backend(instr); - const rogue_backend_op_info *info = &rogue_backend_op_infos[backend->op]; - - for (unsigned d = 0; d < info->num_dsts; ++d) { - if (rogue_ref_get_reg_class(&backend->dst[d].ref) != ROGUE_REG_CLASS_SSA) - continue; - - for (unsigned s = 0; s < info->num_srcs; ++s) { - if (!rogue_ref_is_reg(&backend->src[s].ref)) - continue; - - if (rogue_ref_get_reg_class(&backend->src[s].ref) != ROGUE_REG_CLASS_SSA) - continue; - - ra_add_node_interference(ra_graph, rogue_ref_get_reg_index(&backend->dst[d].ref), rogue_ref_get_reg_index(&backend->src[s].ref)); - } - } - - break; - } - - case ROGUE_INSTR_TYPE_CTRL: - { - /* TODO: Support control instructions with I/O. */ - break; - } - - default: - unreachable("Unsupported instruction type."); - break; - } - } -#endif - /* TODO: Spilling support. */ if (!ra_allocate(ra_graph)) unreachable("Register allocation failed."); - /* Replace regarray SSA registers with allocated physical registers first. - * Don't want to be in a situation where the reg is updated before the - * regarray. - */ + unsigned regarray_count = list_length(&shader->regarrays); + rogue_regarray **rogue_regarrays = + rzalloc_array_size(shader, sizeof(*rogue_regarrays), regarray_count); + unsigned u = 0; rogue_foreach_regarray (regarray, shader) { + rogue_regarrays[u++] = regarray; + } + assert(u == regarray_count); + + for (u = 0; u < regarray_count; ++u) { + rogue_regarray *regarray = rogue_regarrays[u]; enum rogue_reg_class class = regarray->regs[0]->class; if (class != ROGUE_REG_CLASS_SSA) continue; @@ -240,17 +217,40 @@ bool rogue_regalloc(rogue_shader *shader) unsigned start_index = regarray->regs[0]->index; unsigned new_base_index = ra_get_node_reg(ra_graph, start_index); - for (unsigned u = 0; u < regarray->size; ++u) { - enum rogue_regalloc_class ra_class = - ra_class_index(ra_get_node_class(ra_graph, start_index + u)); - enum rogue_reg_class new_class = regalloc_info[ra_class].class; + enum rogue_regalloc_class ra_class = + ra_class_index(ra_get_node_class(ra_graph, start_index)); + enum rogue_reg_class new_class = regalloc_info[ra_class].class; - /* Register should not have already been used. */ - assert(!BITSET_TEST(shader->regs_used[new_class], new_base_index + u)); - progress |= rogue_reg_rewrite(shader, - regarray->regs[u], - new_class, - new_base_index + u); + bool used = false; + for (unsigned r = 0; r < regarray->size; ++r) + used |= rogue_reg_is_used(shader, new_class, new_base_index + r); + + /* First time using new regarray, modify in place. */ + if (!used) { + progress |= + rogue_regarray_rewrite(shader, regarray, new_class, new_base_index); + } else { + /* Regarray has already been used, replace references and delete. */ + + /* Replace parent regarray first. */ + rogue_regarray *new_regarray = rogue_regarray_cached(shader, + regarray->size, + new_class, + new_base_index); + progress |= rogue_regarray_replace(regarray, new_regarray); + rogue_regarray_delete(regarray); + + /* Replace subarrays. */ + rogue_foreach_subarray_safe (subarray, regarray) { + unsigned idx_offset = + subarray->regs[0]->index - regarray->regs[0]->index; + new_regarray = rogue_regarray_cached(shader, + subarray->size, + new_class, + new_base_index + idx_offset); + progress |= rogue_regarray_replace(subarray, new_regarray); + rogue_regarray_delete(subarray); + } } } @@ -265,7 +265,7 @@ bool rogue_regalloc(rogue_shader *shader) enum rogue_reg_class new_class = regalloc_info[ra_class].class; /* First time using new register, modify in place. */ - if (!BITSET_TEST(shader->regs_used[new_class], new_index)) { + if (!rogue_reg_is_used(shader, new_class, new_index)) { progress |= rogue_reg_rewrite(shader, reg, new_class, new_index); } else { /* Register has already been used, replace references and delete. */ @@ -275,7 +275,14 @@ bool rogue_regalloc(rogue_shader *shader) } } - util_sparse_array_finish(&ssa_live_range); + num_temp_regs = list_length(&shader->regs[ROGUE_REG_CLASS_TEMP]); + /* Ensure that temp regs are continuous from zero, and have no gaps. */ + rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_TEMP) { + assert(reg->index < num_temp_regs); + } + + ralloc_free(rogue_regarrays); + ralloc_free(ssa_live_range); ralloc_free(ra_regs); return progress; } diff --git a/src/imagination/rogue/passes/rogue_trim.c b/src/imagination/rogue/passes/rogue_trim.c index ed4424432d7..d3f3e8d6b66 100644 --- a/src/imagination/rogue/passes/rogue_trim.c +++ b/src/imagination/rogue/passes/rogue_trim.c @@ -68,9 +68,19 @@ static bool rogue_trim_regs(rogue_shader *shader) if (regarray->parent) continue; - for (unsigned u = 0; u < regarray->size; ++u) - progress |= - rogue_reg_set(shader, regarray->regs[u], class, index[class]++); + rogue_regarray_set(shader, regarray, class, index[class], true); + + rogue_foreach_subarray (subarray, regarray) { + unsigned idx_offset = + subarray->regs[0]->index - regarray->regs[0]->index; + progress &= rogue_regarray_set(shader, + subarray, + class, + index[class] + idx_offset, + false); + } + + index[class] += regarray->size; } rogue_foreach_reg (reg, shader, ROGUE_REG_CLASS_SSA) { diff --git a/src/imagination/rogue/rogue.c b/src/imagination/rogue/rogue.c index 246b0e76a3a..94f8d5f3f7d 100644 --- a/src/imagination/rogue/rogue.c +++ b/src/imagination/rogue/rogue.c @@ -64,7 +64,7 @@ bool rogue_reg_set(rogue_shader *shader, if (info->num) { assert(index < info->num); - BITSET_SET(shader->regs_used[class], index); + rogue_set_reg_use(shader, class, index); } if (reg->class != class) { @@ -86,22 +86,6 @@ bool rogue_reg_set(rogue_shader *shader, *reg_cached = reg; reg->cached = reg_cached; - /* If this is the first member of a top-level regarray, update its cache - * entry. */ - rogue_regarray *regarray = reg->regarray; - if (regarray && !regarray->parent && regarray->regs[0] == reg) { - if (regarray->cached && *regarray->cached == regarray) - *regarray->cached = NULL; - - uint64_t key = - rogue_regarray_cache_key(regarray->size, class, index, false, 0); - - rogue_regarray **regarray_cached = - util_sparse_array_get(&shader->regarray_cache, key); - *regarray_cached = regarray; - regarray->cached = regarray_cached; - } - return changed; } @@ -123,14 +107,83 @@ bool rogue_reg_rewrite(rogue_shader *shader, { const rogue_reg_info *info = &rogue_reg_infos[reg->class]; if (info->num) { - assert(BITSET_TEST(reg->shader->regs_used[reg->class], reg->index) && + assert(rogue_reg_is_used(shader, reg->class, reg->index) && "Register not in use!"); - BITSET_CLEAR(reg->shader->regs_used[reg->class], reg->index); + rogue_clear_reg_use(shader, reg->class, reg->index); } return rogue_reg_set(shader, reg, class, index); } +PUBLIC +bool rogue_regarray_set(rogue_shader *shader, + rogue_regarray *regarray, + enum rogue_reg_class class, + unsigned base_index, + bool set_regs) +{ + bool updated = true; + + if (set_regs) { + for (unsigned u = 0; u < regarray->size; ++u) { + updated &= + rogue_reg_set(shader, regarray->regs[u], class, base_index + u); + } + } + + if (regarray->cached && *regarray->cached == regarray) + *regarray->cached = NULL; + + uint64_t key = + rogue_regarray_cache_key(regarray->size, class, base_index, false, 0); + + rogue_regarray **regarray_cached = + util_sparse_array_get(&shader->regarray_cache, key); + assert(*regarray_cached == NULL); + + *regarray_cached = regarray; + regarray->cached = regarray_cached; + + assert(updated); + return updated; +} + +bool rogue_regarray_rewrite(rogue_shader *shader, + rogue_regarray *regarray, + enum rogue_reg_class class, + unsigned base_index) +{ + bool progress = true; + + enum rogue_reg_class orig_class = regarray->regs[0]->class; + unsigned orig_base_index = regarray->regs[0]->index; + const rogue_reg_info *info = &rogue_reg_infos[orig_class]; + + assert(!regarray->parent); + + if (info->num) { + for (unsigned u = 0; u < regarray->size; ++u) { + assert(rogue_reg_is_used(shader, orig_class, orig_base_index) && + "Register not in use!"); + rogue_clear_reg_use(shader, orig_class, orig_base_index); + } + } + + progress &= rogue_regarray_set(shader, regarray, class, base_index, true); + + rogue_foreach_subarray (subarray, regarray) { + unsigned idx_offset = subarray->regs[0]->index - regarray->regs[0]->index; + progress &= rogue_regarray_set(shader, + subarray, + class, + base_index + idx_offset, + false); + } + + assert(progress); + return progress; +} + static void rogue_shader_destructor(void *ptr) { rogue_shader *shader = ptr; @@ -219,9 +272,9 @@ static rogue_reg *rogue_reg_create(rogue_shader *shader, const rogue_reg_info *info = &rogue_reg_infos[class]; if (info->num) { assert(index < info->num); - assert(!BITSET_TEST(shader->regs_used[class], index) && + assert(!rogue_reg_is_used(shader, class, index) && "Register already in use!"); - BITSET_SET(shader->regs_used[class], index); + rogue_set_reg_use(shader, class, index); } return reg; @@ -238,9 +291,9 @@ void rogue_reg_delete(rogue_reg *reg) assert(rogue_reg_is_unused(reg)); const rogue_reg_info *info = &rogue_reg_infos[reg->class]; if (info->num) { - assert(BITSET_TEST(reg->shader->regs_used[reg->class], reg->index) && + assert(rogue_reg_is_used(reg->shader, reg->class, reg->index) && "Register not in use!"); - BITSET_CLEAR(reg->shader->regs_used[reg->class], reg->index); + rogue_clear_reg_use(reg->shader, reg->class, reg->index); } if (reg->cached && *reg->cached == reg) @@ -396,6 +449,9 @@ static rogue_regarray *rogue_regarray_create(rogue_shader *shader, regarray->regs = rzalloc_size(regarray, sizeof(*regarray->regs) * size); regarray->size = size; regarray->cached = regarray_cached; + list_inithead(®array->children); + list_inithead(®array->writes); + list_inithead(®array->uses); for (unsigned u = 0; u < size; ++u) { regarray->regs[u] = @@ -422,12 +478,15 @@ static rogue_regarray *rogue_regarray_create(rogue_shader *shader, common_regarray->parent = regarray; ralloc_free(common_regarray->regs); common_regarray->regs = parent_regptr; + list_addtail(®array->children, &common_regarray->child_link); } else { /* We share registers with another regarray, and we are a subset of it. */ regarray->parent = common_regarray; ralloc_free(regarray->regs); regarray->regs = parent_regptr; + assert(list_is_empty(®array->children)); + list_addtail(®array->child_link, &common_regarray->children); } } @@ -461,10 +520,11 @@ rogue_regarray_cached_common(rogue_shader *shader, return *regarray_cached; } -static inline rogue_regarray *rogue_regarray_cached(rogue_shader *shader, - unsigned size, - enum rogue_reg_class class, - uint32_t start_index) +PUBLIC +rogue_regarray *rogue_regarray_cached(rogue_shader *shader, + unsigned size, + enum rogue_reg_class class, + uint32_t start_index) { return rogue_regarray_cached_common(shader, size, @@ -474,12 +534,12 @@ static inline rogue_regarray *rogue_regarray_cached(rogue_shader *shader, false); } -static inline rogue_regarray * -rogue_vec_regarray_cached(rogue_shader *shader, - unsigned size, - enum rogue_reg_class class, - uint32_t start_index, - uint8_t component) +PUBLIC +rogue_regarray *rogue_vec_regarray_cached(rogue_shader *shader, + unsigned size, + enum rogue_reg_class class, + uint32_t start_index, + uint8_t component) { return rogue_regarray_cached_common(shader, size, @@ -653,11 +713,13 @@ void rogue_link_instr_write(rogue_instr *instr) if (rogue_ref_is_reg(&alu->dst[i].ref)) { rogue_reg_write *write = &alu->dst_write[i].reg; rogue_reg *reg = alu->dst[i].ref.reg; - rogue_link_instr_write_reg(instr, write, reg, 0); + rogue_link_instr_write_reg(instr, write, reg, i); } else if (rogue_ref_is_regarray(&alu->dst[i].ref)) { - struct util_dynarray **writearray = &alu->dst_write[i].regarray; + rogue_regarray_write *write = &alu->dst_write[i].regarray; rogue_regarray *regarray = alu->dst[i].ref.regarray; - rogue_link_instr_write_regarray(instr, writearray, regarray, 0); + rogue_link_instr_write_regarray(instr, write, regarray, i); + } else if (rogue_ref_is_io(&alu->dst[i].ref)) { /* TODO: check WHICH IO + IT IS */ } else { unreachable("Unsupported destination reference type."); } @@ -676,9 +738,12 @@ void rogue_link_instr_write(rogue_instr *instr) rogue_reg *reg = backend->dst[i].ref.reg; rogue_link_instr_write_reg(instr, write, reg, i); } else if (rogue_ref_is_regarray(&backend->dst[i].ref)) { - struct util_dynarray **writearray = &backend->dst_write[i].regarray; + rogue_regarray_write *write = &backend->dst_write[i].regarray; rogue_regarray *regarray = backend->dst[i].ref.regarray; - rogue_link_instr_write_regarray(instr, writearray, regarray, i); + rogue_link_instr_write_regarray(instr, write, regarray, i); + } else if (rogue_ref_is_io(&backend->dst[i].ref)) { /* TODO: check + WHICH IO IT IS + */ } else { unreachable("Unsupported destination reference type."); } @@ -697,9 +762,11 @@ void rogue_link_instr_write(rogue_instr *instr) rogue_reg *reg = ctrl->dst[i].ref.reg; rogue_link_instr_write_reg(instr, write, reg, i); } else if (rogue_ref_is_regarray(&ctrl->dst[i].ref)) { - struct util_dynarray **writearray = &ctrl->dst_write[i].regarray; + rogue_regarray_write *write = &ctrl->dst_write[i].regarray; rogue_regarray *regarray = ctrl->dst[i].ref.regarray; - rogue_link_instr_write_regarray(instr, writearray, regarray, i); + rogue_link_instr_write_regarray(instr, write, regarray, i); + } else if (rogue_ref_is_io(&ctrl->dst[i].ref)) { /* TODO: check WHICH + IO IT IS */ } else { unreachable("Unsupported destination reference type."); } @@ -718,9 +785,12 @@ void rogue_link_instr_write(rogue_instr *instr) rogue_reg *reg = bitwise->dst[i].ref.reg; rogue_link_instr_write_reg(instr, write, reg, i); } else if (rogue_ref_is_regarray(&bitwise->dst[i].ref)) { - rogue_regarray_write *writearray = &bitwise->dst_write[i].regarray; + rogue_regarray_write *write = &bitwise->dst_write[i].regarray; rogue_regarray *regarray = bitwise->dst[i].ref.regarray; - rogue_link_instr_write_regarray(instr, writearray, regarray, i); + rogue_link_instr_write_regarray(instr, write, regarray, i); + } else if (rogue_ref_is_io(&bitwise->dst[i].ref)) { /* TODO: check + WHICH IO IT IS + */ } else { unreachable("Unsupported destination reference type."); } @@ -753,14 +823,17 @@ void rogue_link_instr_use(rogue_instr *instr) rogue_reg *reg = alu->src[i].ref.reg; rogue_link_instr_use_reg(instr, use, reg, i); } else if (rogue_ref_is_regarray(&alu->src[i].ref)) { - struct util_dynarray **usearray = &alu->src_use[i].regarray; + rogue_regarray_use *use = &alu->src_use[i].regarray; rogue_regarray *regarray = alu->src[i].ref.regarray; - rogue_link_instr_use_regarray(instr, usearray, regarray, i); + rogue_link_instr_use_regarray(instr, use, regarray, i); } else if (rogue_ref_is_imm(&alu->src[i].ref)) { rogue_link_imm_use(instr->block->shader, instr, i, rogue_ref_get_imm(&alu->src[i].ref)); + } else if (rogue_ref_is_io(&alu->src[i].ref)) { /* TODO: check WHICH IO + IT IS */ + } else if (rogue_ref_is_val(&alu->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -779,13 +852,17 @@ void rogue_link_instr_use(rogue_instr *instr) rogue_reg *reg = backend->src[i].ref.reg; rogue_link_instr_use_reg(instr, use, reg, i); } else if (rogue_ref_is_regarray(&backend->src[i].ref)) { - struct util_dynarray **usearray = &backend->src_use[i].regarray; + rogue_regarray_use *use = &backend->src_use[i].regarray; rogue_regarray *regarray = backend->src[i].ref.regarray; - rogue_link_instr_use_regarray(instr, usearray, regarray, i); + rogue_link_instr_use_regarray(instr, use, regarray, i); } else if (rogue_ref_is_drc(&backend->src[i].ref)) { rogue_link_drc_trxn(instr->block->shader, instr, rogue_ref_get_drc(&backend->src[i].ref)); + } else if (rogue_ref_is_io(&backend->src[i].ref)) { /* TODO: check + WHICH IO IT IS + */ + } else if (rogue_ref_is_val(&backend->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -812,15 +889,18 @@ void rogue_link_instr_use(rogue_instr *instr) rogue_reg *reg = ctrl->src[i].ref.reg; rogue_link_instr_use_reg(instr, use, reg, i); } else if (rogue_ref_is_regarray(&ctrl->src[i].ref)) { - struct util_dynarray **usearray = &ctrl->src_use[i].regarray; + rogue_regarray_use *use = &ctrl->src_use[i].regarray; rogue_regarray *regarray = ctrl->src[i].ref.regarray; - rogue_link_instr_use_regarray(instr, usearray, regarray, i); + rogue_link_instr_use_regarray(instr, use, regarray, i); } else if (rogue_ref_is_drc(&ctrl->src[i].ref)) { /* WDF instructions consume/release drcs, handled independently. */ if (ctrl->op != ROGUE_CTRL_OP_WDF) rogue_link_drc_trxn(instr->block->shader, instr, rogue_ref_get_drc(&ctrl->src[i].ref)); + } else if (rogue_ref_is_io(&ctrl->src[i].ref)) { /* TODO: check WHICH + IO IT IS */ + } else if (rogue_ref_is_val(&ctrl->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -839,13 +919,17 @@ void rogue_link_instr_use(rogue_instr *instr) rogue_reg *reg = bitwise->src[i].ref.reg; rogue_link_instr_use_reg(instr, use, reg, i); } else if (rogue_ref_is_regarray(&bitwise->src[i].ref)) { - rogue_regarray_use *usearray = &bitwise->src_use[i].regarray; + rogue_regarray_use *use = &bitwise->src_use[i].regarray; rogue_regarray *regarray = bitwise->src[i].ref.regarray; - rogue_link_instr_use_regarray(instr, usearray, regarray, i); + rogue_link_instr_use_regarray(instr, use, regarray, i); } else if (rogue_ref_is_drc(&bitwise->src[i].ref)) { rogue_link_drc_trxn(instr->block->shader, instr, rogue_ref_get_drc(&bitwise->src[i].ref)); + } else if (rogue_ref_is_io(&bitwise->src[i].ref)) { /* TODO: check + WHICH IO IT IS + */ + } else if (rogue_ref_is_val(&bitwise->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -878,8 +962,10 @@ void rogue_unlink_instr_write(rogue_instr *instr) rogue_reg_write *write = &alu->dst_write[i].reg; rogue_unlink_instr_write_reg(instr, write); } else if (rogue_ref_is_regarray(&alu->dst[i].ref)) { - struct util_dynarray **writearray = &alu->dst_write[i].regarray; - rogue_unlink_instr_write_regarray(instr, writearray); + rogue_regarray_write *write = &alu->dst_write[i].regarray; + rogue_unlink_instr_write_regarray(instr, write); + } else if (rogue_ref_is_io(&alu->dst[i].ref)) { /* TODO: check WHICH IO + IT IS */ } else { unreachable("Unsupported destination reference type."); } @@ -897,8 +983,11 @@ void rogue_unlink_instr_write(rogue_instr *instr) rogue_reg_write *write = &backend->dst_write[i].reg; rogue_unlink_instr_write_reg(instr, write); } else if (rogue_ref_is_regarray(&backend->dst[i].ref)) { - struct util_dynarray **writearray = &backend->dst_write[i].regarray; - rogue_unlink_instr_write_regarray(instr, writearray); + rogue_regarray_write *write = &backend->dst_write[i].regarray; + rogue_unlink_instr_write_regarray(instr, write); + } else if (rogue_ref_is_io(&backend->dst[i].ref)) { /* TODO: check + WHICH IO IT IS + */ } else { unreachable("Unsupported destination reference type."); } @@ -916,8 +1005,10 @@ void rogue_unlink_instr_write(rogue_instr *instr) rogue_reg_write *write = &ctrl->dst_write[i].reg; rogue_unlink_instr_write_reg(instr, write); } else if (rogue_ref_is_regarray(&ctrl->dst[i].ref)) { - struct util_dynarray **writearray = &ctrl->dst_write[i].regarray; - rogue_unlink_instr_write_regarray(instr, writearray); + rogue_regarray_write *write = &ctrl->dst_write[i].regarray; + rogue_unlink_instr_write_regarray(instr, write); + } else if (rogue_ref_is_io(&ctrl->dst[i].ref)) { /* TODO: check WHICH + IO IT IS */ } else { unreachable("Unsupported destination reference type."); } @@ -935,8 +1026,8 @@ void rogue_unlink_instr_write(rogue_instr *instr) rogue_reg_write *write = &bitwise->dst_write[i].reg; rogue_unlink_instr_write_reg(instr, write); } else if (rogue_ref_is_regarray(&bitwise->dst[i].ref)) { - rogue_regarray_write *writearray = &bitwise->dst_write[i].regarray; - rogue_unlink_instr_write_regarray(instr, writearray); + rogue_regarray_write *write = &bitwise->dst_write[i].regarray; + rogue_unlink_instr_write_regarray(instr, write); } else { unreachable("Invalid destination reference type."); } @@ -968,11 +1059,14 @@ void rogue_unlink_instr_use(rogue_instr *instr) rogue_reg_use *use = &alu->src_use[i].reg; rogue_unlink_instr_use_reg(instr, use); } else if (rogue_ref_is_regarray(&alu->src[i].ref)) { - struct util_dynarray **usearray = &alu->src_use[i].regarray; - rogue_unlink_instr_use_regarray(instr, usearray); + rogue_regarray_use *use = &alu->src_use[i].regarray; + rogue_unlink_instr_use_regarray(instr, use); } else if (rogue_ref_is_imm(&alu->src[i].ref)) { rogue_unlink_imm_use(instr, &rogue_ref_get_imm(&alu->src[i].ref)->use); + } else if (rogue_ref_is_io(&alu->src[i].ref)) { /* TODO: check WHICH IO + IT IS */ + } else if (rogue_ref_is_val(&alu->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -990,12 +1084,16 @@ void rogue_unlink_instr_use(rogue_instr *instr) rogue_reg_use *use = &backend->src_use[i].reg; rogue_unlink_instr_use_reg(instr, use); } else if (rogue_ref_is_regarray(&backend->src[i].ref)) { - struct util_dynarray **usearray = &backend->src_use[i].regarray; - rogue_unlink_instr_use_regarray(instr, usearray); + rogue_regarray_use *use = &backend->src_use[i].regarray; + rogue_unlink_instr_use_regarray(instr, use); } else if (rogue_ref_is_drc(&backend->src[i].ref)) { rogue_unlink_drc_trxn(instr->block->shader, instr, rogue_ref_get_drc(&backend->src[i].ref)); + } else if (rogue_ref_is_io(&backend->src[i].ref)) { /* TODO: check + WHICH IO IT IS + */ + } else if (rogue_ref_is_val(&backend->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -1019,14 +1117,17 @@ void rogue_unlink_instr_use(rogue_instr *instr) rogue_reg_use *use = &ctrl->src_use[i].reg; rogue_unlink_instr_use_reg(instr, use); } else if (rogue_ref_is_regarray(&ctrl->src[i].ref)) { - struct util_dynarray **usearray = &ctrl->src_use[i].regarray; - rogue_unlink_instr_use_regarray(instr, usearray); + rogue_regarray_use *use = &ctrl->src_use[i].regarray; + rogue_unlink_instr_use_regarray(instr, use); } else if (rogue_ref_is_drc(&ctrl->src[i].ref)) { /* WDF instructions consume/release drcs, handled independently. */ if (ctrl->op != ROGUE_CTRL_OP_WDF) rogue_unlink_drc_trxn(instr->block->shader, instr, rogue_ref_get_drc(&ctrl->src[i].ref)); + } else if (rogue_ref_is_io(&ctrl->src[i].ref)) { /* TODO: check WHICH + IO IT IS */ + } else if (rogue_ref_is_val(&ctrl->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } @@ -1044,12 +1145,16 @@ void rogue_unlink_instr_use(rogue_instr *instr) rogue_reg_use *use = &bitwise->src_use[i].reg; rogue_unlink_instr_use_reg(instr, use); } else if (rogue_ref_is_regarray(&bitwise->src[i].ref)) { - rogue_regarray_use *usearray = &bitwise->src_use[i].regarray; - rogue_unlink_instr_use_regarray(instr, usearray); + rogue_regarray_use *use = &bitwise->src_use[i].regarray; + rogue_unlink_instr_use_regarray(instr, use); } else if (rogue_ref_is_drc(&bitwise->src[i].ref)) { rogue_unlink_drc_trxn(instr->block->shader, instr, rogue_ref_get_drc(&bitwise->src[i].ref)); + } else if (rogue_ref_is_io(&bitwise->src[i].ref)) { /* TODO: check + WHICH IO IT IS + */ + } else if (rogue_ref_is_val(&bitwise->src[i].ref)) { } else { unreachable("Unsupported source reference type."); } diff --git a/src/imagination/rogue/rogue.h b/src/imagination/rogue/rogue.h index 8bb3530c0f4..98a4d6f815f 100644 --- a/src/imagination/rogue/rogue.h +++ b/src/imagination/rogue/rogue.h @@ -107,6 +107,7 @@ static inline enum reg_bank rogue_reg_bank_encoding(enum rogue_reg_class class) enum rogue_regalloc_class { ROGUE_REGALLOC_CLASS_TEMP_1, + ROGUE_REGALLOC_CLASS_TEMP_2, ROGUE_REGALLOC_CLASS_TEMP_4, ROGUE_REGALLOC_CLASS_COUNT, @@ -216,24 +217,6 @@ static inline bool rogue_reg_is_unused(rogue_reg *reg) return list_is_empty(®->uses) && list_is_empty(®->writes); } -typedef struct rogue_regarray { - struct list_head link; /** Link in rogue_shader::regarrays. */ - unsigned size; /** Number of registers in the array. */ - rogue_regarray *parent; - rogue_reg **regs; /** Registers (allocated array if this is a parent, else - pointer to inside parent regarray->regs). */ - rogue_regarray **cached; -} rogue_regarray; - -#define rogue_foreach_regarray(regarray, shader) \ - list_for_each_entry (rogue_regarray, regarray, &(shader)->regarrays, link) - -#define rogue_foreach_regarray_safe(regarray, shader) \ - list_for_each_entry_safe (rogue_regarray, \ - regarray, \ - &(shader)->regarrays, \ - link) - struct rogue_regarray_cache_key { union { struct { @@ -261,6 +244,73 @@ static inline uint64_t rogue_regarray_cache_key(unsigned size, .val; } +typedef struct rogue_regarray { + struct list_head link; /** Link in rogue_shader::regarrays. */ + unsigned size; /** Number of registers in the array. */ + rogue_regarray *parent; + struct list_head children; /** List of subarrays with this regarray as their + parent. */ + struct list_head child_link; /** Link in rogue_regarray::children. */ + rogue_reg **regs; /** Registers (allocated array if this is a parent, else + pointer to inside parent regarray->regs). */ + rogue_regarray **cached; + + struct list_head writes; /** List of all writes to this register array. */ + struct list_head uses; /** List of all register array uses. */ +} rogue_regarray; + +#define rogue_foreach_regarray(regarray, shader) \ + list_for_each_entry (rogue_regarray, regarray, &(shader)->regarrays, link) + +#define rogue_foreach_regarray_safe(regarray, shader) \ + list_for_each_entry_safe (rogue_regarray, \ + regarray, \ + &(shader)->regarrays, \ + link) + +#define rogue_foreach_subarray(subarray, regarray) \ + assert(!regarray->parent); \ + list_for_each_entry (rogue_regarray, \ + subarray, \ + &(regarray)->children, \ + child_link) + +#define rogue_foreach_subarray_safe(subarray, regarray) \ + assert(!regarray->parent); \ + list_for_each_entry_safe (rogue_regarray, \ + subarray, \ + &(regarray)->children, \ + child_link) + +typedef struct rogue_instr rogue_instr; + +typedef struct rogue_regarray_write { + rogue_instr *instr; + unsigned dst_index; + struct list_head link; /** Link in rogue_regarray::writes. */ +} rogue_regarray_write; + +#define rogue_foreach_regarray_write(write, regarray) \ + list_for_each_entry (rogue_regarray_write, write, &(regarray)->writes, link) + +#define rogue_foreach_regarray_write_safe(write, regarray) \ + list_for_each_entry_safe (rogue_regarray_write, \ + write, \ + &(regarray)->writes, \ + link) + +typedef struct rogue_regarray_use { + rogue_instr *instr; + unsigned src_index; + struct list_head link; /** Link in rogue_regarray::uses. */ +} rogue_regarray_use; + +#define rogue_foreach_regarray_use(use, regarray) \ + list_for_each_entry (rogue_regarray_use, use, &(regarray)->uses, link) + +#define rogue_foreach_regarray_use_safe(use, regarray) \ + list_for_each_entry_safe (rogue_regarray_use, use, &(regarray)->uses, link) + /** Instruction phases, used in bitset. */ enum rogue_instr_phase { /** Main/ALU (and backend) instructions. */ @@ -920,12 +970,12 @@ typedef struct rogue_reg_use { typedef union rogue_dst_write { rogue_reg_write reg; - struct util_dynarray *regarray; + rogue_regarray_write regarray; } rogue_dst_write; typedef union rogue_src_use { rogue_reg_use reg; - struct util_dynarray *regarray; + rogue_regarray_use regarray; } rogue_src_use; typedef struct rogue_block_use { @@ -1696,6 +1746,27 @@ static inline void rogue_set_shader_name(rogue_shader *shader, const char *name) shader->name = ralloc_strdup(shader, name); } +static inline bool rogue_reg_is_used(const rogue_shader *shader, + enum rogue_reg_class class, + unsigned index) +{ + return BITSET_TEST(shader->regs_used[class], index); +} + +static inline void rogue_set_reg_use(rogue_shader *shader, + enum rogue_reg_class class, + unsigned index) +{ + BITSET_SET(shader->regs_used[class], index); +} + +static inline void rogue_clear_reg_use(rogue_shader *shader, + enum rogue_reg_class class, + unsigned index) +{ + BITSET_CLEAR(shader->regs_used[class], index); +} + /** * \brief Allocates and initializes a new rogue_shader object. * @@ -1740,6 +1811,40 @@ rogue_regarray *rogue_ssa_vec_regarray(rogue_shader *shader, unsigned start_index, unsigned component); +rogue_regarray *rogue_regarray_cached(rogue_shader *shader, + unsigned size, + enum rogue_reg_class class, + uint32_t start_index); + +rogue_regarray *rogue_vec_regarray_cached(rogue_shader *shader, + unsigned size, + enum rogue_reg_class class, + uint32_t start_index, + uint8_t component); + +static inline bool rogue_regarray_is_unused(rogue_regarray *regarray) +{ + return list_is_empty(®array->uses) && list_is_empty(®array->writes); +} + +static void rogue_regarray_delete(rogue_regarray *regarray) +{ + assert(rogue_regarray_is_unused(regarray)); + + if (!regarray->parent) { + for (unsigned u = 0; u < regarray->size; ++u) + rogue_reg_delete(regarray->regs[u]); + } + + if (regarray->cached && *regarray->cached == regarray) + *regarray->cached = NULL; + + list_del(®array->link); + if (regarray->parent) + list_del(®array->child_link); + ralloc_free(regarray); +} + static inline void rogue_reset_reg_usage(rogue_shader *shader, enum rogue_reg_class class) { @@ -1766,6 +1871,17 @@ bool rogue_reg_rewrite(rogue_shader *shader, enum rogue_reg_class class, unsigned index); +bool rogue_regarray_set(rogue_shader *shader, + rogue_regarray *regarray, + enum rogue_reg_class class, + unsigned base_index, + bool set_regs); + +bool rogue_regarray_rewrite(rogue_shader *shader, + rogue_regarray *regarray, + enum rogue_reg_class class, + unsigned base_index); + /** Cursor for Rogue instructions/groups and basic blocks. */ typedef struct rogue_cursor { bool block; @@ -2003,33 +2119,23 @@ static inline void rogue_unlink_instr_write_reg(rogue_instr *instr, list_del(&write->link); } -static inline void -rogue_link_instr_write_regarray(rogue_instr *instr, - struct util_dynarray **writearray, - rogue_regarray *regarray, - unsigned dst_index) +static inline void rogue_link_instr_write_regarray(rogue_instr *instr, + rogue_regarray_write *write, + rogue_regarray *regarray, + unsigned dst_index) { - assert(!*writearray); - *writearray = rzalloc_size(instr, sizeof(**writearray)); - util_dynarray_init(*writearray, instr); - ASSERTED void *mem = - util_dynarray_resize(*writearray, rogue_reg_write, regarray->size); - assert(mem); - - unsigned u = 0; - util_dynarray_foreach (*writearray, rogue_reg_write, write) - rogue_link_instr_write_reg(instr, write, regarray->regs[u++], dst_index); + write->instr = instr; + write->dst_index = dst_index; + list_addtail(&write->link, ®array->writes); } static inline void rogue_unlink_instr_write_regarray(rogue_instr *instr, - struct util_dynarray **writearray) + rogue_regarray_write *write) { - util_dynarray_foreach (*writearray, rogue_reg_write, write) - rogue_unlink_instr_write_reg(instr, write); - - ralloc_free(*writearray); - *writearray = NULL; + assert(write->instr == instr); + write->instr = NULL; + list_del(&write->link); } static inline void rogue_link_instr_use_reg(rogue_instr *instr, @@ -2050,33 +2156,22 @@ static inline void rogue_unlink_instr_use_reg(rogue_instr *instr, list_del(&use->link); } -static inline void -rogue_link_instr_use_regarray(rogue_instr *instr, - struct util_dynarray **usearray, - rogue_regarray *regarray, - unsigned src_index) +static inline void rogue_link_instr_use_regarray(rogue_instr *instr, + rogue_regarray_use *use, + rogue_regarray *regarray, + unsigned src_index) { - assert(!*usearray); - *usearray = rzalloc_size(instr, sizeof(**usearray)); - util_dynarray_init(*usearray, instr); - ASSERTED void *mem = - util_dynarray_resize(*usearray, rogue_reg_use, regarray->size); - assert(mem); - - unsigned u = 0; - util_dynarray_foreach (*usearray, rogue_reg_use, use) - rogue_link_instr_use_reg(instr, use, regarray->regs[u++], src_index); + use->instr = instr; + use->src_index = src_index; + list_addtail(&use->link, ®array->uses); } -static inline void -rogue_unlink_instr_use_regarray(rogue_instr *instr, - struct util_dynarray **usearray) +static inline void rogue_unlink_instr_use_regarray(rogue_instr *instr, + rogue_regarray_use *use) { - util_dynarray_foreach (*usearray, rogue_reg_use, use) - rogue_unlink_instr_use_reg(instr, use); - - ralloc_free(*usearray); - *usearray = NULL; + assert(use->instr == instr); + use->instr = NULL; + list_del(&use->link); } static inline void rogue_link_instr_use_block(rogue_instr *instr, @@ -2192,6 +2287,106 @@ static inline bool rogue_reg_replace(rogue_reg *old_reg, rogue_reg *new_reg) return replaced; } +/* TODO: try and commonise this with the reg one! */ +static inline bool rogue_dst_regarray_replace(rogue_regarray_write *write, + rogue_regarray *new_regarray) +{ + unsigned dst_index = write->dst_index; + rogue_instr *instr = write->instr; + rogue_ref *ref; + + switch (instr->type) { + case ROGUE_INSTR_TYPE_ALU: + ref = &rogue_instr_as_alu(instr)->dst[dst_index].ref; + break; + + case ROGUE_INSTR_TYPE_BACKEND: + ref = &rogue_instr_as_backend(instr)->dst[dst_index].ref; + break; + + case ROGUE_INSTR_TYPE_CTRL: + ref = &rogue_instr_as_ctrl(instr)->dst[dst_index].ref; + break; + + case ROGUE_INSTR_TYPE_BITWISE: + ref = &rogue_instr_as_bitwise(instr)->dst[dst_index].ref; + break; + + default: + unreachable("Unsupported instruction type."); + return false; + } + + /* We don't want to be modifying regs. */ + assert(rogue_ref_is_regarray(ref)); + + if (ref->regarray == new_regarray) + return false; + + rogue_unlink_instr_write_regarray(instr, write); + *ref = rogue_ref_regarray(new_regarray); + rogue_link_instr_write_regarray(instr, write, new_regarray, dst_index); + + return true; +} + +static inline bool rogue_src_regarray_replace(rogue_regarray_use *use, + rogue_regarray *new_regarray) +{ + unsigned src_index = use->src_index; + rogue_instr *instr = use->instr; + rogue_ref *ref; + + switch (instr->type) { + case ROGUE_INSTR_TYPE_ALU: + ref = &rogue_instr_as_alu(instr)->src[src_index].ref; + break; + + case ROGUE_INSTR_TYPE_BACKEND: + ref = &rogue_instr_as_backend(instr)->src[src_index].ref; + break; + + case ROGUE_INSTR_TYPE_CTRL: + ref = &rogue_instr_as_ctrl(instr)->src[src_index].ref; + break; + + default: + unreachable("Unsupported instruction type."); + return false; + } + + /* We don't want to be modifying reg. */ + assert(rogue_ref_is_regarray(ref)); + + if (ref->regarray == new_regarray) + return false; + + rogue_unlink_instr_use_regarray(instr, use); + *ref = rogue_ref_regarray(new_regarray); + rogue_link_instr_use_regarray(instr, use, new_regarray, src_index); + + return true; +} + +static inline bool rogue_regarray_replace(rogue_regarray *old_regarray, + rogue_regarray *new_regarray) +{ + bool replaced = true; + + rogue_foreach_regarray_write_safe (write, old_regarray) { + replaced &= rogue_dst_regarray_replace(write, new_regarray); + } + + rogue_foreach_regarray_use_safe (use, old_regarray) { + replaced &= rogue_src_regarray_replace(use, new_regarray); + } + + /* N.B. The old regarray isn't automatically deleted here, this needs to be + * done manually. */ + + return replaced; +} + static inline bool rogue_src_imm_replace(rogue_imm_use *imm_use, rogue_reg *new_reg) { diff --git a/src/imagination/rogue/rogue_build_data.c b/src/imagination/rogue/rogue_build_data.c index c615f77967a..938237801eb 100644 --- a/src/imagination/rogue/rogue_build_data.c +++ b/src/imagination/rogue/rogue_build_data.c @@ -442,11 +442,11 @@ static void collect_ubo_data(struct rogue_common_build_data *common_data, if (intr->intrinsic != nir_intrinsic_load_ubo) continue; - assert(nir_src_num_components(intr->src[0]) == 2); + assert(nir_src_num_components(intr->src[0]) == 1); + assert(nir_src_num_components(intr->src[1]) == 1); unsigned load_desc_set = nir_src_comp_as_uint(intr->src[0], 0); - unsigned load_binding = nir_src_comp_as_uint(intr->src[0], 1); - + unsigned load_binding = nir_src_comp_as_uint(intr->src[1], 0); if (load_desc_set != desc_set || load_binding != binding) continue; diff --git a/src/imagination/rogue/rogue_compile.c b/src/imagination/rogue/rogue_compile.c index ebd300b5289..047fc4288ce 100644 --- a/src/imagination/rogue/rogue_compile.c +++ b/src/imagination/rogue/rogue_compile.c @@ -33,6 +33,62 @@ * \brief Contains NIR to Rogue translation functions, and Rogue passes. */ +/* For ALU scalars */ + +static rogue_ref nir_ssa_reg(rogue_shader *shader, + unsigned index, + unsigned num_components, + unsigned component) +{ + if (num_components > 1) { + return rogue_ref_regarray( + rogue_ssa_vec_regarray(shader, 1, index, component)); + } + + return rogue_ref_reg(rogue_ssa_reg(shader, index)); +} + +static rogue_ref nir_ssa_regarray(rogue_shader *shader, + unsigned index, + unsigned num_components, + unsigned component) +{ + return rogue_ref_regarray( + rogue_ssa_vec_regarray(shader, num_components, index, component)); +} + +static rogue_ref nir_ssa_reg_alu_src(rogue_shader *shader, + const nir_alu_instr *alu, + unsigned src_num, + bool vec) +{ + unsigned index = alu->src[src_num].src.ssa->index; + unsigned num_components = alu->src[src_num].src.ssa->num_components; + + unsigned write_mask = alu->dest.write_mask; + unsigned bit_pos = ffs(write_mask) - 1; + assert(util_is_power_of_two_nonzero(write_mask)); + + unsigned component = alu->src[src_num].swizzle[bit_pos]; + + return vec ? nir_ssa_regarray(shader, index, num_components, component) + : nir_ssa_reg(shader, index, num_components, component); +} + +static rogue_ref +nir_ssa_reg_alu_dst(rogue_shader *shader, const nir_alu_instr *alu, bool vec) +{ + unsigned num_components = alu->dest.dest.ssa.num_components; + unsigned index = alu->dest.dest.ssa.index; + + unsigned write_mask = alu->dest.write_mask; + unsigned component = ffs(write_mask) - 1; + assert(util_is_power_of_two_nonzero(write_mask)); + + return vec ? nir_ssa_regarray(shader, index, num_components, component) + : nir_ssa_reg(shader, index, num_components, component); +} + static void trans_nir_jump_return(rogue_builder *b, nir_jump_instr *jump) { rogue_END(b); @@ -198,7 +254,7 @@ static void trans_nir_intrinsic_load_ubo(rogue_builder *b, &b->shader->ctx->common_data[b->shader->stage].ubo_data; unsigned desc_set = nir_src_comp_as_uint(intr->src[0], 0); - unsigned binding = nir_src_comp_as_uint(intr->src[0], 1); + unsigned binding = nir_src_comp_as_uint(intr->src[1], 0); unsigned offset = nir_intrinsic_range_base(intr); unsigned sh_index = rogue_ubo_reg(ubo_data, desc_set, binding, offset); @@ -231,49 +287,43 @@ static void trans_nir_intrinsic(rogue_builder *b, nir_intrinsic_instr *intr) static void trans_nir_alu_pack_unorm_4x8(rogue_builder *b, nir_alu_instr *alu) { - rogue_reg *dst = rogue_ssa_reg(b->shader, alu->dest.dest.ssa.index); - rogue_regarray *src_array = - rogue_ssa_vec_regarray(b->shader, 4, alu->src[0].src.ssa->index, 0); + rogue_ref dst = nir_ssa_reg_alu_dst(b->shader, alu, false); + rogue_ref src = nir_ssa_reg_alu_src(b->shader, alu, 0, true); - rogue_alu_instr *pck_u8888 = - rogue_PCK_U8888(b, rogue_ref_reg(dst), rogue_ref_regarray(src_array)); + rogue_alu_instr *pck_u8888 = rogue_PCK_U8888(b, dst, src); rogue_set_instr_repeat(&pck_u8888->instr, 4); rogue_set_alu_op_mod(pck_u8888, ROGUE_ALU_OP_MOD_SCALE); } static void trans_nir_alu_fmul(rogue_builder *b, nir_alu_instr *alu) { - rogue_reg *dst = rogue_ssa_reg(b->shader, alu->dest.dest.ssa.index); - rogue_reg *src0 = rogue_ssa_reg(b->shader, alu->src[0].src.ssa->index); - rogue_reg *src1 = rogue_ssa_reg(b->shader, alu->src[1].src.ssa->index); + rogue_ref dst = nir_ssa_reg_alu_dst(b->shader, alu, false); + rogue_ref src0 = nir_ssa_reg_alu_src(b->shader, alu, 0, false); + rogue_ref src1 = nir_ssa_reg_alu_src(b->shader, alu, 1, false); - rogue_FMUL(b, rogue_ref_reg(dst), rogue_ref_reg(src0), rogue_ref_reg(src1)); + rogue_FMUL(b, dst, src0, src1); } static void trans_nir_alu_ffma(rogue_builder *b, nir_alu_instr *alu) { - rogue_reg *dst = rogue_ssa_reg(b->shader, alu->dest.dest.ssa.index); - rogue_reg *src0 = rogue_ssa_reg(b->shader, alu->src[0].src.ssa->index); - rogue_reg *src1 = rogue_ssa_reg(b->shader, alu->src[1].src.ssa->index); - rogue_reg *src2 = rogue_ssa_reg(b->shader, alu->src[2].src.ssa->index); + rogue_ref dst = nir_ssa_reg_alu_dst(b->shader, alu, false); + rogue_ref src0 = nir_ssa_reg_alu_src(b->shader, alu, 0, false); + rogue_ref src1 = nir_ssa_reg_alu_src(b->shader, alu, 1, false); + rogue_ref src2 = nir_ssa_reg_alu_src(b->shader, alu, 2, false); - rogue_FMAD(b, - rogue_ref_reg(dst), - rogue_ref_reg(src0), - rogue_ref_reg(src1), - rogue_ref_reg(src2)); + rogue_FMAD(b, dst, src0, src1, src2); } static void trans_nir_alu_vecN(rogue_builder *b, nir_alu_instr *alu, unsigned n) { unsigned dst_index = alu->dest.dest.ssa.index; - rogue_reg *dst; + rogue_regarray *dst; rogue_reg *src; for (unsigned u = 0; u < n; ++u) { - dst = rogue_ssa_vec_reg(b->shader, dst_index, u); + dst = rogue_ssa_vec_regarray(b->shader, 1, dst_index, u); src = rogue_ssa_reg(b->shader, alu->src[u].src.ssa->index); - rogue_MOV(b, rogue_ref_reg(dst), rogue_ref_reg(src)); + rogue_MOV(b, rogue_ref_regarray(dst), rogue_ref_reg(src)); } } @@ -312,6 +362,23 @@ static inline void rogue_feedback_used_regs(rogue_build_ctx *ctx, BITSET_WORDS(rogue_reg_infos[ROGUE_REG_CLASS_INTERNAL].num)); } +static bool index_ssa_def_cb(nir_ssa_def *ssa, void *state) +{ + rogue_shader *shader = (rogue_shader *)state; + + if (ssa->num_components == 1) { + if (ssa->bit_size == 32) { + rogue_ssa_reg(shader, ssa->index); + } else if (ssa->bit_size == 64) { + rogue_ssa_vec_regarray(shader, 2, ssa->index, 0); + } + } else { + rogue_ssa_vec_regarray(shader, ssa->num_components, ssa->index, 0); + } + + return true; +} + /** * \brief Translates a NIR shader to Rogue. * @@ -323,7 +390,7 @@ PUBLIC rogue_shader *rogue_nir_to_rogue(rogue_build_ctx *ctx, const nir_shader *nir) { gl_shader_stage stage = nir->info.stage; - struct rogue_shader *shader = rogue_shader_create(ctx, stage); + rogue_shader *shader = rogue_shader_create(ctx, stage); if (!shader) return NULL; @@ -335,8 +402,17 @@ rogue_shader *rogue_nir_to_rogue(rogue_build_ctx *ctx, const nir_shader *nir) rogue_builder b; rogue_builder_init(&b, shader); - /* Translate shader entrypoint. */ nir_function_impl *entry = nir_shader_get_entrypoint((nir_shader *)nir); + + /* Go through SSA used by NIR and "reserve" them so that sub-arrays won't be + * declared before the parent arrays. */ + nir_foreach_block_unstructured (block, entry) { + nir_foreach_instr (instr, block) { + nir_foreach_ssa_def(instr, index_ssa_def_cb, shader); + } + } + + /* Translate shader entrypoint. */ nir_foreach_block (block, entry) { rogue_push_block(&b); diff --git a/src/imagination/rogue/rogue_info.c b/src/imagination/rogue/rogue_info.c index cf5b3c3d08c..ae11b126e1f 100644 --- a/src/imagination/rogue/rogue_info.c +++ b/src/imagination/rogue/rogue_info.c @@ -50,6 +50,7 @@ const rogue_reg_info rogue_reg_infos[ROGUE_REG_CLASS_COUNT] = { const rogue_regalloc_info regalloc_info[ROGUE_REGALLOC_CLASS_COUNT] = { [ROGUE_REGALLOC_CLASS_TEMP_1] = { .class = ROGUE_REG_CLASS_TEMP, .stride = 1, }, + [ROGUE_REGALLOC_CLASS_TEMP_2] = { .class = ROGUE_REG_CLASS_TEMP, .stride = 2, }, [ROGUE_REGALLOC_CLASS_TEMP_4] = { .class = ROGUE_REG_CLASS_TEMP, .stride = 4, }, }; @@ -364,7 +365,7 @@ const rogue_alu_op_info rogue_alu_op_infos[ROGUE_ALU_OP_COUNT] = { .supported_src_mods = { [0] = SM(ABS) | SM(NEG), }, - .supported_dst_types = { [0] = T(REG), }, + .supported_dst_types = { [0] = T(REG) | T(REGARRAY), }, .supported_src_types = { [0] = T(REG), }, @@ -440,7 +441,7 @@ const rogue_alu_op_info rogue_alu_op_infos[ROGUE_ALU_OP_COUNT] = { }, /* This mov is "fake" since it can be lowered to a MBYP, make a new instruction for real mov (call it MOVD?). */ [ROGUE_ALU_OP_MOV] = { .str = "mov", .num_dsts = 1, .num_srcs = 1, - .supported_dst_types = { [0] = T(REG), }, + .supported_dst_types = { [0] = T(REG) | T(REGARRAY), }, .supported_src_types = { [0] = T(REG) | T(IMM), }, diff --git a/src/imagination/rogue/rogue_nir.c b/src/imagination/rogue/rogue_nir.c index 4268a757c5c..4c5acc1f599 100644 --- a/src/imagination/rogue/rogue_nir.c +++ b/src/imagination/rogue/rogue_nir.c @@ -41,7 +41,7 @@ static const struct spirv_to_nir_options spirv_options = { .environment = NIR_SPIRV_VULKAN, /* Buffer address: (descriptor_set, binding), offset. */ - .ubo_addr_format = nir_address_format_vec2_index_32bit_offset, + .ubo_addr_format = nir_address_format_32bit_index_offset, }; static const nir_shader_compiler_options nir_options = { diff --git a/src/imagination/rogue/rogue_print.c b/src/imagination/rogue/rogue_print.c index ccc920096de..44acd23c1b6 100644 --- a/src/imagination/rogue/rogue_print.c +++ b/src/imagination/rogue/rogue_print.c @@ -145,11 +145,14 @@ static inline void rogue_print_regarray(FILE *fp, const rogue_reg *reg = regarray->regs[0]; const rogue_reg_info *info = &rogue_reg_infos[reg->class]; YELLOW(fp); - fprintf(fp, "%s%" PRIu32, info->str, reg->index); - RESET(fp); - fputs("-", fp); - YELLOW(fp); - fprintf(fp, "%" PRIu32, regarray->size + reg->index - 1); + fprintf(fp, "%s[%" PRIu32, info->str, reg->index); + if (regarray->size > 1) { + RESET(fp); + fputs("..", fp); + YELLOW(fp); + fprintf(fp, "%" PRIu32, regarray->size + reg->index - 1); + } + fputs("]", fp); RESET(fp); } @@ -533,6 +536,8 @@ rogue_print_instr_group_io_sel(FILE *fp, const rogue_instr_group_io_sel *io_sel) if (rogue_ref_is_reg(&io_sel->dsts[i])) rogue_print_reg(fp, io_sel->dsts[i].reg); + else if (rogue_ref_is_regarray(&io_sel->dsts[i])) + rogue_print_regarray(fp, io_sel->dsts[i].regarray); else if (rogue_ref_is_io(&io_sel->dsts[i])) rogue_print_io(fp, io_sel->dsts[i].io); else @@ -678,16 +683,14 @@ void rogue_print_reg_writes(FILE *fp, const rogue_shader *shader) for (enum rogue_reg_class class = 0; class < ROGUE_REG_CLASS_COUNT; ++class) { rogue_foreach_reg (reg, shader, class) { + bool unused = true; + rogue_print_reg(fp, reg); fputs(":", fp); - if (list_is_empty(®->writes)) { - fputs(" \n", fp); - continue; - } - rogue_foreach_reg_write (write, reg) { assert(write->instr); + unused = false; fputs(" ", fp); rogue_print_instr_ref(fp, @@ -697,6 +700,44 @@ void rogue_print_reg_writes(FILE *fp, const rogue_shader *shader) shader->is_grouped); } + if (reg->regarray) { + rogue_foreach_regarray_write (write, reg->regarray) { + assert(write->instr); + unused = false; + + fputs(" ", fp); + rogue_print_instr_ref(fp, + write->instr, + false, + write->dst_index, + shader->is_grouped); + } + + rogue_foreach_subarray (subarray, reg->regarray) { + unsigned subarray_start = subarray->regs[0]->index; + unsigned subarray_end = subarray_start + subarray->size - 1; + if (reg->index < subarray_start || reg->index > subarray_end) + continue; + + rogue_foreach_regarray_write (write, subarray) { + assert(write->instr); + unused = false; + + fputs(" ", fp); + rogue_print_instr_ref(fp, + write->instr, + false, + write->dst_index, + shader->is_grouped); + } + } + } + + if (unused) { + fputs(" \n", fp); + continue; + } + fputs("\n", fp); } } @@ -709,16 +750,14 @@ void rogue_print_reg_uses(FILE *fp, const rogue_shader *shader) for (enum rogue_reg_class class = 0; class < ROGUE_REG_CLASS_COUNT; ++class) { rogue_foreach_reg (reg, shader, class) { + bool unused = true; + rogue_print_reg(fp, reg); fputs(":", fp); - if (list_is_empty(®->uses)) { - fputs(" \n", fp); - continue; - } - rogue_foreach_reg_use (use, reg) { assert(use->instr); + unused = false; fputs(" ", fp); rogue_print_instr_ref(fp, @@ -728,6 +767,44 @@ void rogue_print_reg_uses(FILE *fp, const rogue_shader *shader) shader->is_grouped); } + if (reg->regarray) { + rogue_foreach_regarray_use (use, reg->regarray) { + assert(use->instr); + unused = false; + + fputs(" ", fp); + rogue_print_instr_ref(fp, + use->instr, + false, + use->src_index, + shader->is_grouped); + } + + rogue_foreach_subarray (subarray, reg->regarray) { + unsigned subarray_start = subarray->regs[0]->index; + unsigned subarray_end = subarray_start + subarray->size - 1; + if (reg->index < subarray_start || reg->index > subarray_end) + continue; + + rogue_foreach_regarray_use (use, subarray) { + assert(use->instr); + unused = false; + + fputs(" ", fp); + rogue_print_instr_ref(fp, + use->instr, + false, + use->src_index, + shader->is_grouped); + } + } + } + + if (unused) { + fputs(" \n", fp); + continue; + } + fputs("\n", fp); } }