diff --git a/src/gallium/drivers/panfrost/pan_blit.c b/src/gallium/drivers/panfrost/pan_blit.c index 45bf1befd42..b396d59fd7f 100644 --- a/src/gallium/drivers/panfrost/pan_blit.c +++ b/src/gallium/drivers/panfrost/pan_blit.c @@ -107,11 +107,11 @@ panfrost_blit(struct pipe_context *pipe, const struct pipe_blit_info *info) /* Legalize here because it could trigger a recursive blit otherwise */ struct panfrost_resource *src = pan_resource(info->src.resource); enum pipe_format src_view_format = util_format_linear(info->src.format); - pan_legalize_afbc_format(ctx, src, src_view_format, false, false); + pan_legalize_format(ctx, src, src_view_format, false, false); struct panfrost_resource *dst = pan_resource(info->dst.resource); enum pipe_format dst_view_format = util_format_linear(info->dst.format); - pan_legalize_afbc_format(ctx, dst, dst_view_format, true, false); + pan_legalize_format(ctx, dst, dst_view_format, true, false); panfrost_blit_no_afbc_legalization(pipe, info); } diff --git a/src/gallium/drivers/panfrost/pan_cmdstream.c b/src/gallium/drivers/panfrost/pan_cmdstream.c index 54f6b706422..e4ce73263b9 100644 --- a/src/gallium/drivers/panfrost/pan_cmdstream.c +++ b/src/gallium/drivers/panfrost/pan_cmdstream.c @@ -3442,8 +3442,8 @@ panfrost_create_sampler_view(struct pipe_context *pctx, struct panfrost_sampler_view *so = rzalloc(pctx, struct panfrost_sampler_view); - pan_legalize_afbc_format(ctx, pan_resource(texture), template->format, - false, false); + pan_legalize_format(ctx, pan_resource(texture), template->format, false, + false); pipe_reference(NULL, &texture->reference); diff --git a/src/gallium/drivers/panfrost/pan_context.c b/src/gallium/drivers/panfrost/pan_context.c index 11dabe199fa..f6d84fc950c 100644 --- a/src/gallium/drivers/panfrost/pan_context.c +++ b/src/gallium/drivers/panfrost/pan_context.c @@ -280,9 +280,10 @@ panfrost_set_shader_images(struct pipe_context *pctx, struct panfrost_resource *rsrc = pan_resource(image->resource); - /* Images don't work with AFBC, since they require pixel-level granularity - */ - if (drm_is_afbc(rsrc->image.layout.modifier)) { + /* Images don't work with AFBC/AFRC, since they require pixel-level + * granularity */ + if (drm_is_afbc(rsrc->image.layout.modifier) || + drm_is_afrc(rsrc->image.layout.modifier)) { pan_resource_modifier_convert( ctx, rsrc, DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, true, "Shader image"); diff --git a/src/gallium/drivers/panfrost/pan_job.c b/src/gallium/drivers/panfrost/pan_job.c index 9cc2f77fb87..4733cd5d680 100644 --- a/src/gallium/drivers/panfrost/pan_job.c +++ b/src/gallium/drivers/panfrost/pan_job.c @@ -65,7 +65,7 @@ panfrost_batch_add_surface(struct panfrost_batch *batch, { if (surf) { struct panfrost_resource *rsrc = pan_resource(surf->texture); - pan_legalize_afbc_format(batch->ctx, rsrc, surf->format, true, false); + pan_legalize_format(batch->ctx, rsrc, surf->format, true, false); panfrost_batch_write_rsrc(batch, rsrc, PIPE_SHADER_FRAGMENT); } } diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c index d90baa92535..22cc89cf294 100644 --- a/src/gallium/drivers/panfrost/pan_resource.c +++ b/src/gallium/drivers/panfrost/pan_resource.c @@ -1120,8 +1120,9 @@ panfrost_ptr_map(struct pipe_context *pctx, struct pipe_resource *resource, if (usage & PIPE_MAP_WRITE) rsrc->constant_stencil = false; - /* We don't have s/w routines for AFBC, so use a staging texture */ - if (drm_is_afbc(rsrc->image.layout.modifier)) { + /* We don't have s/w routines for AFBC/AFRC, so use a staging texture */ + if (drm_is_afbc(rsrc->image.layout.modifier) || + drm_is_afrc(rsrc->image.layout.modifier)) { struct panfrost_resource *staging = pan_alloc_staging(ctx, rsrc, level, box); assert(staging); @@ -1147,7 +1148,7 @@ panfrost_ptr_map(struct pipe_context *pctx, struct pipe_resource *resource, if ((usage & PIPE_MAP_READ) && (valid || panfrost_any_batch_writes_rsrc(ctx, rsrc))) { pan_blit_to_staging(pctx, transfer); - panfrost_flush_writer(ctx, staging, "AFBC read staging blit"); + panfrost_flush_writer(ctx, staging, "AFBC/AFRC tex read staging blit"); panfrost_bo_wait(staging->bo, INT64_MAX, false); } @@ -1337,7 +1338,7 @@ pan_resource_modifier_convert(struct panfrost_context *ctx, }; /* data_valid is not valid until flushed */ - panfrost_flush_writer(ctx, rsrc, "AFBC decompressing blit"); + panfrost_flush_writer(ctx, rsrc, "AFBC/AFRC decompressing blit"); for (int i = 0; i <= rsrc->base.last_level; i++) { if (BITSET_TEST(rsrc->valid.data, i)) { @@ -1355,7 +1356,7 @@ pan_resource_modifier_convert(struct panfrost_context *ctx, /* we lose track of tmp_rsrc after this point, and the BO migration * (from tmp_rsrc to rsrc) doesn't transfer the last_writer to rsrc */ - panfrost_flush_writer(ctx, tmp_rsrc, "AFBC decompressing blit"); + panfrost_flush_writer(ctx, tmp_rsrc, "AFBC/AFRC decompressing blit"); } panfrost_bo_unreference(rsrc->bo); @@ -1373,32 +1374,51 @@ pan_resource_modifier_convert(struct panfrost_context *ctx, pipe_resource_reference(&tmp_prsrc, NULL); } -/* Validate that an AFBC resource may be used as a particular format. If it may - * not, decompress it on the fly. Failure to do so can produce wrong results or - * invalid data faults when sampling or rendering to AFBC */ +/* Validate that an AFBC/AFRC resource may be used as a particular format. If it + * may not, decompress it on the fly. Failure to do so can produce wrong results + * or invalid data faults when sampling or rendering to AFBC */ void -pan_legalize_afbc_format(struct panfrost_context *ctx, - struct panfrost_resource *rsrc, - enum pipe_format format, bool write, bool discard) +pan_legalize_format(struct panfrost_context *ctx, + struct panfrost_resource *rsrc, enum pipe_format format, + bool write, bool discard) { struct panfrost_device *dev = pan_device(ctx->base.screen); + enum pipe_format old_format = rsrc->base.format; + enum pipe_format new_format = format; + bool compatible = true; - if (!drm_is_afbc(rsrc->image.layout.modifier)) + if (!drm_is_afbc(rsrc->image.layout.modifier) && + !drm_is_afrc(rsrc->image.layout.modifier)) return; - if (panfrost_afbc_format(dev->arch, rsrc->base.format) != - panfrost_afbc_format(dev->arch, format)) { + if (drm_is_afbc(rsrc->image.layout.modifier)) { + compatible = (panfrost_afbc_format(dev->arch, old_format) == + panfrost_afbc_format(dev->arch, new_format)); + } else if (drm_is_afrc(rsrc->image.layout.modifier)) { + struct pan_afrc_format_info old_info = + panfrost_afrc_get_format_info(old_format); + struct pan_afrc_format_info new_info = + panfrost_afrc_get_format_info(new_format); + compatible = !memcmp(&old_info, &new_info, sizeof(old_info)); + } + + if (!compatible) { pan_resource_modifier_convert( ctx, rsrc, DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, !discard, - "Reinterpreting AFBC surface as incompatible format"); + drm_is_afbc(rsrc->image.layout.modifier) + ? "Reinterpreting AFBC surface as incompatible format" + : "Reinterpreting AFRC surface as incompatible format"); return; } - if (write && (rsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPARSE) == 0) + /* Can't write to AFBC-P resources */ + if (write && drm_is_afbc(rsrc->image.layout.modifier) && + (rsrc->image.layout.modifier & AFBC_FORMAT_MOD_SPARSE) == 0) { pan_resource_modifier_convert( ctx, rsrc, rsrc->image.layout.modifier | AFBC_FORMAT_MOD_SPARSE, !discard, "Legalizing resource to allow writing"); + } } static bool @@ -1588,10 +1608,10 @@ panfrost_ptr_unmap(struct pipe_context *pctx, struct pipe_transfer *transfer) if (transfer->usage & PIPE_MAP_WRITE) prsrc->valid.crc = false; - /* AFBC will use a staging resource. `initialized` will be set when the - * fragment job is created; this is deferred to prevent useless surface + /* AFBC/AFRC will use a staging resource. `initialized` will be set when + * the fragment job is created; this is deferred to prevent useless surface * reloads that can cascade into DATA_INVALID_FAULTs due to reading - * malformed AFBC data if uninitialized */ + * malformed AFBC/AFRC data if uninitialized */ if (trans->staging.rsrc) { if (transfer->usage & PIPE_MAP_WRITE) { @@ -1608,8 +1628,8 @@ panfrost_ptr_unmap(struct pipe_context *pctx, struct pipe_transfer *transfer) } else { bool discard = panfrost_can_discard(&prsrc->base, &transfer->box, transfer->usage); - pan_legalize_afbc_format(ctx, prsrc, prsrc->image.layout.format, - true, discard); + pan_legalize_format(ctx, prsrc, prsrc->image.layout.format, true, + discard); pan_blit_from_staging(pctx, trans); panfrost_flush_batches_accessing_rsrc( ctx, pan_resource(trans->staging.rsrc), diff --git a/src/gallium/drivers/panfrost/pan_resource.h b/src/gallium/drivers/panfrost/pan_resource.h index a750f27c106..e63bcc9244d 100644 --- a/src/gallium/drivers/panfrost/pan_resource.h +++ b/src/gallium/drivers/panfrost/pan_resource.h @@ -194,10 +194,10 @@ void pan_resource_modifier_convert(struct panfrost_context *ctx, uint64_t modifier, bool copy_resource, const char *reason); -void pan_legalize_afbc_format(struct panfrost_context *ctx, - struct panfrost_resource *rsrc, - enum pipe_format format, bool write, - bool discard); +void pan_legalize_format(struct panfrost_context *ctx, + struct panfrost_resource *rsrc, + enum pipe_format format, bool write, + bool discard); void pan_dump_resource(struct panfrost_context *ctx, struct panfrost_resource *rsc); diff --git a/src/gallium/drivers/panfrost/pan_screen.c b/src/gallium/drivers/panfrost/pan_screen.c index 13e4c6ba83c..0f2986dfb31 100644 --- a/src/gallium/drivers/panfrost/pan_screen.c +++ b/src/gallium/drivers/panfrost/pan_screen.c @@ -613,6 +613,7 @@ panfrost_walk_dmabuf_modifiers(struct pipe_screen *screen, dev->has_afbc && panfrost_format_supports_afbc(dev->arch, format); bool ytr = panfrost_afbc_can_ytr(format); bool tiled_afbc = panfrost_afbc_can_tile(dev->arch); + bool afrc = dev->has_afrc && panfrost_format_supports_afrc(format); unsigned count = 0; @@ -626,6 +627,9 @@ panfrost_walk_dmabuf_modifiers(struct pipe_screen *screen, if ((pan_best_modifiers[i] & AFBC_FORMAT_MOD_TILED) && !tiled_afbc) continue; + if (drm_is_afrc(pan_best_modifiers[i]) && !afrc) + continue; + if (test_modifier != DRM_FORMAT_MOD_INVALID && test_modifier != pan_best_modifiers[i]) continue; diff --git a/src/panfrost/lib/pan_afrc.c b/src/panfrost/lib/pan_afrc.c index 473a3d0d26d..14bc2946746 100644 --- a/src/panfrost/lib/pan_afrc.c +++ b/src/panfrost/lib/pan_afrc.c @@ -80,3 +80,15 @@ panfrost_afrc_get_format_info(enum pipe_format format) info.num_comps = util_format_get_nr_components(format); return info; } + +bool +panfrost_format_supports_afrc(enum pipe_format format) +{ + const struct util_format_description *desc = util_format_description(format); + int c = util_format_get_first_non_void_channel(desc->format); + + if (c == -1) + return false; + + return desc->is_array && desc->channel[c].size == 8; +} diff --git a/src/panfrost/lib/pan_layout.c b/src/panfrost/lib/pan_layout.c index ab35f0b2a9e..6862057d81a 100644 --- a/src/panfrost/lib/pan_layout.c +++ b/src/panfrost/lib/pan_layout.c @@ -32,6 +32,7 @@ * List of supported modifiers, in descending order of preference. AFBC is * faster than u-interleaved tiling which is faster than linear. Within AFBC, * enabling the YUV-like transform is typically a win where possible. + * AFRC is only used if explicitely asked for (only for RGB formats). */ uint64_t pan_best_modifiers[PAN_MODIFIER_COUNT] = { DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | @@ -49,7 +50,24 @@ uint64_t pan_best_modifiers[PAN_MODIFIER_COUNT] = { AFBC_FORMAT_MOD_SPARSE), DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, - DRM_FORMAT_MOD_LINEAR}; + DRM_FORMAT_MOD_LINEAR, + + DRM_FORMAT_MOD_ARM_AFRC( + AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_16)), + DRM_FORMAT_MOD_ARM_AFRC( + AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_24)), + DRM_FORMAT_MOD_ARM_AFRC( + AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_32)), + DRM_FORMAT_MOD_ARM_AFRC( + AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_16) | + AFRC_FORMAT_MOD_LAYOUT_SCAN), + DRM_FORMAT_MOD_ARM_AFRC( + AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_24) | + AFRC_FORMAT_MOD_LAYOUT_SCAN), + DRM_FORMAT_MOD_ARM_AFRC( + AFRC_FORMAT_MOD_CU_SIZE_P0(AFRC_FORMAT_MOD_CU_SIZE_32) | + AFRC_FORMAT_MOD_LAYOUT_SCAN), +}; /* Table of AFBC superblock sizes */ static const struct pan_block_size afbc_superblock_sizes[] = { @@ -126,6 +144,88 @@ panfrost_afrc_is_scan(uint64_t modifier) return modifier & AFRC_FORMAT_MOD_LAYOUT_SCAN; } +struct pan_block_size +panfrost_afrc_clump_size(enum pipe_format format, bool scan) +{ + struct pan_afrc_format_info finfo = panfrost_afrc_get_format_info(format); + + switch (finfo.num_comps) { + case 1: + return scan ? (struct pan_block_size){16, 4} + : (struct pan_block_size){8, 8}; + case 2: + return (struct pan_block_size){8, 4}; + case 3: + case 4: + return (struct pan_block_size){4, 4}; + default: + assert(0); + return (struct pan_block_size){0, 0}; + } +} + +static struct pan_block_size +panfrost_afrc_layout_size(uint64_t modifier) +{ + if (panfrost_afrc_is_scan(modifier)) + return (struct pan_block_size){16, 4}; + else + return (struct pan_block_size){8, 8}; +} + +struct pan_block_size +panfrost_afrc_tile_size(enum pipe_format format, uint64_t modifier) +{ + bool scan = panfrost_afrc_is_scan(modifier); + struct pan_block_size clump_sz = panfrost_afrc_clump_size(format, scan); + struct pan_block_size layout_sz = panfrost_afrc_layout_size(modifier); + + return (struct pan_block_size){clump_sz.width * layout_sz.width, + clump_sz.height * layout_sz.height}; +} + +unsigned +panfrost_afrc_block_size_from_modifier(uint64_t modifier) +{ + switch (modifier & AFRC_FORMAT_MOD_CU_SIZE_MASK) { + case AFRC_FORMAT_MOD_CU_SIZE_16: + return 16; + case AFRC_FORMAT_MOD_CU_SIZE_24: + return 24; + case AFRC_FORMAT_MOD_CU_SIZE_32: + return 32; + default: + unreachable("invalid coding unit size flag in modifier"); + }; +} + +static unsigned +panfrost_afrc_buffer_alignment_from_modifier(uint64_t modifier) +{ + switch (modifier & AFRC_FORMAT_MOD_CU_SIZE_MASK) { + case AFRC_FORMAT_MOD_CU_SIZE_16: + return 1024; + case AFRC_FORMAT_MOD_CU_SIZE_24: + return 512; + case AFRC_FORMAT_MOD_CU_SIZE_32: + return 2048; + default: + unreachable("invalid coding unit size flag in modifier"); + }; +} + +/* + * Determine the number of bytes between rows of paging tiles in an AFRC image + */ +uint32_t +pan_afrc_row_stride(enum pipe_format format, uint64_t modifier, uint32_t width) +{ + struct pan_block_size tile_size = panfrost_afrc_tile_size(format, modifier); + unsigned block_size = panfrost_afrc_block_size_from_modifier(modifier); + + return (width / tile_size.width) * block_size * AFRC_CLUMPS_PER_TILE; +} + /* * Given a format, determine the tile size used for u-interleaving. For formats * that are already block compressed, this is 4x4. For all other formats, this @@ -142,8 +242,8 @@ panfrost_u_interleaved_tile_size(enum pipe_format format) /* * Determine the block size used for interleaving. For u-interleaving, this is - * the tile size. For AFBC, this is the superblock size. For linear textures, - * this is trivially 1x1. + * the tile size. For AFBC, this is the superblock size. For AFRC, this is the + * paging tile size. For linear textures, this is trivially 1x1. */ struct pan_block_size panfrost_block_size(uint64_t modifier, enum pipe_format format) @@ -152,6 +252,8 @@ panfrost_block_size(uint64_t modifier, enum pipe_format format) return panfrost_u_interleaved_tile_size(format); else if (drm_is_afbc(modifier)) return panfrost_afbc_superblock_size(modifier); + else if (drm_is_afrc(modifier)) + return panfrost_afrc_tile_size(format, modifier); else return (struct pan_block_size){1, 1}; } @@ -216,11 +318,14 @@ pan_afbc_body_align(uint64_t modifier) } static inline unsigned -format_minimum_alignment(unsigned arch, enum pipe_format format, bool afbc) +format_minimum_alignment(unsigned arch, enum pipe_format format, uint64_t mod) { - if (afbc) + if (drm_is_afbc(mod)) return 16; + if (drm_is_afrc(mod)) + return panfrost_afrc_buffer_alignment_from_modifier(mod); + if (arch < 7) return 64; @@ -282,6 +387,11 @@ panfrost_get_legacy_stride(const struct pan_image_layout *layout, width = ALIGN_POT(width, alignment); return width * util_format_get_blocksize(layout->format); + } else if (drm_is_afrc(layout->modifier)) { + struct pan_block_size tile_size = + panfrost_afrc_tile_size(layout->format, layout->modifier); + + return row_stride / tile_size.height; } else { return row_stride / block_size.height; } @@ -297,6 +407,11 @@ panfrost_from_legacy_stride(unsigned legacy_stride, enum pipe_format format, unsigned width = legacy_stride / util_format_get_blocksize(format); return pan_afbc_row_stride(modifier, width); + } else if (drm_is_afrc(modifier)) { + struct pan_block_size tile_size = + panfrost_afrc_tile_size(format, modifier); + + return legacy_stride * tile_size.height; } else { return legacy_stride * block_size.height; } @@ -327,7 +442,9 @@ pan_image_layout_init(unsigned arch, struct pan_image_layout *layout, return false; bool afbc = drm_is_afbc(layout->modifier); - int align_req = format_minimum_alignment(arch, layout->format, afbc); + bool afrc = drm_is_afrc(layout->modifier); + int align_req = + format_minimum_alignment(arch, layout->format, layout->modifier); /* Mandate alignment */ if (explicit_layout) { @@ -384,6 +501,7 @@ pan_image_layout_init(unsigned arch, struct pan_image_layout *layout, ALIGN_POT(util_format_get_nblocksx(layout->format, width), align_w); unsigned effective_height = ALIGN_POT(util_format_get_nblocksy(layout->format, height), align_h); + unsigned row_stride; /* Align levels to cache-line as a performance improvement for * linear/tiled and as a requirement for AFBC */ @@ -392,14 +510,19 @@ pan_image_layout_init(unsigned arch, struct pan_image_layout *layout, slice->offset = offset; - unsigned row_stride = fmt_blocksize * effective_width * block_size.height; + if (afrc) { + row_stride = pan_afrc_row_stride(layout->format, layout->modifier, + effective_width); + } else { + row_stride = fmt_blocksize * effective_width * block_size.height; + } /* On v7+ row_stride and offset alignment requirement are equal */ if (arch >= 7) { row_stride = ALIGN_POT(row_stride, align_req); } - if (explicit_layout && !afbc) { + if (explicit_layout && !afbc && !afrc) { /* Make sure the explicit stride is valid */ if (explicit_layout->row_stride < row_stride) { mesa_loge("panfrost: rejecting image due to invalid row stride.\n"); diff --git a/src/panfrost/lib/pan_texture.h b/src/panfrost/lib/pan_texture.h index 3a0080c7567..1de1a7ab370 100644 --- a/src/panfrost/lib/pan_texture.h +++ b/src/panfrost/lib/pan_texture.h @@ -43,7 +43,7 @@ extern "C" { #endif -#define PAN_MODIFIER_COUNT 6 +#define PAN_MODIFIER_COUNT 12 extern uint64_t pan_best_modifiers[PAN_MODIFIER_COUNT]; struct pan_image_slice_layout { @@ -299,8 +299,21 @@ struct pan_afrc_format_info { struct pan_afrc_format_info panfrost_afrc_get_format_info(enum pipe_format format); +bool panfrost_format_supports_afrc(enum pipe_format format); + bool panfrost_afrc_is_scan(uint64_t modifier); +struct pan_block_size panfrost_afrc_clump_size(enum pipe_format format, + bool scan); + +struct pan_block_size panfrost_afrc_tile_size(enum pipe_format format, + uint64_t modifier); + +unsigned panfrost_afrc_block_size_from_modifier(uint64_t modifier); + +unsigned pan_afrc_row_stride(enum pipe_format format, uint64_t modifier, + uint32_t width); + struct pan_block_size panfrost_block_size(uint64_t modifier, enum pipe_format format);