diff --git a/src/gallium/drivers/panfrost/pan_cmdstream.c b/src/gallium/drivers/panfrost/pan_cmdstream.c index fe1e9598faa..6f43f995c17 100644 --- a/src/gallium/drivers/panfrost/pan_cmdstream.c +++ b/src/gallium/drivers/panfrost/pan_cmdstream.c @@ -3608,7 +3608,7 @@ panfrost_launch_afbc_conv_shader(struct panfrost_batch *batch, void *cso, static void panfrost_afbc_size(struct panfrost_batch *batch, struct panfrost_resource *src, - struct panfrost_bo *metadata, unsigned offset, + struct panfrost_bo *layout, unsigned offset, unsigned level) { MESA_TRACE_FUNC(); @@ -3616,7 +3616,7 @@ panfrost_afbc_size(struct panfrost_batch *batch, struct panfrost_resource *src, struct pan_image_slice_layout *slice = &src->plane.layout.slices[level]; struct panfrost_afbc_size_info consts = { .src = src->plane.base + slice->offset_B, - .metadata = metadata->ptr.gpu + offset, + .layout = layout->ptr.gpu + offset, }; unsigned stride_sb = pan_afbc_stride_blocks(src->image.props.modifier, slice->afbc.header.row_stride_B); @@ -3626,7 +3626,7 @@ panfrost_afbc_size(struct panfrost_batch *batch, struct panfrost_resource *src, u_minify(src->image.props.extent_px.height, level)); panfrost_batch_read_rsrc(batch, src, PIPE_SHADER_COMPUTE); - panfrost_batch_write_bo(batch, metadata, PIPE_SHADER_COMPUTE); + panfrost_batch_write_bo(batch, layout, PIPE_SHADER_COMPUTE); LAUNCH_AFBC_CONV_SHADER(size, batch, src, consts, nr_sblocks); } @@ -3635,7 +3635,7 @@ static void panfrost_afbc_pack(struct panfrost_batch *batch, struct panfrost_resource *src, struct panfrost_bo *dst, struct pan_image_slice_layout *dst_slice, - struct panfrost_bo *metadata, unsigned metadata_offset_B, + struct panfrost_bo *layout, unsigned layout_offset_B, unsigned level) { MESA_TRACE_FUNC(); @@ -3653,7 +3653,7 @@ panfrost_afbc_pack(struct panfrost_batch *batch, struct panfrost_resource *src, struct panfrost_afbc_pack_info consts = { .src = src->plane.base + src_slice->offset_B, .dst = dst->ptr.gpu + dst_slice->offset_B, - .metadata = metadata->ptr.gpu + metadata_offset_B, + .layout = layout->ptr.gpu + layout_offset_B, .header_size = pan_afbc_body_offset(dev->arch, src->image.props.modifier, src_slice->afbc.header.surface_size_B), @@ -3663,7 +3663,7 @@ panfrost_afbc_pack(struct panfrost_batch *batch, struct panfrost_resource *src, panfrost_batch_read_rsrc(batch, src, PIPE_SHADER_COMPUTE); panfrost_batch_write_bo(batch, dst, PIPE_SHADER_COMPUTE); - panfrost_batch_add_bo(batch, metadata, PIPE_SHADER_COMPUTE); + panfrost_batch_add_bo(batch, layout, PIPE_SHADER_COMPUTE); LAUNCH_AFBC_CONV_SHADER(pack, batch, src, consts, nr_sblocks); } diff --git a/src/gallium/drivers/panfrost/pan_mod_conv_cso.c b/src/gallium/drivers/panfrost/pan_mod_conv_cso.c index 68ce77120f9..a17c53b27e5 100644 --- a/src/gallium/drivers/panfrost/pan_mod_conv_cso.c +++ b/src/gallium/drivers/panfrost/pan_mod_conv_cso.c @@ -125,20 +125,20 @@ get_superblock_size(nir_builder *b, unsigned arch, nir_def *hdr, } static nir_def * -get_packed_offset(nir_builder *b, nir_def *metadata, nir_def *idx, +get_packed_offset(nir_builder *b, nir_def *layout, nir_def *idx, nir_def **out_size) { - nir_def *metadata_offset = - nir_u2u64(b, nir_imul_imm(b, idx, sizeof(struct pan_afbc_block_info))); - nir_def *range_ptr = nir_iadd(b, metadata, metadata_offset); + nir_def *layout_offset = + nir_u2u64(b, nir_imul_imm(b, idx, sizeof(struct pan_afbc_payload_extent))); + nir_def *range_ptr = nir_iadd(b, layout, layout_offset); nir_def *entry = nir_load_global(b, range_ptr, 4, - sizeof(struct pan_afbc_block_info) / 4, 32); + sizeof(struct pan_afbc_payload_extent) / 4, 32); nir_def *offset = - nir_channel(b, entry, offsetof(struct pan_afbc_block_info, offset) / 4); + nir_channel(b, entry, offsetof(struct pan_afbc_payload_extent, offset) / 4); if (out_size) *out_size = - nir_channel(b, entry, offsetof(struct pan_afbc_block_info, size) / 4); + nir_channel(b, entry, offsetof(struct pan_afbc_payload_extent, size) / 4); return nir_u2u64(b, offset); } @@ -147,14 +147,14 @@ get_packed_offset(nir_builder *b, nir_def *metadata, nir_def *idx, static void copy_superblock(nir_builder *b, nir_def *dst, nir_def *hdr_sz, nir_def *src, - nir_def *metadata, nir_def *idx, unsigned align) + nir_def *layout, nir_def *idx, unsigned align) { nir_def *hdr = read_afbc_header(b, src, idx); nir_def *src_body_base_ptr = nir_u2u64(b, nir_channel(b, hdr, 0)); nir_def *src_bodyptr = nir_iadd(b, src, src_body_base_ptr); nir_def *size; - nir_def *dst_offset = get_packed_offset(b, metadata, idx, &size); + nir_def *dst_offset = get_packed_offset(b, layout, idx, &size); nir_def *dst_body_base_ptr = nir_iadd(b, dst_offset, hdr_sz); nir_def *dst_bodyptr = nir_iadd(b, dst, dst_body_base_ptr); @@ -208,7 +208,7 @@ panfrost_create_afbc_size_shader(struct panfrost_screen *screen, nir_def *coord = nir_load_global_invocation_id(&b, 32); nir_def *block_idx = nir_channel(&b, coord, 0); nir_def *src = panfrost_afbc_size_get_info_field(&b, src); - nir_def *metadata = panfrost_afbc_size_get_info_field(&b, metadata); + nir_def *layout = panfrost_afbc_size_get_info_field(&b, layout); nir_def *uncompressed_size = nir_imm_int(&b, 4 * 4 * bpp / 8); /* bytes */ nir_def *hdr = read_afbc_header(&b, src, block_idx); @@ -219,9 +219,9 @@ panfrost_create_afbc_size_shader(struct panfrost_screen *screen, nir_def *offset = nir_u2u64( &b, nir_iadd(&b, - nir_imul_imm(&b, block_idx, sizeof(struct pan_afbc_block_info)), - nir_imm_int(&b, offsetof(struct pan_afbc_block_info, size)))); - nir_store_global(&b, nir_iadd(&b, metadata, offset), 4, size, 0x1); + nir_imul_imm(&b, block_idx, sizeof(struct pan_afbc_payload_extent)), + nir_imm_int(&b, offsetof(struct pan_afbc_payload_extent, size)))); + nir_store_global(&b, nir_iadd(&b, layout, offset), 4, size, 0x1); return b.shader; } @@ -247,9 +247,9 @@ panfrost_create_afbc_pack_shader(struct panfrost_screen *screen, nir_def *dst = panfrost_afbc_pack_get_info_field(&b, dst); nir_def *header_size = nir_u2u64(&b, panfrost_afbc_pack_get_info_field(&b, header_size)); - nir_def *metadata = panfrost_afbc_pack_get_info_field(&b, metadata); + nir_def *layout = panfrost_afbc_pack_get_info_field(&b, layout); - copy_superblock(&b, dst, header_size, src, metadata, idx, align); + copy_superblock(&b, dst, header_size, src, layout, idx, align); return b.shader; } diff --git a/src/gallium/drivers/panfrost/pan_mod_conv_cso.h b/src/gallium/drivers/panfrost/pan_mod_conv_cso.h index 9566abc3bd6..cb7bcbaea4b 100644 --- a/src/gallium/drivers/panfrost/pan_mod_conv_cso.h +++ b/src/gallium/drivers/panfrost/pan_mod_conv_cso.h @@ -67,20 +67,20 @@ struct pan_mod_convert_shaders { pthread_mutex_t lock; }; -struct pan_afbc_block_info { +struct pan_afbc_payload_extent { uint32_t size; uint32_t offset; }; struct panfrost_afbc_size_info { uint64_t src; - uint64_t metadata; + uint64_t layout; } PACKED; struct panfrost_afbc_pack_info { uint64_t src; uint64_t dst; - uint64_t metadata; + uint64_t layout; uint32_t header_size; uint32_t src_stride; uint32_t dst_stride; diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c index e641d3baabd..a252c642db7 100644 --- a/src/gallium/drivers/panfrost/pan_resource.c +++ b/src/gallium/drivers/panfrost/pan_resource.c @@ -1956,43 +1956,45 @@ panfrost_should_linear_convert(struct panfrost_context *ctx, } static struct panfrost_bo * -panfrost_get_afbc_superblock_sizes(struct panfrost_context *ctx, - struct panfrost_resource *rsrc, - unsigned first_level, unsigned last_level, - unsigned *out_offsets) +panfrost_get_afbc_payload_sizes(struct panfrost_context *ctx, + struct panfrost_resource *rsrc, + unsigned first_level, unsigned last_level, + unsigned *out_offsets) { struct panfrost_screen *screen = pan_screen(ctx->base.screen); struct panfrost_device *dev = pan_device(ctx->base.screen); struct panfrost_batch *batch; struct panfrost_bo *bo; - unsigned metadata_size = 0; + unsigned layout_size = 0; for (int level = first_level; level <= last_level; ++level) { struct pan_image_slice_layout *slice = &rsrc->plane.layout.slices[level]; - unsigned stride_sb = pan_afbc_stride_blocks( + unsigned stride_blocks = pan_afbc_stride_blocks( rsrc->image.props.modifier, slice->afbc.header.row_stride_B); - unsigned nr_sblocks = - stride_sb * pan_afbc_height_blocks( + unsigned nr_blocks = + stride_blocks * pan_afbc_height_blocks( rsrc->image.props.modifier, u_minify(rsrc->image.props.extent_px.height, level)); - unsigned sz = nr_sblocks * sizeof(struct pan_afbc_block_info); - out_offsets[level - first_level] = metadata_size; - metadata_size += sz; + unsigned sz = nr_blocks * sizeof(struct pan_afbc_payload_extent); + out_offsets[level - first_level] = layout_size; + layout_size += sz; } - bo = panfrost_bo_create(dev, metadata_size, 0, "AFBC superblock sizes"); + bo = panfrost_bo_create(dev, layout_size, 0, "AFBC-P payload layout"); if (!bo) return NULL; - panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC before size flush"); - batch = panfrost_get_fresh_batch_for_fbo(ctx, "AFBC superblock sizes"); + panfrost_flush_batches_accessing_rsrc(ctx, rsrc, + "AFBC-P payload layout pre-flush"); + batch = panfrost_get_fresh_batch_for_fbo(ctx, "AFBC-P payload layout"); for (int level = first_level; level <= last_level; ++level) { unsigned offset = out_offsets[level - first_level]; screen->vtbl.afbc_size(batch, rsrc, bo, offset, level); } - panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "AFBC after size flush"); + panfrost_flush_batches_accessing_rsrc(ctx, rsrc, + "AFBC-P payload layout post-flush"); return bo; } @@ -2005,8 +2007,8 @@ panfrost_pack_afbc(struct panfrost_context *ctx, struct panfrost_screen *screen = pan_screen(ctx->base.screen); struct panfrost_device *dev = pan_device(ctx->base.screen); - struct panfrost_bo *metadata_bo; - unsigned metadata_offsets[PIPE_MAX_TEXTURE_LEVELS]; + struct panfrost_bo *layout_bo; + unsigned layout_offsets[PIPE_MAX_TEXTURE_LEVELS]; assert(prsrc->base.array_size == 1); @@ -2022,21 +2024,22 @@ panfrost_pack_afbc(struct panfrost_context *ctx, return; } - metadata_bo = panfrost_get_afbc_superblock_sizes(ctx, prsrc, 0, last_level, - metadata_offsets); + layout_bo = panfrost_get_afbc_payload_sizes(ctx, prsrc, 0, last_level, + layout_offsets); - if (!metadata_bo) { - mesa_loge("panfrost_pack_afbc: failed to get afbc superblock sizes"); + if (!layout_bo) { + mesa_loge("panfrost_pack_afbc: failed to create AFBC-P payload layout " + "BO"); return; } - panfrost_bo_wait(metadata_bo, INT64_MAX, false); + panfrost_bo_wait(layout_bo, INT64_MAX, false); for (unsigned level = 0; level <= last_level; ++level) { struct pan_image_slice_layout *src_slice = &prsrc->plane.layout.slices[level]; struct pan_image_slice_layout *dst_slice = &slice_infos[level]; - unsigned nr_sblocks_total = + unsigned nr_blocks_total = pan_afbc_stride_blocks( modifier, src_slice->afbc.header.row_stride_B) * pan_afbc_height_blocks( @@ -2044,30 +2047,30 @@ panfrost_pack_afbc(struct panfrost_context *ctx, uint32_t body_offset_B = pan_afbc_body_offset( dev->arch, modifier, src_slice->afbc.header.surface_size_B); uint32_t offset = 0; - struct pan_afbc_block_info *meta = - metadata_bo->ptr.cpu + metadata_offsets[level]; + struct pan_afbc_payload_extent *layout = + layout_bo->ptr.cpu + layout_offsets[level]; - /* Stack allocated chunk used to copy AFBC block info from non-cacheable - * memory to cacheable memory. Each iteration of the offset computation - * loop below otherwise forces a flush of the write combining buffer - * because of the 32-bit read interleaved with the 32-bit write. A tile - * is composed of 8x8 header blocks. A chunk is made of 16 tiles so that - * at most 8 kB can be copied at each iteration (smaller values tend to - * increase latency). */ - alignas(16) struct pan_afbc_block_info meta_chunk[64 * 16]; - unsigned nr_sblocks_per_chunk = ARRAY_SIZE(meta_chunk); + /* Stack allocated chunk used to copy the AFBC-P payload layout from + * non-cacheable memory to cacheable memory. Each iteration of the + * offset computation loop below otherwise forces a flush of the write + * combining buffer because of the 32-bit read interleaved with the + * 32-bit write. A tile is composed of 8x8 header blocks. A chunk is + * made of 16 tiles so that at most 8 kB can be copied at each iteration + * (smaller values tend to increase latency). */ + alignas(16) struct pan_afbc_payload_extent layout_chunk[64 * 16]; + unsigned nr_blocks_per_chunk = ARRAY_SIZE(layout_chunk); - for (unsigned i = 0; i < nr_sblocks_total; i += nr_sblocks_per_chunk) { - unsigned nr_sblocks = - MIN2(nr_sblocks_per_chunk, nr_sblocks_total - i); + for (unsigned i = 0; i < nr_blocks_total; i += nr_blocks_per_chunk) { + unsigned nr_blocks = + MIN2(nr_blocks_per_chunk, nr_blocks_total - i); util_streaming_load_memcpy( - meta_chunk, &meta[i], - nr_sblocks * sizeof(struct pan_afbc_block_info)); + layout_chunk, &layout[i], + nr_blocks * sizeof(struct pan_afbc_payload_extent)); - for (unsigned j = 0; j < nr_sblocks; j++) { - meta[i + j].offset = offset; - offset += meta_chunk[j].size; + for (unsigned j = 0; j < nr_blocks; j++) { + layout[i + j].offset = offset; + offset += layout_chunk[j].size; } } @@ -2093,7 +2096,7 @@ panfrost_pack_afbc(struct panfrost_context *ctx, unsigned ratio = 100 * new_size / old_size; if (ratio > screen->max_afbc_packing_ratio) { - panfrost_bo_unreference(metadata_bo); + panfrost_bo_unreference(layout_bo); return; } perf_debug(ctx, "%i%%: %i KB -> %i KB\n", ratio, old_size / 1024, @@ -2113,8 +2116,8 @@ panfrost_pack_afbc(struct panfrost_context *ctx, panfrost_bo_create(dev, new_size, 0, new_label); if (!dst) { - mesa_loge("panfrost_pack_afbc: failed to get afbc superblock sizes"); - panfrost_bo_unreference(metadata_bo); + mesa_loge("panfrost_pack_afbc: failed to create AFBC-P BO"); + panfrost_bo_unreference(layout_bo); free(new_label); return; } @@ -2124,8 +2127,8 @@ panfrost_pack_afbc(struct panfrost_context *ctx, for (unsigned level = 0; level <= last_level; ++level) { struct pan_image_slice_layout *slice = &slice_infos[level]; - screen->vtbl.afbc_pack(batch, prsrc, dst, slice, metadata_bo, - metadata_offsets[level], level); + screen->vtbl.afbc_pack(batch, prsrc, dst, slice, layout_bo, + layout_offsets[level], level); prsrc->plane.layout.slices[level] = *slice; } prsrc->plane.layout.array_stride_B = new_size; @@ -2146,7 +2149,7 @@ panfrost_pack_afbc(struct panfrost_context *ctx, prsrc->plane.base = dst->ptr.gpu; prsrc->image.props.crc = false; prsrc->valid.crc = false; - panfrost_bo_unreference(metadata_bo); + panfrost_bo_unreference(layout_bo); } static void