diff --git a/src/gallium/drivers/panfrost/pan_resource.c b/src/gallium/drivers/panfrost/pan_resource.c index 1c819bd70dc..d727da2eb8f 100644 --- a/src/gallium/drivers/panfrost/pan_resource.c +++ b/src/gallium/drivers/panfrost/pan_resource.c @@ -106,30 +106,33 @@ panfrost_clear_render_target(struct pipe_context *pipe, static void panfrost_resource_init_image(struct panfrost_resource *rsc, + const struct pan_mod_handler *mod_handler, const struct pan_image_props *iprops, unsigned plane_idx) { - if (util_format_get_num_planes(iprops->format) == 1) { - rsc->image.props = *iprops; - rsc->image.planes[0] = &rsc->plane; - return; - } - - /* The resource props will be initialized when we hit the first plane. */ - if (plane_idx > 0) - return; - + rsc->image.mod_handler = mod_handler; rsc->image.props = *iprops; - for (struct panfrost_resource *plane = rsc; + rsc->image.planes[plane_idx] = &rsc->plane; + + /* The rest of the resource planes will be initialized when we hit the first + * plane. */ + if (plane_idx > 0 || util_format_get_num_planes(iprops->format) == 1) + return; + + plane_idx = 1; + for (struct panfrost_resource *plane = pan_resource(rsc->base.next); plane && plane_idx < ARRAY_SIZE(rsc->image.planes); plane = pan_resource(plane->base.next)) rsc->image.planes[plane_idx++] = &plane->plane; assert(plane_idx == util_format_get_num_planes(iprops->format)); + plane_idx = 1; for (struct panfrost_resource *plane = pan_resource(rsc->base.next); - plane; plane = pan_resource(plane->base.next)) - plane->image = rsc->image; + plane; plane = pan_resource(plane->base.next)) { + memcpy(plane->image.planes, rsc->image.planes, + plane_idx * sizeof(plane->image.planes[0])); + } } static bool @@ -340,15 +343,18 @@ panfrost_resource_from_handle(struct pipe_screen *pscreen, unsigned format_plane = util_format_get_num_planes(iprops.format) > 1 ? whandle->plane : 0; - bool valid = pan_image_layout_init(dev->arch, &iprops, format_plane, - &explicit_layout, &rsc->plane.layout); + const struct pan_mod_handler *mod_handler = + pan_mod_get_handler(dev->arch, iprops.modifier); + bool valid = + pan_image_layout_init(dev->arch, mod_handler, &iprops, format_plane, + &explicit_layout, &rsc->plane.layout); if (!valid) { panfrost_resource_destroy(pscreen, &rsc->base); return NULL; } - panfrost_resource_init_image(rsc, &iprops, whandle->plane); + panfrost_resource_init_image(rsc, mod_handler, &iprops, whandle->plane); int ret = panfrost_resource_import_bo(rsc, dev, whandle->handle); /* Sometimes an import can fail e.g. on an invalid buffer fd, out of @@ -432,9 +438,7 @@ panfrost_resource_get_handle(struct pipe_screen *pscreen, return false; } - handle->stride = pan_image_get_wsi_row_pitch( - &rsrc->image.props, handle->plane, - &rsrc->image.planes[handle->plane]->layout, 0); + handle->stride = pan_image_get_wsi_row_pitch(&rsrc->image, handle->plane, 0); handle->offset = pan_image_get_wsi_offset(&rsrc->image.planes[handle->plane]->layout, 0); @@ -472,8 +476,7 @@ panfrost_resource_get_param(struct pipe_screen *pscreen, switch (param) { case PIPE_RESOURCE_PARAM_STRIDE: - *value = pan_image_get_wsi_row_pitch(&rsrc->image.props, plane, - &rsrc->plane.layout, level); + *value = pan_image_get_wsi_row_pitch(&rsrc->image, plane, level); return true; case PIPE_RESOURCE_PARAM_OFFSET: *value = pan_image_get_wsi_offset(&rsrc->plane.layout, level); @@ -805,11 +808,14 @@ panfrost_resource_try_setup(struct pipe_screen *screen, * want the real bitrate and not DEFAULT */ pres->base.compression_rate = pan_afrc_get_rate(fmt, chosen_mod); - if (!pan_image_layout_init(dev->arch, &iprops, plane_idx, NULL, + const struct pan_mod_handler *mod_handler = + pan_mod_get_handler(dev->arch, iprops.modifier); + + if (!pan_image_layout_init(dev->arch, mod_handler, &iprops, plane_idx, NULL, &pres->plane.layout)) return false; - panfrost_resource_init_image(pres, &iprops, plane_idx); + panfrost_resource_init_image(pres, mod_handler, &iprops, plane_idx); return true; } @@ -1029,8 +1035,7 @@ panfrost_resource_create_with_modifier(struct pipe_screen *screen, * means you're working on WSI and so it's already too late for * you. I'm sorry. */ - unsigned stride = pan_image_get_wsi_row_pitch(&so->image.props, plane_idx, - &so->plane.layout, 0); + unsigned stride = pan_image_get_wsi_row_pitch(&so->image, plane_idx, 0); enum pipe_format plane_format = util_format_get_plane_format(template->format, plane_idx); unsigned width = stride / util_format_get_blocksize(plane_format); diff --git a/src/panfrost/lib/meson.build b/src/panfrost/lib/meson.build index 781c6768c94..17bbd8a54e3 100644 --- a/src/panfrost/lib/meson.build +++ b/src/panfrost/lib/meson.build @@ -28,6 +28,7 @@ foreach ver : ['4', '5', '6', '7', '9', '10', '12', '13'] [ 'pan_blend.c', 'pan_desc.c', + 'pan_mod.c', 'pan_texture.c', ], include_directories : [inc_include, inc_src, inc_panfrost], diff --git a/src/panfrost/lib/pan_desc.c b/src/panfrost/lib/pan_desc.c index 2f2ea6deb43..bc9af977f4e 100644 --- a/src/panfrost/lib/pan_desc.c +++ b/src/panfrost/lib/pan_desc.c @@ -233,9 +233,9 @@ get_afbc_att_mem_props(struct pan_image_plane_ref pref, unsigned mip_level, } } -static void -pan_emit_linear_s_attachment(const struct pan_fb_info *fb, - unsigned layer_or_z_slice, void *payload) +void +GENX(pan_emit_linear_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload) { const struct pan_image_view *s = fb->zs.view.s; @@ -250,9 +250,9 @@ pan_emit_linear_s_attachment(const struct pan_fb_info *fb, } } -static void -pan_emit_afbc_s_attachment(const struct pan_fb_info *fb, - unsigned layer_or_z_slice, void *payload) +void +GENX(pan_emit_afbc_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload) { assert(PAN_ARCH >= 9); @@ -272,9 +272,9 @@ pan_emit_afbc_s_attachment(const struct pan_fb_info *fb, #endif } -static void -pan_emit_u_tiled_s_attachment(const struct pan_fb_info *fb, - unsigned layer_or_z_slice, void *payload) +void +GENX(pan_emit_u_tiled_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload) { const struct pan_image_view *s = fb->zs.view.s; @@ -289,9 +289,9 @@ pan_emit_u_tiled_s_attachment(const struct pan_fb_info *fb, } } -static void -pan_emit_linear_zs_attachment(const struct pan_fb_info *fb, - unsigned layer_or_z_slice, void *payload) +void +GENX(pan_emit_linear_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload) { const struct pan_image_view *zs = fb->zs.view.zs; @@ -309,9 +309,9 @@ pan_emit_linear_zs_attachment(const struct pan_fb_info *fb, } } -static void -pan_emit_u_tiled_zs_attachment(const struct pan_fb_info *fb, - unsigned layer_or_z_slice, void *payload) +void +GENX(pan_emit_u_tiled_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload) { const struct pan_image_view *zs = fb->zs.view.zs; @@ -329,9 +329,9 @@ pan_emit_u_tiled_zs_attachment(const struct pan_fb_info *fb, } } -static void -pan_emit_afbc_zs_attachment(const struct pan_fb_info *fb, - unsigned layer_or_z_slice, void *payload) +void +GENX(pan_emit_afbc_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload) { const struct pan_image_view *zs = fb->zs.view.zs; const struct pan_image_plane_ref pref = pan_image_view_get_zs_plane(zs); @@ -394,40 +394,6 @@ pan_prepare_crc(const struct pan_fb_info *fb, int rt_crc, #endif } -static void -pan_emit_zs_att(const struct pan_fb_info *fb, unsigned layer_idx, - struct mali_zs_crc_extension_packed *desc) -{ - const struct pan_image_plane_ref pref = - pan_image_view_get_zs_plane(fb->zs.view.zs); - const uint64_t mod = pref.image->props.modifier; - - if (drm_is_afbc(mod)) { - pan_emit_afbc_zs_attachment(fb, layer_idx, desc); - } else if (mod == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - pan_emit_u_tiled_zs_attachment(fb, layer_idx, desc); - } else { - assert(mod == DRM_FORMAT_MOD_LINEAR); - pan_emit_linear_zs_attachment(fb, layer_idx, desc); - } -} - -static void -pan_emit_s_att(const struct pan_fb_info *fb, unsigned layer_idx, - struct mali_zs_crc_extension_packed *desc) -{ - const struct pan_image_plane_ref pref = - pan_image_view_get_s_plane(fb->zs.view.s); - const uint64_t mod = pref.image->props.modifier; - - if (mod == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - pan_emit_u_tiled_s_attachment(fb, layer_idx, desc); - } else { - assert(mod == DRM_FORMAT_MOD_LINEAR); - pan_emit_linear_s_attachment(fb, layer_idx, desc); - } -} - static void pan_emit_zs_crc_ext(const struct pan_fb_info *fb, unsigned layer_idx, int rt_crc, struct mali_zs_crc_extension_packed *zs_crc_ext) @@ -440,16 +406,24 @@ pan_emit_zs_crc_ext(const struct pan_fb_info *fb, unsigned layer_idx, } if (fb->zs.view.zs) { + const struct pan_image_plane_ref pref = + pan_image_view_get_zs_plane(fb->zs.view.zs); + const struct pan_mod_handler *mod_handler = pref.image->mod_handler; struct mali_zs_crc_extension_packed zs_part; - pan_emit_zs_att(fb, layer_idx + fb->zs.view.zs->first_layer, &zs_part); + mod_handler->emit_zs_attachment( + fb, layer_idx + fb->zs.view.zs->first_layer, &zs_part); pan_merge(&desc, &zs_part, ZS_CRC_EXTENSION); } if (fb->zs.view.s) { + const struct pan_image_plane_ref pref = + pan_image_view_get_s_plane(fb->zs.view.s); + const struct pan_mod_handler *mod_handler = pref.image->mod_handler; struct mali_zs_crc_extension_packed s_part; - pan_emit_s_att(fb, layer_idx + fb->zs.view.s->first_layer, &s_part); + mod_handler->emit_s_attachment(fb, layer_idx + fb->zs.view.s->first_layer, + &s_part); pan_merge(&desc, &s_part, ZS_CRC_EXTENSION); } @@ -682,10 +656,11 @@ pan_prepare_rt_common(const struct pan_fb_info *fb, unsigned rt_idx, } } -static void -pan_emit_afbc_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, - unsigned layer_or_z_slice, unsigned cbuf_offset, - void *payload) +void +GENX(pan_emit_afbc_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload) { const struct pan_image_view *iview = fb->rts[rt_idx].view; const unsigned mip_level = iview->first_level; @@ -725,7 +700,7 @@ pan_emit_afbc_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, const struct pan_image_slice_layout *slayout = &plane->layout.slices[mip_level]; - cfg.afbc.body_size = slayout->afbc.body.size_B; + cfg.afbc.body_size = slayout->afbc.body.surface_stride_B; cfg.afbc.chunk_size = 9; cfg.afbc.sparse = true; #endif @@ -733,10 +708,11 @@ pan_emit_afbc_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, } } -static void -pan_emit_u_tiled_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, - unsigned layer_or_z_slice, - unsigned cbuf_offset, void *payload) +void +GENX(pan_emit_u_tiled_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload) { const struct pan_image_view *iview = fb->rts[rt_idx].view; @@ -752,10 +728,11 @@ pan_emit_u_tiled_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, } } -static void -pan_emit_linear_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, - unsigned layer_or_z_slice, - unsigned cbuf_offset, void *payload) +void +GENX(pan_emit_linear_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload) { const struct pan_image_view *iview = fb->rts[rt_idx].view; @@ -772,10 +749,10 @@ pan_emit_linear_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, } #if PAN_ARCH >= 10 -static void -pan_emit_afrc_color_attachment(const struct pan_fb_info *fb, unsigned rt_idx, - unsigned layer_or_z_slice, unsigned cbuf_offset, - void *payload) +void +GENX(pan_emit_afrc_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload) { const struct pan_image_view *iview = fb->rts[rt_idx].view; const struct pan_image_plane_ref pref = pan_image_view_get_color_plane(iview); @@ -873,29 +850,6 @@ pan_emit_midgard_tiler(const struct pan_fb_info *fb, #endif #if PAN_ARCH >= 5 -static void -pan_emit_color_att(const struct pan_fb_info *fb, unsigned idx, - unsigned layer_idx, unsigned cbuf_offset, - struct mali_render_target_packed *desc) -{ - const struct pan_image_plane_ref pref = - pan_image_view_get_first_plane(fb->rts[idx].view); - const uint64_t mod = pref.image->props.modifier; - - if (drm_is_afbc(mod)) { - pan_emit_afbc_color_attachment(fb, idx, layer_idx, cbuf_offset, desc); -#if PAN_ARCH >= 10 - } else if (drm_is_afrc(mod)) { - pan_emit_afrc_color_attachment(fb, idx, layer_idx, cbuf_offset, desc); -#endif - } else if (mod == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - pan_emit_u_tiled_color_attachment(fb, idx, layer_idx, cbuf_offset, desc); - } else { - assert(mod == DRM_FORMAT_MOD_LINEAR); - pan_emit_linear_color_attachment(fb, idx, layer_idx, cbuf_offset, desc); - } -} - static void pan_emit_rt(const struct pan_fb_info *fb, unsigned layer_idx, unsigned idx, unsigned cbuf_offset, struct mali_render_target_packed *out) @@ -918,6 +872,8 @@ pan_emit_rt(const struct pan_fb_info *fb, unsigned layer_idx, unsigned idx, struct pan_image_plane_ref pref = pan_image_view_get_color_plane(rt); assert(pref.image); + const struct pan_mod_handler *mod_handler = pref.image->mod_handler; + assert(mod_handler); ASSERTED unsigned layer_count = rt->dim == MALI_TEXTURE_DIMENSION_3D ? pref.image->props.extent_px.depth @@ -926,7 +882,8 @@ pan_emit_rt(const struct pan_fb_info *fb, unsigned layer_idx, unsigned idx, assert(rt->last_level == rt->first_level); assert(layer_idx < layer_count); - pan_emit_color_att(fb, idx, layer_idx + rt->first_layer, cbuf_offset, out); + mod_handler->emit_color_attachment(fb, idx, layer_idx + rt->first_layer, + cbuf_offset, out); } #if PAN_ARCH >= 6 diff --git a/src/panfrost/lib/pan_desc.h b/src/panfrost/lib/pan_desc.h index 17e7e0079cf..52cb7c2c9cc 100644 --- a/src/panfrost/lib/pan_desc.h +++ b/src/panfrost/lib/pan_desc.h @@ -264,6 +264,48 @@ void GENX(pan_emit_tls)(const struct pan_tls_info *info, int GENX(pan_select_crc_rt)(const struct pan_fb_info *fb, unsigned tile_size); +#if PAN_ARCH >= 5 +void GENX(pan_emit_linear_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, + void *payload); +void GENX(pan_emit_linear_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, + void *payload); +void GENX(pan_emit_linear_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, + void *payload); +void GENX(pan_emit_u_tiled_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, + void *payload); +void GENX(pan_emit_u_tiled_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, + void *payload); +void GENX(pan_emit_u_tiled_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, + void *payload); +void GENX(pan_emit_afbc_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload); +void GENX(pan_emit_afbc_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, + void *payload); +void GENX(pan_emit_afbc_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, + void *payload); +#endif + +#if PAN_ARCH >= 10 +void GENX(pan_emit_afrc_color_attachment)(const struct pan_fb_info *fb, + unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload); +#endif + unsigned GENX(pan_emit_fbd)(const struct pan_fb_info *fb, unsigned layer_idx, const struct pan_tls_info *tls, const struct pan_tiler_context *tiler_ctx, diff --git a/src/panfrost/lib/pan_image.h b/src/panfrost/lib/pan_image.h index aafbc3767e0..34b257dc2e7 100644 --- a/src/panfrost/lib/pan_image.h +++ b/src/panfrost/lib/pan_image.h @@ -16,6 +16,7 @@ #include "util/format/u_format.h" #include "pan_format.h" #include "pan_layout.h" +#include "pan_mod.h" #include "util/log.h" @@ -23,6 +24,8 @@ extern "C" { #endif +struct pan_mod_handler; + struct pan_image_plane { struct pan_image_layout layout; uint64_t base; @@ -30,6 +33,7 @@ struct pan_image_plane { struct pan_image { struct pan_image_props props; + const struct pan_mod_handler *mod_handler; struct pan_image_plane *planes[MAX_IMAGE_PLANES]; }; @@ -196,6 +200,19 @@ pan_image_view_check(const struct pan_image_view *iview) #endif } +static inline unsigned +pan_image_get_wsi_row_pitch(const struct pan_image *image, unsigned plane_idx, + unsigned mip_level) +{ + assert(image->mod_handler); + assert(image->mod_handler->get_wsi_row_pitch); + assert(plane_idx < ARRAY_SIZE(image->planes) && + plane_idx < util_format_get_num_planes(image->props.format)); + assert(image->planes[plane_idx]); + + return image->mod_handler->get_wsi_row_pitch(image, plane_idx, mip_level); +} + #ifdef __cplusplus } /* extern C */ #endif diff --git a/src/panfrost/lib/pan_layout.c b/src/panfrost/lib/pan_layout.c index 758185ce5d1..3b8dac1a278 100644 --- a/src/panfrost/lib/pan_layout.c +++ b/src/panfrost/lib/pan_layout.c @@ -29,6 +29,7 @@ #include "pan_afbc.h" #include "pan_afrc.h" #include "pan_layout.h" +#include "pan_mod.h" #include "pan_props.h" /* @@ -78,458 +79,10 @@ get_mip_level_extent(const struct pan_image_props *props, unsigned plane_idx, }; } -static unsigned -get_afbc_wsi_row_pitch(const struct pan_image_props *props, unsigned plane_idx, - const struct pan_image_slice_layout *slayout) -{ - const unsigned header_row_stride_B = slayout->afbc.header.row_stride_B; - const struct pan_image_block_size tile_extent_el = - pan_afbc_superblock_size_el(props->format, props->modifier); - const unsigned tile_payload_size_B = - tile_extent_el.width * tile_extent_el.height * - pan_format_get_plane_blocksize(props->format, plane_idx); - const unsigned tile_row_payload_size_B = - pan_afbc_stride_blocks(props->modifier, header_row_stride_B) * - tile_payload_size_B; - - return tile_row_payload_size_B / pan_afbc_superblock_height(props->modifier); -} - -static bool -init_afbc_slice_layout( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, - struct pan_image_extent mip_extent_px, - const struct pan_image_layout_constraints *layout_constraints, - struct pan_image_slice_layout *slayout) -{ - /* Use explicit layout only when wsi_row_pitch_B is non-zero */ - const bool use_explicit_layout = - layout_constraints && layout_constraints->wsi_row_pitch_B; - struct pan_image_block_size afbc_tile_extent_px = - pan_afbc_superblock_size(props->modifier); - unsigned offset_align_mask = - pan_afbc_header_align(arch, props->modifier) - 1; - unsigned row_align_mask = - pan_afbc_header_row_stride_align(arch, props->format, props->modifier) - - 1; - struct pan_image_block_size afbc_tile_extent_el = - pan_afbc_superblock_size_el(props->format, props->modifier); - unsigned afbc_tile_payload_size_B = - afbc_tile_extent_el.width * afbc_tile_extent_el.height * - pan_format_get_plane_blocksize(props->format, plane_idx); - - struct pan_image_block_size align_px = - pan_afbc_renderblock_size(props->modifier); - - /* If superblock tiling is used, align on a superblock tile. */ - if (props->modifier & AFBC_FORMAT_MOD_TILED) { - align_px.width = - ALIGN_POT(align_px.width, afbc_tile_extent_px.width * - pan_afbc_tile_size(props->modifier)); - align_px.height = - ALIGN_POT(align_px.height, afbc_tile_extent_px.height * - pan_afbc_tile_size(props->modifier)); - } - - struct pan_image_extent aligned_extent_px = { - .width = ALIGN_POT(mip_extent_px.width, align_px.width), - .height = ALIGN_POT(mip_extent_px.height, align_px.height), - .depth = mip_extent_px.depth, - }; - - if (use_explicit_layout) { - unsigned afbc_tile_payload_row_stride_B = - layout_constraints->wsi_row_pitch_B * - pan_afbc_superblock_height(props->modifier); - - /* For quite some time, we've been accepting WSI row pitch that - * didn't match exactly the image size and have been assuming tightly - * packed tile rows instead of using the explicit stride in that case. - * This is something we can't change without risking breaking existing - * users, so we enforce this explicit tile alignment only if we were - * asked to. */ - if (layout_constraints->strict && - (afbc_tile_payload_row_stride_B % afbc_tile_payload_size_B)) { - mesa_loge("WSI pitch is not aligned on an AFBC tile"); - return false; - } - - unsigned width_from_wsi_row_stride = - (afbc_tile_payload_row_stride_B / afbc_tile_payload_size_B) * - pan_afbc_superblock_width(props->modifier); - - if (width_from_wsi_row_stride < mip_extent_px.width) { - mesa_loge("WSI pitch too small"); - return false; - } - - slayout->afbc.header.row_stride_B = - pan_afbc_row_stride(props->modifier, width_from_wsi_row_stride); - if (slayout->afbc.header.row_stride_B & row_align_mask) { - mesa_loge("WSI pitch not properly aligned"); - return false; - } - - slayout->offset_B = layout_constraints->offset_B; - if (slayout->offset_B & offset_align_mask) { - mesa_loge("WSI offset not properly aligned"); - return false; - } - - /* If this is not a strict import, ignore the WSI row pitch and use - * the resource width to get the size. */ - if (!layout_constraints->strict) { - slayout->afbc.header.row_stride_B = ALIGN_POT( - pan_afbc_row_stride(props->modifier, aligned_extent_px.width), - row_align_mask + 1); - } - } else { - slayout->offset_B = - ALIGN_POT(layout_constraints ? layout_constraints->offset_B : 0, - offset_align_mask + 1); - slayout->afbc.header.row_stride_B = ALIGN_POT( - pan_afbc_row_stride(props->modifier, aligned_extent_px.width), - row_align_mask + 1); - } - - const unsigned row_stride_sb = pan_afbc_stride_blocks( - props->modifier, slayout->afbc.header.row_stride_B); - const unsigned surface_stride_sb = - row_stride_sb * (aligned_extent_px.height / afbc_tile_extent_px.height); - - uint64_t hdr_surface_stride_B = (uint64_t)surface_stride_sb * - AFBC_HEADER_BYTES_PER_TILE; - hdr_surface_stride_B = - ALIGN_POT(hdr_surface_stride_B, (uint64_t)(offset_align_mask + 1)); - - slayout->afbc.header.surface_stride_B = hdr_surface_stride_B; - - uint64_t header_size_B = hdr_surface_stride_B * aligned_extent_px.depth; - header_size_B = ALIGN_POT( - header_size_B, (uint64_t)pan_afbc_body_align(arch, props->modifier)); - - slayout->afbc.header.size_B = header_size_B; - - uint64_t body_surf_stride_B = - (uint64_t)surface_stride_sb * afbc_tile_payload_size_B; - uint64_t body_size_B = body_surf_stride_B * aligned_extent_px.depth; - - /* Each AFBC header encodes the offset to its AFBC data in a 32-bit field. - * AFBC headers of all 3D slices are placed at the beginning, meaning the - * maximum offset that exists is between the last header, and the last - * tile. */ - ASSERTED uint64_t max_body_offset = body_size_B - afbc_tile_payload_size_B + - header_size_B - - AFBC_HEADER_BYTES_PER_TILE; - - if (max_body_offset > UINT32_MAX) - return false; - - slayout->afbc.body.surface_stride_B = body_surf_stride_B; - slayout->afbc.body.size_B = body_size_B; - slayout->size_B = header_size_B + body_size_B; - return true; -} - -static unsigned -get_afrc_wsi_row_pitch(const struct pan_image_props *props, unsigned plane_idx, - const struct pan_image_slice_layout *slayout) -{ - const struct pan_image_block_size tile_extent_px = - pan_afrc_tile_size(props->format, props->modifier); - - return slayout->tiled_or_linear.row_stride_B / tile_extent_px.height; -} - -static bool -init_afrc_slice_layout( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, - struct pan_image_extent mip_extent_px, - const struct pan_image_layout_constraints *layout_constraints, - struct pan_image_slice_layout *slayout) -{ - /* Use explicit layout only when wsi_row_pitch_B is non-zero */ - const bool use_explicit_layout = - layout_constraints && layout_constraints->wsi_row_pitch_B; - const unsigned align_mask = - pan_afrc_buffer_alignment_from_modifier(props->modifier) - 1; - struct pan_image_block_size tile_extent_px = - pan_afrc_tile_size(props->format, props->modifier); - struct pan_image_extent aligned_extent_px = { - .width = ALIGN_POT(mip_extent_px.width, tile_extent_px.width), - .height = ALIGN_POT(mip_extent_px.height, tile_extent_px.height), - .depth = mip_extent_px.depth, - }; - - if (use_explicit_layout) { - slayout->tiled_or_linear.row_stride_B = - layout_constraints->wsi_row_pitch_B * tile_extent_px.height; - if (slayout->tiled_or_linear.row_stride_B & align_mask) { - mesa_loge("WSI pitch not properly aligned"); - return false; - } - - slayout->offset_B = layout_constraints->offset_B; - if (slayout->offset_B & align_mask) { - mesa_loge("WSI offset not properly aligned"); - return false; - } - - unsigned afrc_blk_size_B = - pan_afrc_block_size_from_modifier(props->modifier) * - AFRC_CLUMPS_PER_TILE; - unsigned width_from_wsi_row_stride = - (slayout->tiled_or_linear.row_stride_B / afrc_blk_size_B) * - tile_extent_px.width; - - if (width_from_wsi_row_stride < mip_extent_px.width) { - mesa_loge("WSI pitch too small"); - return false; - } - - /* If this is not a strict import, ignore the WSI row pitch and use - * the resource width to get the size. */ - if (!layout_constraints->strict) { - slayout->tiled_or_linear.row_stride_B = pan_afrc_row_stride( - props->format, props->modifier, mip_extent_px.width); - } - } else { - /* Align levels to cache-line as a performance improvement for - * linear/tiled and as a requirement for AFBC */ - slayout->offset_B = ALIGN_POT( - layout_constraints ? layout_constraints->offset_B : 0, align_mask + 1); - slayout->tiled_or_linear.row_stride_B = - ALIGN_POT(pan_afrc_row_stride(props->format, props->modifier, - mip_extent_px.width), - align_mask + 1); - } - - uint64_t surf_stride_B = - (uint64_t)slayout->tiled_or_linear.row_stride_B * - DIV_ROUND_UP(aligned_extent_px.height, aligned_extent_px.height); - - /* Surface stride is passed as a 32-bit unsigned integer to RT/ZS and texture - * descriptors, make sure it fits. */ - if (surf_stride_B > UINT32_MAX) - return false; - - slayout->tiled_or_linear.surface_stride_B = surf_stride_B; - slayout->size_B = - surf_stride_B * aligned_extent_px.depth * props->nr_samples; - return true; -} - -static unsigned -get_u_tiled_wsi_row_pitch(const struct pan_image_props *props, - unsigned plane_idx, - const struct pan_image_slice_layout *slayout) -{ - return slayout->tiled_or_linear.row_stride_B / - pan_u_interleaved_tile_size_el(props->format).height; -} - -static bool -init_u_tiled_slice_layout( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, - struct pan_image_extent mip_extent_px, - const struct pan_image_layout_constraints *layout_constraints, - struct pan_image_slice_layout *slayout) -{ - /* Use explicit layout only when wsi_row_pitch_B is non-zero */ - const bool use_explicit_layout = - layout_constraints && layout_constraints->wsi_row_pitch_B; - unsigned align_mask = - pan_linear_or_tiled_row_align_req(arch, props->format, plane_idx) - 1; - struct pan_image_block_size tile_extent_el = - pan_u_interleaved_tile_size_el(props->format); - struct pan_image_extent mip_extent_el; - unsigned tile_size_B; - - if (util_format_is_compressed(props->format)) { - assert(util_format_get_num_planes(props->format) == 1); - mip_extent_el.width = DIV_ROUND_UP( - mip_extent_px.width, util_format_get_blockwidth(props->format)); - mip_extent_el.height = DIV_ROUND_UP( - mip_extent_px.height, util_format_get_blockheight(props->format)); - mip_extent_el.depth = DIV_ROUND_UP( - mip_extent_px.depth, util_format_get_blockdepth(props->format)); - tile_size_B = tile_extent_el.width * tile_extent_el.height * - pan_format_get_plane_blocksize(props->format, plane_idx); - } else { - /* Block-based YUV needs special care, because the U-tile extent - * is in pixels, not blocks in that case. */ - assert(tile_extent_el.width % util_format_get_blockwidth(props->format) == - 0); - assert(tile_extent_el.height % - util_format_get_blockheight(props->format) == - 0); - mip_extent_el = mip_extent_px; - tile_size_B = - (tile_extent_el.width / util_format_get_blockwidth(props->format)) * - (tile_extent_el.height / util_format_get_blockheight(props->format)) * - pan_format_get_plane_blocksize(props->format, plane_idx); - } - - if (use_explicit_layout) { - slayout->tiled_or_linear.row_stride_B = - layout_constraints->wsi_row_pitch_B * tile_extent_el.height; - if (slayout->tiled_or_linear.row_stride_B & align_mask) { - mesa_loge("WSI pitch not properly aligned"); - return false; - } - - const unsigned width_from_wsi_row_stride = - (slayout->tiled_or_linear.row_stride_B / tile_size_B) * - tile_extent_el.width; - - if (width_from_wsi_row_stride < mip_extent_el.width) { - mesa_loge("WSI pitch too small"); - return false; - } - - slayout->offset_B = layout_constraints->offset_B; - if (slayout->offset_B & align_mask) { - mesa_loge("WSI offset not properly aligned"); - return false; - } - } else { - /* When we can decide of the layout, we want things aligned on at least a - * cacheline for performance reasons. */ - align_mask = MAX2(align_mask, 63); - slayout->offset_B = ALIGN_POT( - layout_constraints ? layout_constraints->offset_B : 0, - MAX2(align_mask + 1, pan_image_slice_align(props->modifier))); - slayout->tiled_or_linear.row_stride_B = ALIGN_POT( - tile_size_B * DIV_ROUND_UP(mip_extent_el.width, tile_extent_el.width), - align_mask + 1); - } - - uint64_t surf_stride_B = - (uint64_t)slayout->tiled_or_linear.row_stride_B * - DIV_ROUND_UP(mip_extent_el.height, tile_extent_el.height); - surf_stride_B = ALIGN_POT(surf_stride_B, (uint64_t)align_mask + 1); - - /* Surface stride is passed as a 32-bit unsigned integer to RT/ZS and texture - * descriptors, make sure it fits. */ - if (surf_stride_B > UINT32_MAX) - return false; - - slayout->tiled_or_linear.surface_stride_B = surf_stride_B; - slayout->size_B = surf_stride_B * mip_extent_el.depth * props->nr_samples; - return true; -} - -static unsigned -get_linear_wsi_row_pitch(const struct pan_image_props *props, - unsigned plane_idx, - const struct pan_image_slice_layout *slayout) -{ - return slayout->tiled_or_linear.row_stride_B; -} - -static bool -init_linear_slice_layout( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, - struct pan_image_extent mip_extent_px, - const struct pan_image_layout_constraints *layout_constraints, - struct pan_image_slice_layout *slayout) -{ - /* Use explicit layout only when wsi_row_pitch_B is non-zero */ - const bool use_explicit_layout = - layout_constraints && layout_constraints->wsi_row_pitch_B; - unsigned align_mask = - pan_linear_or_tiled_row_align_req(arch, props->format, plane_idx) - 1; - const unsigned fmt_blksize_B = - pan_format_get_plane_blocksize(props->format, plane_idx); - struct pan_image_extent mip_extent_el; - - if (util_format_is_compressed(props->format)) { - assert(util_format_get_num_planes(props->format) == 1); - mip_extent_el.width = DIV_ROUND_UP( - mip_extent_px.width, util_format_get_blockwidth(props->format)); - mip_extent_el.height = DIV_ROUND_UP( - mip_extent_px.height, util_format_get_blockheight(props->format)); - mip_extent_el.depth = DIV_ROUND_UP( - mip_extent_px.depth, util_format_get_blockdepth(props->format)); - } else { - mip_extent_el = mip_extent_px; - } - - if (use_explicit_layout) { - unsigned width_from_wsi_row_stride = - layout_constraints->wsi_row_pitch_B / fmt_blksize_B; - - if (!util_format_is_compressed(props->format)) - width_from_wsi_row_stride *= util_format_get_blockwidth(props->format); - - if (width_from_wsi_row_stride < mip_extent_el.width) { - mesa_loge("WSI pitch too small"); - return false; - } - - slayout->tiled_or_linear.row_stride_B = - layout_constraints->wsi_row_pitch_B; - if (slayout->tiled_or_linear.row_stride_B & align_mask) { - mesa_loge("WSI pitch not properly aligned"); - return false; - } - - slayout->offset_B = layout_constraints->offset_B; - if (slayout->offset_B & align_mask) { - mesa_loge("WSI offset not properly aligned"); - return false; - } - } else { - /* When we can decide of the layout, we want things aligned on at least a - * cacheline for performance reasons. */ - align_mask = MAX2(align_mask, 63); - slayout->offset_B = ALIGN_POT( - layout_constraints ? layout_constraints->offset_B : 0, - MAX2(align_mask + 1, pan_image_slice_align(props->modifier))); - slayout->tiled_or_linear.row_stride_B = - ALIGN_POT(mip_extent_el.width * fmt_blksize_B, align_mask + 1); - } - - uint64_t surf_stride_B = - (uint64_t)slayout->tiled_or_linear.row_stride_B * mip_extent_el.height; - surf_stride_B = ALIGN_POT(surf_stride_B, (uint64_t)align_mask + 1); - - /* Surface stride is passed as a 32-bit unsigned integer to RT/ZS and texture - * descriptors, make sure it fits. */ - if (surf_stride_B > UINT32_MAX) - return false; - - slayout->tiled_or_linear.surface_stride_B = surf_stride_B; - slayout->size_B = surf_stride_B * mip_extent_el.depth * props->nr_samples; - return true; -} - -static bool -init_slice_layout( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, - struct pan_image_extent mip_extent_px, - const struct pan_image_layout_constraints *layout_constraints, - struct pan_image_slice_layout *slayout) -{ - if (drm_is_afbc(props->modifier)) { - return init_afbc_slice_layout(arch, props, plane_idx, mip_extent_px, - layout_constraints, slayout); - } else if (drm_is_afrc(props->modifier)) { - return init_afrc_slice_layout(arch, props, plane_idx, mip_extent_px, - layout_constraints, slayout); - } else if (props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - return init_u_tiled_slice_layout(arch, props, plane_idx, mip_extent_px, - layout_constraints, slayout); - } else { - assert(props->modifier == DRM_FORMAT_MOD_LINEAR); - return init_linear_slice_layout(arch, props, plane_idx, mip_extent_px, - layout_constraints, slayout); - } -} - bool pan_image_layout_init( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, + unsigned arch, const struct pan_mod_handler *mod_handler, + const struct pan_image_props *props, unsigned plane_idx, const struct pan_image_layout_constraints *explicit_layout_constraints, struct pan_image_layout *layout) { @@ -572,8 +125,8 @@ pan_image_layout_init( for (unsigned l = 0; l < props->nr_slices; ++l) { struct pan_image_slice_layout *slayout = &layout->slices[l]; - if (!init_slice_layout(arch, props, plane_idx, mip_extent_px, - &layout_constraints, slayout)) + if (!mod_handler->init_slice_layout(props, plane_idx, mip_extent_px, + &layout_constraints, slayout)) return false; layout_constraints.offset_B += slayout->size_B; @@ -611,23 +164,3 @@ pan_image_layout_init( return true; } - -unsigned -pan_image_get_wsi_row_pitch(const struct pan_image_props *props, - unsigned plane_idx, - const struct pan_image_layout *layout, - unsigned level) -{ - const struct pan_image_slice_layout *slayout = &layout->slices[level]; - - if (drm_is_afbc(props->modifier)) { - return get_afbc_wsi_row_pitch(props, plane_idx, slayout); - } else if (drm_is_afrc(props->modifier)) { - return get_afrc_wsi_row_pitch(props, plane_idx, slayout); - } else if (props->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - return get_u_tiled_wsi_row_pitch(props, plane_idx, slayout); - } else { - assert(props->modifier == DRM_FORMAT_MOD_LINEAR); - return get_linear_wsi_row_pitch(props, plane_idx, slayout); - } -} diff --git a/src/panfrost/lib/pan_layout.h b/src/panfrost/lib/pan_layout.h index a7fce34fb07..41616d49199 100644 --- a/src/panfrost/lib/pan_layout.h +++ b/src/panfrost/lib/pan_layout.h @@ -23,6 +23,8 @@ extern "C" { #define MAX_MIP_LEVELS 17 #define MAX_IMAGE_PLANES 3 +struct pan_mod_handler; + struct pan_afbc_image_slice_layout { struct { /* Number of bytes between two rows of AFBC headers. */ @@ -177,7 +179,8 @@ pan_image_mip_level_size(const struct pan_image_props *props, } bool pan_image_layout_init( - unsigned arch, const struct pan_image_props *props, unsigned plane_idx, + unsigned arch, const struct pan_mod_handler *mod_handler, + const struct pan_image_props *props, unsigned plane_idx, const struct pan_image_layout_constraints *layout_constraints, struct pan_image_layout *layout); @@ -187,11 +190,6 @@ pan_image_get_wsi_offset(const struct pan_image_layout *layout, unsigned level) return layout->slices[level].offset_B; } -unsigned pan_image_get_wsi_row_pitch(const struct pan_image_props *props, - unsigned plane_idx, - const struct pan_image_layout *layout, - unsigned level); - static inline uint32_t pan_linear_or_tiled_row_align_req(unsigned arch, enum pipe_format format, unsigned plane_idx) diff --git a/src/panfrost/lib/pan_mod.c b/src/panfrost/lib/pan_mod.c new file mode 100644 index 00000000000..9d3138416ea --- /dev/null +++ b/src/panfrost/lib/pan_mod.c @@ -0,0 +1,569 @@ +/* + * Copyright (C) 2019-2025 Collabora, Ltd. + * Copyright (C) 2018-2019 Alyssa Rosenzweig + * + * SPDX-License-Identifier: MIT + */ + +#include "pan_mod.h" +#include "pan_afbc.h" +#include "pan_afrc.h" +#include "pan_desc.h" +#include "pan_format.h" +#include "pan_image.h" +#include "pan_layout.h" +#include "pan_texture.h" + +#include "util/format/u_format.h" + +static bool +pan_mod_afbc_match(uint64_t mod) +{ + return drm_is_afbc(mod); +} + +static bool +pan_mod_afbc_supports_format(uint64_t mod, enum pipe_format format) +{ + return pan_afbc_supports_format(PAN_ARCH, format); +} + +static uint32_t +pan_mod_afbc_get_wsi_row_pitch(const struct pan_image *image, + unsigned plane_idx, unsigned mip_level) +{ + const struct pan_image_props *props = &image->props; + const struct pan_image_layout *layout = &image->planes[plane_idx]->layout; + const unsigned header_row_stride_B = + layout->slices[mip_level].afbc.header.row_stride_B; + const struct pan_image_block_size tile_extent_el = + pan_afbc_superblock_size_el(props->format, props->modifier); + const unsigned tile_payload_size_B = + tile_extent_el.width * tile_extent_el.height * + pan_format_get_plane_blocksize(props->format, plane_idx); + const unsigned tile_row_payload_size_B = + pan_afbc_stride_blocks(props->modifier, header_row_stride_B) * + tile_payload_size_B; + + return tile_row_payload_size_B / pan_afbc_superblock_height(props->modifier); +} + +static bool +pan_mod_afbc_init_slice_layout( + const struct pan_image_props *props, unsigned plane_idx, + struct pan_image_extent mip_extent_px, + const struct pan_image_layout_constraints *layout_constraints, + struct pan_image_slice_layout *slayout) +{ + /* Use explicit layout only when wsi_row_pitch_B is non-zero */ + const bool use_explicit_layout = + layout_constraints && layout_constraints->wsi_row_pitch_B; + struct pan_image_block_size afbc_tile_extent_px = + pan_afbc_superblock_size(props->modifier); + unsigned offset_align_mask = + pan_afbc_header_align(PAN_ARCH, props->modifier) - 1; + unsigned row_align_mask = pan_afbc_header_row_stride_align( + PAN_ARCH, props->format, props->modifier) - + 1; + struct pan_image_block_size afbc_tile_extent_el = + pan_afbc_superblock_size_el(props->format, props->modifier); + unsigned afbc_tile_payload_size_B = + afbc_tile_extent_el.width * afbc_tile_extent_el.height * + pan_format_get_plane_blocksize(props->format, plane_idx); + + struct pan_image_block_size align_px = + pan_afbc_renderblock_size(props->modifier); + + /* If superblock tiling is used, align on a superblock tile. */ + if (props->modifier & AFBC_FORMAT_MOD_TILED) { + align_px.width = + ALIGN_POT(align_px.width, afbc_tile_extent_px.width * + pan_afbc_tile_size(props->modifier)); + align_px.height = + ALIGN_POT(align_px.height, afbc_tile_extent_px.height * + pan_afbc_tile_size(props->modifier)); + } + + struct pan_image_extent aligned_extent_px = { + .width = ALIGN_POT(mip_extent_px.width, align_px.width), + .height = ALIGN_POT(mip_extent_px.height, align_px.height), + .depth = mip_extent_px.depth, + }; + + if (use_explicit_layout) { + unsigned afbc_tile_payload_row_stride_B = + layout_constraints->wsi_row_pitch_B * + pan_afbc_superblock_height(props->modifier); + + /* For quite some time, we've been accepting WSI row pitch that + * didn't match exactly the image size and have been assuming tightly + * packed tile rows instead of using the explicit stride in that case. + * This is something we can't change without risking breaking existing + * users, so we enforce this explicit tile alignment only if we were + * asked to. */ + if (layout_constraints->strict && + (afbc_tile_payload_row_stride_B % afbc_tile_payload_size_B)) { + mesa_loge("WSI pitch is not aligned on an AFBC tile"); + return false; + } + + unsigned width_from_wsi_row_stride = + (afbc_tile_payload_row_stride_B / afbc_tile_payload_size_B) * + pan_afbc_superblock_width(props->modifier); + + if (width_from_wsi_row_stride < mip_extent_px.width) { + mesa_loge("WSI pitch too small"); + return false; + } + + slayout->afbc.header.row_stride_B = + pan_afbc_row_stride(props->modifier, width_from_wsi_row_stride); + if (slayout->afbc.header.row_stride_B & row_align_mask) { + mesa_loge("WSI pitch not properly aligned"); + return false; + } + + slayout->offset_B = layout_constraints->offset_B; + if (slayout->offset_B & offset_align_mask) { + mesa_loge("WSI offset not properly aligned"); + return false; + } + + /* If this is not a strict import, ignore the WSI row pitch and use + * the resource width to get the size. */ + if (!layout_constraints->strict) { + slayout->afbc.header.row_stride_B = ALIGN_POT( + pan_afbc_row_stride(props->modifier, aligned_extent_px.width), + row_align_mask + 1); + } + } else { + slayout->offset_B = + ALIGN_POT(layout_constraints ? layout_constraints->offset_B : 0, + offset_align_mask + 1); + slayout->afbc.header.row_stride_B = ALIGN_POT( + pan_afbc_row_stride(props->modifier, aligned_extent_px.width), + row_align_mask + 1); + } + + const unsigned row_stride_sb = pan_afbc_stride_blocks( + props->modifier, slayout->afbc.header.row_stride_B); + const unsigned surface_stride_sb = + row_stride_sb * (aligned_extent_px.height / afbc_tile_extent_px.height); + + uint64_t hdr_surface_stride_B = (uint64_t)surface_stride_sb * + AFBC_HEADER_BYTES_PER_TILE; + hdr_surface_stride_B = + ALIGN_POT(hdr_surface_stride_B, (uint64_t)(offset_align_mask + 1)); + + slayout->afbc.header.surface_stride_B = hdr_surface_stride_B; + + uint64_t header_size_B = hdr_surface_stride_B * aligned_extent_px.depth; + header_size_B = ALIGN_POT( + header_size_B, (uint64_t)pan_afbc_body_align(PAN_ARCH, props->modifier)); + + slayout->afbc.header.size_B = header_size_B; + + uint64_t body_surf_stride_B = + (uint64_t)surface_stride_sb * afbc_tile_payload_size_B; + uint64_t body_size_B = body_surf_stride_B * aligned_extent_px.depth; + + /* Each AFBC header encodes the offset to its AFBC data in a 32-bit field. + * AFBC headers of all 3D slices are placed at the beginning, meaning the + * maximum offset that exists is between the last header, and the last + * tile. */ + ASSERTED uint64_t max_body_offset = body_size_B - afbc_tile_payload_size_B + + header_size_B - + AFBC_HEADER_BYTES_PER_TILE; + + if (max_body_offset > UINT32_MAX) + return false; + + slayout->afbc.body.surface_stride_B = body_surf_stride_B; + slayout->afbc.body.size_B = body_size_B; + slayout->size_B = header_size_B + body_size_B; + return true; +} + +#define pan_mod_afbc_emit_tex_payload_entry \ + GENX(pan_tex_emit_afbc_payload_entry) +#define pan_mod_afbc_emit_color_attachment GENX(pan_emit_afbc_color_attachment) +#define pan_mod_afbc_emit_zs_attachment GENX(pan_emit_afbc_zs_attachment) +#define pan_mod_afbc_emit_s_attachment GENX(pan_emit_afbc_s_attachment) + +#if PAN_ARCH >= 10 +static bool +pan_mod_afrc_match(uint64_t mod) +{ + return drm_is_afrc(mod); +} + +static bool +pan_mod_afrc_supports_format(uint64_t mod, enum pipe_format format) +{ + return pan_afrc_supports_format(format); +} + +static uint32_t +pan_mod_afrc_get_wsi_row_pitch(const struct pan_image *image, + unsigned plane_idx, unsigned mip_level) +{ + const struct pan_image_props *props = &image->props; + const struct pan_image_layout *layout = &image->planes[plane_idx]->layout; + const struct pan_image_block_size tile_extent_px = + pan_afrc_tile_size(props->format, props->modifier); + + return layout->slices[mip_level].tiled_or_linear.row_stride_B / + tile_extent_px.height; +} + +static bool +pan_mod_afrc_init_slice_layout( + const struct pan_image_props *props, unsigned plane_idx, + struct pan_image_extent mip_extent_px, + const struct pan_image_layout_constraints *layout_constraints, + struct pan_image_slice_layout *slayout) +{ + /* Use explicit layout only when wsi_row_pitch_B is non-zero */ + const bool use_explicit_layout = + layout_constraints && layout_constraints->wsi_row_pitch_B; + const unsigned align_mask = + pan_afrc_buffer_alignment_from_modifier(props->modifier) - 1; + struct pan_image_block_size tile_extent_px = + pan_afrc_tile_size(props->format, props->modifier); + struct pan_image_extent aligned_extent_px = { + .width = ALIGN_POT(mip_extent_px.width, tile_extent_px.width), + .height = ALIGN_POT(mip_extent_px.height, tile_extent_px.height), + .depth = mip_extent_px.depth, + }; + + if (use_explicit_layout) { + slayout->tiled_or_linear.row_stride_B = + layout_constraints->wsi_row_pitch_B * tile_extent_px.height; + if (slayout->tiled_or_linear.row_stride_B & align_mask) { + mesa_loge("WSI pitch not properly aligned"); + return false; + } + + slayout->offset_B = layout_constraints->offset_B; + if (slayout->offset_B & align_mask) { + mesa_loge("WSI offset not properly aligned"); + return false; + } + + unsigned afrc_blk_size_B = + pan_afrc_block_size_from_modifier(props->modifier) * + AFRC_CLUMPS_PER_TILE; + unsigned width_from_wsi_row_stride = + (slayout->tiled_or_linear.row_stride_B / afrc_blk_size_B) * + tile_extent_px.width; + + if (width_from_wsi_row_stride < mip_extent_px.width) { + mesa_loge("WSI pitch too small"); + return false; + } + + /* If this is not a strict import, ignore the WSI row pitch and use + * the resource width to get the size. */ + if (!layout_constraints->strict) { + slayout->tiled_or_linear.row_stride_B = pan_afrc_row_stride( + props->format, props->modifier, mip_extent_px.width); + } + } else { + /* Align levels to cache-line as a performance improvement for + * linear/tiled and as a requirement for AFBC */ + slayout->offset_B = ALIGN_POT( + layout_constraints ? layout_constraints->offset_B : 0, align_mask + 1); + slayout->tiled_or_linear.row_stride_B = + ALIGN_POT(pan_afrc_row_stride(props->format, props->modifier, + mip_extent_px.width), + align_mask + 1); + } + + uint64_t surf_stride_B = + (uint64_t)slayout->tiled_or_linear.row_stride_B * + DIV_ROUND_UP(aligned_extent_px.height, aligned_extent_px.height); + + /* Surface stride is passed as a 32-bit unsigned integer to RT/ZS and texture + * descriptors, make sure it fits. */ + if (surf_stride_B > UINT32_MAX) + return false; + + slayout->tiled_or_linear.surface_stride_B = surf_stride_B; + slayout->size_B = + surf_stride_B * aligned_extent_px.depth * props->nr_samples; + return true; +} + +#define pan_mod_afrc_emit_tex_payload_entry \ + GENX(pan_tex_emit_afrc_payload_entry) +#define pan_mod_afrc_emit_color_attachment GENX(pan_emit_afrc_color_attachment) +#define pan_mod_afrc_emit_zs_attachment NULL +#define pan_mod_afrc_emit_s_attachment NULL +#endif + +static bool +pan_mod_u_tiled_match(uint64_t mod) +{ + return mod == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED; +} + +static bool +pan_mod_u_tiled_supports_format(uint64_t mod, enum pipe_format format) +{ + return true; +} + +static uint32_t +pan_mod_u_tiled_get_wsi_row_pitch(const struct pan_image *image, + unsigned plane_idx, unsigned mip_level) +{ + const struct pan_image_props *props = &image->props; + const struct pan_image_layout *layout = &image->planes[plane_idx]->layout; + + return layout->slices[mip_level].tiled_or_linear.row_stride_B / + pan_u_interleaved_tile_size_el(props->format).height; +} + +static bool +pan_mod_u_tiled_init_slice_layout( + const struct pan_image_props *props, unsigned plane_idx, + struct pan_image_extent mip_extent_px, + const struct pan_image_layout_constraints *layout_constraints, + struct pan_image_slice_layout *slayout) +{ + /* Use explicit layout only when wsi_row_pitch_B is non-zero */ + const bool use_explicit_layout = + layout_constraints && layout_constraints->wsi_row_pitch_B; + unsigned align_mask = + pan_linear_or_tiled_row_align_req(PAN_ARCH, props->format, plane_idx) - 1; + struct pan_image_block_size tile_extent_el = + pan_u_interleaved_tile_size_el(props->format); + struct pan_image_extent mip_extent_el; + unsigned tile_size_B; + + if (util_format_is_compressed(props->format)) { + assert(util_format_get_num_planes(props->format) == 1); + mip_extent_el.width = DIV_ROUND_UP( + mip_extent_px.width, util_format_get_blockwidth(props->format)); + mip_extent_el.height = DIV_ROUND_UP( + mip_extent_px.height, util_format_get_blockheight(props->format)); + mip_extent_el.depth = DIV_ROUND_UP( + mip_extent_px.depth, util_format_get_blockdepth(props->format)); + tile_size_B = tile_extent_el.width * tile_extent_el.height * + pan_format_get_plane_blocksize(props->format, plane_idx); + } else { + /* Block-based YUV needs special care, because the U-tile extent + * is in pixels, not blocks in that case. */ + assert(tile_extent_el.width % util_format_get_blockwidth(props->format) == + 0); + assert(tile_extent_el.height % + util_format_get_blockheight(props->format) == + 0); + mip_extent_el = mip_extent_px; + tile_size_B = + (tile_extent_el.width / util_format_get_blockwidth(props->format)) * + (tile_extent_el.height / util_format_get_blockheight(props->format)) * + pan_format_get_plane_blocksize(props->format, plane_idx); + } + + if (use_explicit_layout) { + slayout->tiled_or_linear.row_stride_B = + layout_constraints->wsi_row_pitch_B * tile_extent_el.height; + if (slayout->tiled_or_linear.row_stride_B & align_mask) { + mesa_loge("WSI pitch not properly aligned"); + return false; + } + + const unsigned width_from_wsi_row_stride = + (slayout->tiled_or_linear.row_stride_B / tile_size_B) * + tile_extent_el.width; + + if (width_from_wsi_row_stride < mip_extent_el.width) { + mesa_loge("WSI pitch too small"); + return false; + } + + slayout->offset_B = layout_constraints->offset_B; + if (slayout->offset_B & align_mask) { + mesa_loge("WSI offset not properly aligned"); + return false; + } + } else { + /* When we can decide of the layout, we want things aligned on at least a + * cacheline for performance reasons. */ + align_mask = MAX2(align_mask, 63); + slayout->offset_B = ALIGN_POT( + layout_constraints ? layout_constraints->offset_B : 0, + MAX2(align_mask + 1, pan_image_slice_align(props->modifier))); + slayout->tiled_or_linear.row_stride_B = ALIGN_POT( + tile_size_B * DIV_ROUND_UP(mip_extent_el.width, tile_extent_el.width), + align_mask + 1); + } + + uint64_t surf_stride_B = + (uint64_t)slayout->tiled_or_linear.row_stride_B * + DIV_ROUND_UP(mip_extent_el.height, tile_extent_el.height); + surf_stride_B = ALIGN_POT(surf_stride_B, (uint64_t)align_mask + 1); + + /* Surface stride is passed as a 32-bit unsigned integer to RT/ZS and texture + * descriptors, make sure it fits. */ + if (surf_stride_B > UINT32_MAX) + return false; + + slayout->tiled_or_linear.surface_stride_B = surf_stride_B; + slayout->size_B = surf_stride_B * mip_extent_el.depth * props->nr_samples; + return true; +} + +#define pan_mod_u_tiled_emit_tex_payload_entry \ + GENX(pan_tex_emit_u_tiled_payload_entry) +#define pan_mod_u_tiled_emit_color_attachment \ + GENX(pan_emit_u_tiled_color_attachment) +#define pan_mod_u_tiled_emit_zs_attachment GENX(pan_emit_u_tiled_zs_attachment) +#define pan_mod_u_tiled_emit_s_attachment GENX(pan_emit_u_tiled_s_attachment) + +static bool +pan_mod_linear_match(uint64_t mod) +{ + return mod == DRM_FORMAT_MOD_LINEAR; +} + +static bool +pan_mod_linear_supports_format(uint64_t mod, enum pipe_format format) +{ + return true; +} + +static uint32_t +pan_mod_linear_get_wsi_row_pitch(const struct pan_image *image, + unsigned plane_idx, unsigned mip_level) +{ + const struct pan_image_layout *layout = &image->planes[plane_idx]->layout; + + return layout->slices[mip_level].tiled_or_linear.row_stride_B; +} + +static bool +pan_mod_linear_init_slice_layout( + const struct pan_image_props *props, unsigned plane_idx, + struct pan_image_extent mip_extent_px, + const struct pan_image_layout_constraints *layout_constraints, + struct pan_image_slice_layout *slayout) +{ + /* Use explicit layout only when wsi_row_pitch_B is non-zero */ + const bool use_explicit_layout = + layout_constraints && layout_constraints->wsi_row_pitch_B; + unsigned align_mask = + pan_linear_or_tiled_row_align_req(PAN_ARCH, props->format, plane_idx) - 1; + const unsigned fmt_blksize_B = + pan_format_get_plane_blocksize(props->format, plane_idx); + struct pan_image_extent mip_extent_el; + + if (util_format_is_compressed(props->format)) { + assert(util_format_get_num_planes(props->format) == 1); + mip_extent_el.width = DIV_ROUND_UP( + mip_extent_px.width, util_format_get_blockwidth(props->format)); + mip_extent_el.height = DIV_ROUND_UP( + mip_extent_px.height, util_format_get_blockheight(props->format)); + mip_extent_el.depth = DIV_ROUND_UP( + mip_extent_px.depth, util_format_get_blockdepth(props->format)); + } else { + mip_extent_el = mip_extent_px; + } + + if (use_explicit_layout) { + unsigned width_from_wsi_row_stride = + layout_constraints->wsi_row_pitch_B / fmt_blksize_B; + + if (!util_format_is_compressed(props->format)) + width_from_wsi_row_stride *= util_format_get_blockwidth(props->format); + + if (width_from_wsi_row_stride < mip_extent_el.width) { + mesa_loge("WSI pitch too small"); + return false; + } + + slayout->tiled_or_linear.row_stride_B = + layout_constraints->wsi_row_pitch_B; + if (slayout->tiled_or_linear.row_stride_B & align_mask) { + mesa_loge("WSI pitch not properly aligned"); + return false; + } + + slayout->offset_B = layout_constraints->offset_B; + if (slayout->offset_B & align_mask) { + mesa_loge("WSI offset not properly aligned"); + return false; + } + } else { + /* When we can decide of the layout, we want things aligned on at least a + * cacheline for performance reasons. */ + align_mask = MAX2(align_mask, 63); + slayout->offset_B = ALIGN_POT( + layout_constraints ? layout_constraints->offset_B : 0, + MAX2(align_mask + 1, pan_image_slice_align(props->modifier))); + slayout->tiled_or_linear.row_stride_B = + ALIGN_POT(mip_extent_el.width * fmt_blksize_B, align_mask + 1); + } + + uint64_t surf_stride_B = + (uint64_t)slayout->tiled_or_linear.row_stride_B * mip_extent_el.height; + surf_stride_B = ALIGN_POT(surf_stride_B, (uint64_t)align_mask + 1); + + /* Surface stride is passed as a 32-bit unsigned integer to RT/ZS and texture + * descriptors, make sure it fits. */ + if (surf_stride_B > UINT32_MAX) + return false; + + slayout->tiled_or_linear.surface_stride_B = surf_stride_B; + slayout->size_B = surf_stride_B * mip_extent_el.depth * props->nr_samples; + return true; +} + +#define pan_mod_linear_emit_tex_payload_entry \ + GENX(pan_tex_emit_linear_payload_entry) +#define pan_mod_linear_emit_color_attachment \ + GENX(pan_emit_linear_color_attachment) +#define pan_mod_linear_emit_zs_attachment GENX(pan_emit_linear_zs_attachment) +#define pan_mod_linear_emit_s_attachment GENX(pan_emit_linear_s_attachment) + +#if PAN_ARCH >= 5 +#define EMIT_ATT(__name) \ + .emit_color_attachment = pan_mod_##__name##_emit_color_attachment, \ + .emit_zs_attachment = pan_mod_##__name##_emit_zs_attachment, \ + .emit_s_attachment = pan_mod_##__name##_emit_s_attachment +#else +#define EMIT_ATT(__name) \ + .emit_color_attachment = NULL, .emit_zs_attachment = NULL, \ + .emit_s_attachment = NULL +#endif + +#define PAN_MOD_DEF(__name) \ + { \ + .match = pan_mod_##__name##_match, \ + .supports_format = pan_mod_##__name##_supports_format, \ + .get_wsi_row_pitch = pan_mod_##__name##_get_wsi_row_pitch, \ + .init_slice_layout = pan_mod_##__name##_init_slice_layout, \ + .emit_tex_payload_entry = pan_mod_##__name##_emit_tex_payload_entry, \ + EMIT_ATT(__name), \ + } + +static const struct pan_mod_handler pan_mod_handlers[] = { + PAN_MOD_DEF(afbc), + PAN_MOD_DEF(u_tiled), + PAN_MOD_DEF(linear), +#if PAN_ARCH >= 10 + PAN_MOD_DEF(afrc), +#endif +}; + +const struct pan_mod_handler * +GENX(pan_mod_get_handler)(uint64_t modifier) +{ + for (unsigned i = 0; i < ARRAY_SIZE(pan_mod_handlers); i++) { + if (pan_mod_handlers[i].match(modifier)) + return &pan_mod_handlers[i]; + } + + return NULL; +} diff --git a/src/panfrost/lib/pan_mod.h b/src/panfrost/lib/pan_mod.h new file mode 100644 index 00000000000..3146895d2b5 --- /dev/null +++ b/src/panfrost/lib/pan_mod.h @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2025 Collabora, Ltd. + * + * SPDX-License-Identifier: MIT + */ + +#ifndef __PAN_MOD_H +#define __PAN_MOD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "util/format/u_format.h" + +#include "pan_layout.h" + +struct pan_fb_info; +struct pan_image; +struct pan_image_view; +struct pan_mod_handler; + +struct pan_mod_handler { + bool (*match)(uint64_t mod); + bool (*supports_format)(uint64_t mod, enum pipe_format format); + + bool (*init_slice_layout)( + const struct pan_image_props *props, unsigned plane_idx, + struct pan_image_extent mip_extent_px, + const struct pan_image_layout_constraints *layout_constraints, + struct pan_image_slice_layout *slice); + uint32_t (*get_wsi_row_pitch)(const struct pan_image *image, + unsigned plane_idx, unsigned mip_level); + + void (*emit_tex_payload_entry)(const struct pan_image_view *iview, + unsigned mip_level, unsigned layer_or_z_slice, + unsigned sample, void **payload); + + void (*emit_color_attachment)(const struct pan_fb_info *fb, unsigned rt_idx, + unsigned layer_or_z_slice, + unsigned cbuf_offset, void *payload); + void (*emit_zs_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload); + void (*emit_s_attachment)(const struct pan_fb_info *fb, + unsigned layer_or_z_slice, void *payload); +}; + +#ifdef PAN_ARCH +const struct pan_mod_handler *GENX(pan_mod_get_handler)(uint64_t modifier); +#else +const struct pan_mod_handler *pan_mod_get_handler_v4(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v5(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v6(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v7(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v9(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v10(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v12(uint64_t modifier); +const struct pan_mod_handler *pan_mod_get_handler_v13(uint64_t modifier); + +static inline const struct pan_mod_handler * +pan_mod_get_handler(unsigned arch, uint64_t modifier) +{ + switch (arch) { + case 4: + return pan_mod_get_handler_v4(modifier); + case 5: + return pan_mod_get_handler_v5(modifier); + case 6: + return pan_mod_get_handler_v6(modifier); + case 7: + return pan_mod_get_handler_v7(modifier); + case 9: + return pan_mod_get_handler_v9(modifier); + case 10: + return pan_mod_get_handler_v10(modifier); + case 12: + return pan_mod_get_handler_v12(modifier); + case 13: + return pan_mod_get_handler_v13(modifier); + default: + unreachable("Unsupported arch"); + } +} +#endif + +#ifdef __cplusplus +} /* extern C */ +#endif + +#endif diff --git a/src/panfrost/lib/pan_texture.c b/src/panfrost/lib/pan_texture.c index 48b5dbaa558..cfdbf44b0ea 100644 --- a/src/panfrost/lib/pan_texture.c +++ b/src/panfrost/lib/pan_texture.c @@ -1007,7 +1007,7 @@ emit_afbc_multiplane_surface(const struct pan_image_view *iview, #if PAN_ARCH >= 9 #define PAN_TEX_EMIT_HELPER(mod) \ - static void pan_tex_emit_##mod##_payload_entry( \ + void GENX(pan_tex_emit_##mod##_payload_entry)( \ const struct pan_image_view *iview, unsigned mip_level, \ unsigned layer_or_z_slice, unsigned sample, void **payload) \ { \ @@ -1031,7 +1031,7 @@ emit_afbc_multiplane_surface(const struct pan_image_view *iview, } #elif PAN_ARCH >= 7 #define PAN_TEX_EMIT_HELPER(mod) \ - static void pan_tex_emit_##mod##_payload_entry( \ + void GENX(pan_tex_emit_##mod##_payload_entry)( \ const struct pan_image_view *iview, unsigned mip_level, \ unsigned layer_or_z_slice, unsigned sample, void **payload) \ { \ @@ -1047,7 +1047,7 @@ emit_afbc_multiplane_surface(const struct pan_image_view *iview, } #else #define PAN_TEX_EMIT_HELPER(mod) \ - static void pan_tex_emit_##mod##_payload_entry( \ + void GENX(pan_tex_emit_##mod##_payload_entry)( \ const struct pan_image_view *iview, unsigned mip_level, \ unsigned layer_or_z_slice, unsigned sample, void **payload) \ { \ @@ -1065,37 +1065,13 @@ PAN_TEX_EMIT_HELPER(afbc) PAN_TEX_EMIT_HELPER(afrc) #endif -static void -pan_tex_emit_payload_entry(const struct pan_image_view *iview, - unsigned mip_level, unsigned layer_or_z_slice, - unsigned sample, void **payload) -{ - const struct pan_image_plane_ref pref = - pan_image_view_get_first_plane(iview); - uint64_t mod = pref.image->props.modifier; - - if (drm_is_afbc(mod)) { - pan_tex_emit_afbc_payload_entry(iview, mip_level, layer_or_z_slice, - sample, payload); -#if PAN_ARCH >= 10 - } else if (drm_is_afrc(mod)) { - pan_tex_emit_afrc_payload_entry(iview, mip_level, layer_or_z_slice, - sample, payload); -#endif - } else if (mod == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { - pan_tex_emit_u_tiled_payload_entry(iview, mip_level, layer_or_z_slice, - sample, payload); - } else { - assert(mod == DRM_FORMAT_MOD_LINEAR); - pan_tex_emit_linear_payload_entry(iview, mip_level, layer_or_z_slice, - sample, payload); - } -} - static void pan_emit_iview_texture_payload(const struct pan_image_view *iview, void *payload) { + const struct pan_image_plane_ref pref = + pan_image_view_get_first_plane(iview); + const struct pan_mod_handler *mod_handler = pref.image->mod_handler; unsigned nr_samples = PAN_ARCH < 9 ? pan_image_view_get_nr_samples(iview) : 1; @@ -1111,7 +1087,8 @@ pan_emit_iview_texture_payload(const struct pan_image_view *iview, for (int sample = 0; sample < nr_samples; ++sample) { for (int level = iview->first_level; level <= iview->last_level; ++level) { - pan_tex_emit_payload_entry(iview, level, layer, sample, &payload); + mod_handler->emit_tex_payload_entry(iview, level, layer, sample, + &payload); } } } @@ -1134,7 +1111,7 @@ pan_emit_iview_texture_payload(const struct pan_image_view *iview, */ for (int face = 0; face < face_count; ++face) { for (int sample = 0; sample < nr_samples; ++sample) { - pan_tex_emit_payload_entry( + mod_handler->emit_tex_payload_entry( iview, level, (face_count * layer) + face, sample, &payload); } } diff --git a/src/panfrost/lib/pan_texture.h b/src/panfrost/lib/pan_texture.h index ad0e168d747..0b1624471c6 100644 --- a/src/panfrost/lib/pan_texture.h +++ b/src/panfrost/lib/pan_texture.h @@ -53,12 +53,44 @@ void GENX(pan_sampled_texture_emit)(const struct pan_image_view *iview, struct mali_texture_packed *out, const struct pan_ptr *payload); +void GENX(pan_tex_emit_linear_payload_entry)(const struct pan_image_view *iview, + unsigned mip_level, + unsigned layer_or_z_slice, + unsigned sample, void **payload); + +void GENX(pan_tex_emit_u_tiled_payload_entry)( + const struct pan_image_view *iview, unsigned mip_level, + unsigned layer_or_z_slice, unsigned sample, void **payload); + +void +GENX(pan_tex_emit_afbc_payload_entry)(const struct pan_image_view *iview, + unsigned mip_level, unsigned layer_or_z_slice, + unsigned sample, void **payload); + #if PAN_ARCH >= 9 void GENX(pan_storage_texture_emit)(const struct pan_image_view *iview, struct mali_texture_packed *out, const struct pan_ptr *payload); #endif +void GENX(pan_tex_emit_linear_payload_entry)(const struct pan_image_view *iview, + unsigned mip_level, + unsigned layer_or_z_slice, + unsigned sample, void **payload); +void GENX(pan_tex_emit_u_tiled_payload_entry)( + const struct pan_image_view *iview, unsigned mip_level, + unsigned layer_or_z_slice, unsigned sample, void **payload); +void GENX(pan_tex_emit_afbc_payload_entry)(const struct pan_image_view *iview, + unsigned mip_level, + unsigned layer_or_z_slice, + unsigned sample, void **payload); + +#if PAN_ARCH >= 10 +void GENX(pan_tex_emit_afrc_payload_entry)( + const struct pan_image_view *iview, unsigned mip_level, + unsigned layer_or_z_slice, unsigned sample, void **payload); +#endif + void GENX(pan_buffer_texture_emit)(const struct pan_buffer_view *bview, struct mali_texture_packed *out, diff --git a/src/panfrost/lib/tests/test-layout.cpp b/src/panfrost/lib/tests/test-layout.cpp index ca215c1e533..7e10776ce49 100644 --- a/src/panfrost/lib/tests/test-layout.cpp +++ b/src/panfrost/lib/tests/test-layout.cpp @@ -26,6 +26,7 @@ #include "pan_format.h" #include "pan_image.h" #include "pan_layout.h" +#include "pan_mod.h" #include @@ -240,6 +241,21 @@ TEST(AFBCStride, Tiled) } } +static bool +layout_init(unsigned arch, const struct pan_image_props *props, + unsigned plane_idx, + const struct pan_image_layout_constraints *layout_constraints, + struct pan_image_layout *layout) +{ + /* Pick the first supported arch if it's zero. */ + if (!arch) + arch = 4; + + return pan_image_layout_init(arch, + pan_mod_get_handler(arch, props->modifier), + props, plane_idx, layout_constraints, layout); +} + /* dEQP-GLES3.functional.texture.format.compressed.etc1_2d_pot */ TEST(Layout, ImplicitLayoutInterleavedETC2) { @@ -260,7 +276,7 @@ TEST(Layout, ImplicitLayoutInterleavedETC2) unsigned offsets[9] = {0, 8192, 10240, 10752, 10880, 11008, 11136, 11264, 11392}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); for (unsigned i = 0; i < 8; ++i) { unsigned size = (offsets[i + 1] - offsets[i]); @@ -289,7 +305,7 @@ TEST(Layout, ImplicitLayoutInterleavedASTC5x5) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); /* The image is 50x50 pixels, with 5x5 blocks. So it is a 10x10 grid of ASTC * blocks. 4x4 tiles of ASTC blocks are u-interleaved, so we have to round up @@ -319,7 +335,7 @@ TEST(Layout, ImplicitLayoutLinearASTC5x5) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); /* The image is 50x50 pixels, with 5x5 blocks. So it is a 10x10 grid of ASTC * blocks. Each ASTC block is 16 bytes, so the row stride is 160 bytes, @@ -352,7 +368,7 @@ TEST(AFBCLayout, Linear3D) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); /* AFBC Surface stride is bytes between consecutive surface headers, which is * the header size since this is a 3D texture. At superblock size 16x16, the @@ -396,7 +412,7 @@ TEST(AFBCLayout, Tiled16x16) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); /* The image is 917x417. Superblocks are 16x16, so there are 58x27 * superblocks. Superblocks are grouped into 8x8 tiles, so there are 8x4 @@ -438,7 +454,7 @@ TEST(AFBCLayout, Linear16x16Minimal) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); /* Image is 1x1 to test for correct alignment everywhere. */ EXPECT_EQ(l.slices[0].offset_B, 0); @@ -469,7 +485,7 @@ TEST(AFBCLayout, Linear16x16Minimalv6) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(6, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(6, &p, 0, NULL, &l)); /* Image is 1x1 to test for correct alignment everywhere. */ EXPECT_EQ(l.slices[0].offset_B, 0); @@ -501,7 +517,7 @@ TEST(AFBCLayout, Tiled16x16Minimal) }; struct pan_image_layout l = {}; - ASSERT_TRUE(pan_image_layout_init(0, &p, 0, NULL, &l)); + ASSERT_TRUE(layout_init(0, &p, 0, NULL, &l)); /* Image is 1x1 to test for correct alignment everywhere. */ EXPECT_EQ(l.slices[0].offset_B, 0); @@ -524,8 +540,8 @@ static unsigned archs[] = {4, 5, 6, 7, 9, 12, 13}; #define EXPECT_IMPORT_SUCCESS(__arch, __iprops, __plane, __wsi_layout, \ __out_layout, __test_desc) \ do { \ - bool __result = pan_image_layout_init(__arch, __iprops, __plane, \ - __wsi_layout, __out_layout); \ + bool __result = \ + layout_init(__arch, __iprops, __plane, __wsi_layout, __out_layout); \ EXPECT_TRUE(__result) \ << __test_desc \ << " for format) \ @@ -536,8 +552,16 @@ static unsigned archs[] = {4, 5, 6, 7, 9, 12, 13}; if (!__result) \ break; \ \ + struct pan_image_plane img_plane = { \ + .layout = *(__out_layout), \ + }; \ + struct pan_image img = { \ + .props = *(__iprops), \ + .mod_handler = pan_mod_get_handler(__arch, (__iprops)->modifier), \ + }; \ + img.planes[__plane] = &img_plane; \ unsigned __export_row_pitch_B = \ - pan_image_get_wsi_row_pitch(&iprops, __plane, &layout, 0); \ + pan_image_get_wsi_row_pitch(&img, __plane, 0); \ unsigned __export_offset_B = pan_image_get_wsi_offset(&layout, 0); \ EXPECT_TRUE(__export_row_pitch_B == (__wsi_layout)->wsi_row_pitch_B && \ __export_offset_B == (__wsi_layout)->offset_B) \ @@ -549,8 +573,8 @@ static unsigned archs[] = {4, 5, 6, 7, 9, 12, 13}; #define EXPECT_IMPORT_FAIL(__arch, __iprops, __plane, __wsi_layout, \ __out_layout, __test_desc) \ - EXPECT_FALSE(pan_image_layout_init(__arch, __iprops, __plane, __wsi_layout, \ - __out_layout)) \ + EXPECT_FALSE( \ + layout_init(__arch, __iprops, __plane, __wsi_layout, __out_layout)) \ << __test_desc \ << " for format) \ << ",plane=" << __plane << ",mod=" << std::hex << (__iprops)->modifier \ diff --git a/src/panfrost/vulkan/panvk_image.c b/src/panfrost/vulkan/panvk_image.c index 4fbc32ca29b..74c140c5091 100644 --- a/src/panfrost/vulkan/panvk_image.c +++ b/src/panfrost/vulkan/panvk_image.c @@ -247,6 +247,8 @@ panvk_image_init_layouts(struct panvk_image *image, if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) image->plane_count = 2; + const struct pan_mod_handler *mod_handler = + pan_mod_get_handler(arch, image->vk.drm_format_mod); struct pan_image_layout_constraints plane_layout = { .offset_B = 0, }; @@ -281,12 +283,13 @@ panvk_image_init_layouts(struct panvk_image *image, .nr_samples = image->vk.samples, .nr_slices = image->vk.mip_levels, }, + .mod_handler = mod_handler, .planes = {&image->planes[plane].plane}, }; - if (!pan_image_layout_init(arch, &image->planes[plane].image.props, 0, - &plane_layout, - &image->planes[plane].plane.layout)) { + if (!pan_image_layout_init( + arch, mod_handler, &image->planes[plane].image.props, 0, + &plane_layout, &image->planes[plane].plane.layout)) { return panvk_error(image->vk.base.device, VK_ERROR_INITIALIZATION_FAILED); }