diff --git a/src/gallium/drivers/panfrost/pan_cmdstream.c b/src/gallium/drivers/panfrost/pan_cmdstream.c index f04f35e684e..ff5f6b190e9 100644 --- a/src/gallium/drivers/panfrost/pan_cmdstream.c +++ b/src/gallium/drivers/panfrost/pan_cmdstream.c @@ -1315,8 +1315,8 @@ panfrost_create_sampler_view_bo(struct panfrost_sampler_view *so, }; unsigned size = - (PAN_ARCH <= 5 ? pan_size(MIDGARD_TEXTURE) : 0) + - panfrost_estimate_texture_payload_size(device, &iview); + (PAN_ARCH <= 5 ? pan_size(TEXTURE) : 0) + + GENX(panfrost_estimate_texture_payload_size)(&iview); struct panfrost_ptr payload = pan_pool_alloc_aligned(&ctx->descs.base, size, 64); so->state = panfrost_pool_take_ref(&ctx->descs, payload.gpu); @@ -1328,7 +1328,7 @@ panfrost_create_sampler_view_bo(struct panfrost_sampler_view *so, payload.gpu += pan_size(MIDGARD_TEXTURE); } - panfrost_new_texture(device, &iview, tex, &payload); + GENX(panfrost_new_texture)(device, &iview, tex, &payload); } static void diff --git a/src/panfrost/lib/meson.build b/src/panfrost/lib/meson.build index 522149466b3..8658637f607 100644 --- a/src/panfrost/lib/meson.build +++ b/src/panfrost/lib/meson.build @@ -60,6 +60,7 @@ foreach ver : ['4', '5', '6', '7'] 'pan_cs.c', 'pan_indirect_dispatch.c', 'pan_indirect_draw.c', + 'pan_texture.c', ], include_directories : [inc_include, inc_src, inc_mapi, inc_mesa, inc_gallium, inc_gallium_aux, inc_panfrost_hw], c_args : ['-DPAN_ARCH=' + ver], diff --git a/src/panfrost/lib/pan_blitter.c b/src/panfrost/lib/pan_blitter.c index abb375b6528..69cc967dd49 100644 --- a/src/panfrost/lib/pan_blitter.c +++ b/src/panfrost/lib/pan_blitter.c @@ -883,12 +883,12 @@ pan_blitter_emit_textures(struct pan_pool *pool, for (unsigned i = 0; i < tex_count; i++) { void *texture = textures.cpu + (pan_size(TEXTURE) * i); size_t payload_size = - panfrost_estimate_texture_payload_size(pool->dev, views[i]); + GENX(panfrost_estimate_texture_payload_size)(views[i]); struct panfrost_ptr surfaces = pan_pool_alloc_aligned(pool, payload_size, pan_alignment(SURFACE_WITH_STRIDE)); - panfrost_new_texture(pool->dev, views[i], texture, &surfaces); + GENX(panfrost_new_texture)(pool->dev, views[i], texture, &surfaces); } return textures.gpu; @@ -897,7 +897,7 @@ pan_blitter_emit_textures(struct pan_pool *pool, for (unsigned i = 0; i < tex_count; i++) { size_t sz = pan_size(TEXTURE) + - panfrost_estimate_texture_payload_size(pool->dev, views[i]); + GENX(panfrost_estimate_texture_payload_size)(views[i]); struct panfrost_ptr texture = pan_pool_alloc_aligned(pool, sz, pan_alignment(TEXTURE)); struct panfrost_ptr surfaces = { @@ -905,7 +905,7 @@ pan_blitter_emit_textures(struct pan_pool *pool, .gpu = texture.gpu + pan_size(TEXTURE), }; - panfrost_new_texture(pool->dev, views[i], texture.cpu, &surfaces); + GENX(panfrost_new_texture)(pool->dev, views[i], texture.cpu, &surfaces); textures[i] = texture.gpu; } diff --git a/src/panfrost/lib/pan_texture.c b/src/panfrost/lib/pan_texture.c index 49da5340d0a..226f98a4be6 100644 --- a/src/panfrost/lib/pan_texture.c +++ b/src/panfrost/lib/pan_texture.c @@ -30,6 +30,8 @@ #include "pan_texture.h" #include "panfrost-quirks.h" +#ifndef PAN_ARCH + /* Generates a texture descriptor. Ideally, descriptors are immutable after the * texture is created, so we can keep these hanging around in GPU memory in a * dedicated BO and not have to worry. In practice there are some minor gotchas @@ -57,165 +59,6 @@ uint64_t pan_best_modifiers[PAN_MODIFIER_COUNT] = { DRM_FORMAT_MOD_LINEAR }; -/* Check if we need to set a custom stride by computing the "expected" - * stride and comparing it to what the user actually wants. Only applies - * to linear textures, since tiled/compressed textures have strict - * alignment requirements for their strides as it is */ - -static bool -panfrost_needs_explicit_stride(const struct panfrost_device *dev, - const struct pan_image_view *iview) -{ - /* Stride is explicit on Bifrost */ - if (pan_is_bifrost(dev)) - return true; - - if (iview->image->layout.modifier != DRM_FORMAT_MOD_LINEAR) - return false; - - unsigned bytes_per_block = util_format_get_blocksize(iview->format); - unsigned block_w = util_format_get_blockwidth(iview->format); - - for (unsigned l = iview->first_level; l <= iview->last_level; ++l) { - unsigned actual = iview->image->layout.slices[l].line_stride; - unsigned expected = - DIV_ROUND_UP(u_minify(iview->image->layout.width, l), block_w) * - bytes_per_block; - - if (actual != expected) - return true; - } - - return false; -} - -/* A Scalable Texture Compression (ASTC) corresponds to just a few texture type - * in the hardware, but in fact can be parametrized to have various widths and - * heights for the so-called "stretch factor". It turns out these parameters - * are stuffed in the bottom bits of the payload pointers. This functions - * computes these magic stuffing constants based on the ASTC format in use. The - * constant in a given dimension is 3-bits, and two are stored side-by-side for - * each active dimension. - */ - -static unsigned -panfrost_astc_stretch(unsigned dim) -{ - assert(dim >= 4 && dim <= 12); - return MIN2(dim, 11) - 4; -} - -/* Texture addresses are tagged with information about compressed formats. - * AFBC uses a bit for whether the colorspace transform is enabled (RGB and - * RGBA only). - * For ASTC, this is a "stretch factor" encoding the block size. */ - -static unsigned -panfrost_compression_tag(const struct panfrost_device *dev, - const struct util_format_description *desc, - enum mali_texture_dimension dim, - uint64_t modifier) -{ - if (drm_is_afbc(modifier)) { - unsigned flags = (modifier & AFBC_FORMAT_MOD_YTR) ? - MALI_AFBC_SURFACE_FLAG_YTR : 0; - - if (!pan_is_bifrost(dev)) - return flags; - - /* Prefetch enable */ - flags |= MALI_AFBC_SURFACE_FLAG_PREFETCH; - - /* Wide blocks (> 16x16) */ - if (panfrost_block_dim(modifier, true, 0) > 16) - flags |= MALI_AFBC_SURFACE_FLAG_WIDE_BLOCK; - - /* Used to make sure AFBC headers don't point outside the AFBC - * body. HW is using the AFBC surface stride to do this check, - * which doesn't work for 3D textures because the surface - * stride does not cover the body. Only supported on v7+. - */ - if (dev->arch >= 7 && dim != MALI_TEXTURE_DIMENSION_3D) - flags |= MALI_AFBC_SURFACE_FLAG_CHECK_PAYLOAD_RANGE; - - return flags; - } else if (desc->layout == UTIL_FORMAT_LAYOUT_ASTC) { - return (panfrost_astc_stretch(desc->block.height) << 3) | - panfrost_astc_stretch(desc->block.width); - } else { - return 0; - } -} - - -/* Cubemaps have 6 faces as "layers" in between each actual layer. We - * need to fix this up. TODO: logic wrong in the asserted out cases ... - * can they happen, perhaps from cubemap arrays? */ - -static void -panfrost_adjust_cube_dimensions( - unsigned *first_face, unsigned *last_face, - unsigned *first_layer, unsigned *last_layer) -{ - *first_face = *first_layer % 6; - *last_face = *last_layer % 6; - *first_layer /= 6; - *last_layer /= 6; - - assert((*first_layer == *last_layer) || (*first_face == 0 && *last_face == 5)); -} - -/* Following the texture descriptor is a number of pointers. How many? */ - -static unsigned -panfrost_texture_num_elements( - unsigned first_level, unsigned last_level, - unsigned first_layer, unsigned last_layer, - unsigned nr_samples, - bool is_cube, bool manual_stride) -{ - unsigned first_face = 0, last_face = 0; - - if (is_cube) { - panfrost_adjust_cube_dimensions(&first_face, &last_face, - &first_layer, &last_layer); - } - - unsigned levels = 1 + last_level - first_level; - unsigned layers = 1 + last_layer - first_layer; - unsigned faces = 1 + last_face - first_face; - unsigned num_elements = levels * layers * faces * MAX2(nr_samples, 1); - - if (manual_stride) - num_elements *= 2; - - return num_elements; -} - -/* Conservative estimate of the size of the texture payload a priori. - * Average case, size equal to the actual size. Worst case, off by 2x (if - * a manual stride is not needed on a linear texture). Returned value - * must be greater than or equal to the actual size, so it's safe to use - * as an allocation amount */ - -unsigned -panfrost_estimate_texture_payload_size(const struct panfrost_device *dev, - const struct pan_image_view *iview) -{ - /* Assume worst case */ - unsigned manual_stride = pan_is_bifrost(dev) || - (iview->image->layout.modifier == DRM_FORMAT_MOD_LINEAR); - - unsigned elements = - panfrost_texture_num_elements(iview->first_level, iview->last_level, - iview->first_layer, iview->last_layer, - iview->image->layout.nr_samples, - iview->dim == MALI_TEXTURE_DIMENSION_CUBE, - manual_stride); - - return sizeof(mali_ptr) * elements; -} - /* If not explicitly, line stride is calculated for block-based formats as * (ceil(width / block_width) * block_size). As a special case, this is left * zero if there is only a single block vertically. So, we have a helper to @@ -245,278 +88,6 @@ panfrost_block_dim(uint64_t modifier, bool width, unsigned plane) } } -static void -panfrost_get_surface_strides(const struct panfrost_device *dev, - const struct pan_image_layout *layout, - unsigned l, - int32_t *row_stride, int32_t *surf_stride) -{ - const struct pan_image_slice_layout *slice = &layout->slices[l]; - - if (drm_is_afbc(layout->modifier)) { - /* Pre v7 don't have a row stride field. This field is - * repurposed as a Y offset which we don't use */ - *row_stride = dev->arch < 7 ? 0 : slice->afbc.row_stride; - *surf_stride = slice->afbc.surface_stride; - } else { - *row_stride = slice->row_stride; - *surf_stride = slice->surface_stride; - } -} - -static mali_ptr -panfrost_get_surface_pointer(const struct pan_image_layout *layout, - enum mali_texture_dimension dim, - mali_ptr base, - unsigned l, unsigned w, unsigned f, unsigned s) -{ - unsigned face_mult = dim == MALI_TEXTURE_DIMENSION_CUBE ? 6 : 1; - unsigned offset; - - if (layout->dim == MALI_TEXTURE_DIMENSION_3D) { - assert(!f && !s); - offset = layout->slices[l].offset + - (w * panfrost_get_layer_stride(layout, l)); - } else { - offset = panfrost_texture_offset(layout, l, (w * face_mult) + f, s); - } - - return base + offset; -} - -struct panfrost_surface_iter { - unsigned layer, last_layer; - unsigned level, first_level, last_level; - unsigned face, first_face, last_face; - unsigned sample, first_sample, last_sample; -}; - -static void -panfrost_surface_iter_begin(struct panfrost_surface_iter *iter, - unsigned first_layer, unsigned last_layer, - unsigned first_level, unsigned last_level, - unsigned first_face, unsigned last_face, - unsigned nr_samples) -{ - iter->layer = first_layer; - iter->last_layer = last_layer; - iter->level = iter->first_level = first_level; - iter->last_level = last_level; - iter->face = iter->first_face = first_face; - iter->last_face = last_face; - iter->sample = iter->first_sample = 0; - iter->last_sample = nr_samples - 1; -} - -static bool -panfrost_surface_iter_end(const struct panfrost_surface_iter *iter) -{ - return iter->layer > iter->last_layer; -} - -static void -panfrost_surface_iter_next(const struct panfrost_device *dev, - struct panfrost_surface_iter *iter) -{ -#define INC_TEST(field) \ - do { \ - if (iter->field++ < iter->last_ ## field) \ - return; \ - iter->field = iter->first_ ## field; \ - } while (0) - - /* Ordering is different on v7: inner loop is iterating on levels */ - if (dev->arch >= 7) - INC_TEST(level); - - INC_TEST(sample); - INC_TEST(face); - - if (dev->arch < 7) - INC_TEST(level); - - iter->layer++; - -#undef INC_TEST -} - -static void -panfrost_emit_texture_payload(const struct panfrost_device *dev, - const struct pan_image_view *iview, - enum pipe_format format, - bool manual_stride, - void *payload) -{ - const struct pan_image_layout *layout = &iview->image->layout; - const struct util_format_description *desc = - util_format_description(format); - - mali_ptr base = iview->image->data.bo->ptr.gpu + iview->image->data.offset; - - if (iview->buf.size) { - assert (iview->dim == MALI_TEXTURE_DIMENSION_1D); - base += iview->buf.offset; - } - - /* panfrost_compression_tag() wants the dimension of the resource, not the - * one of the image view (those might differ). - */ - base |= panfrost_compression_tag(dev, desc, layout->dim, layout->modifier); - - /* Inject the addresses in, interleaving array indices, mip levels, - * cube faces, and strides in that order */ - - unsigned first_layer = iview->first_layer, last_layer = iview->last_layer; - unsigned nr_samples = layout->nr_samples; - unsigned first_face = 0, last_face = 0; - - if (iview->dim == MALI_TEXTURE_DIMENSION_CUBE) { - panfrost_adjust_cube_dimensions(&first_face, &last_face, - &first_layer, &last_layer); - } - - struct panfrost_surface_iter iter; - - for (panfrost_surface_iter_begin(&iter, first_layer, last_layer, - iview->first_level, iview->last_level, - first_face, last_face, nr_samples); - !panfrost_surface_iter_end(&iter); - panfrost_surface_iter_next(dev, &iter)) { - mali_ptr pointer = - panfrost_get_surface_pointer(layout, iview->dim, base, - iter.level, iter.layer, - iter.face, iter.sample); - - if (!manual_stride) { - pan_pack(payload, SURFACE, cfg) { - cfg.pointer = pointer; - } - payload += pan_size(SURFACE); - } else { - pan_pack(payload, SURFACE_WITH_STRIDE, cfg) { - cfg.pointer = pointer; - panfrost_get_surface_strides(dev, layout, iter.level, - &cfg.row_stride, - &cfg.surface_stride); - } - payload += pan_size(SURFACE_WITH_STRIDE); - } - } -} - -/* Map modifiers to mali_texture_layout for packing in a texture descriptor */ - -static enum mali_texture_layout -panfrost_modifier_to_layout(uint64_t modifier) -{ - if (drm_is_afbc(modifier)) - return MALI_TEXTURE_LAYOUT_AFBC; - else if (modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) - return MALI_TEXTURE_LAYOUT_TILED; - else if (modifier == DRM_FORMAT_MOD_LINEAR) - return MALI_TEXTURE_LAYOUT_LINEAR; - else - unreachable("Invalid modifer"); -} - -void -panfrost_new_texture(const struct panfrost_device *dev, - const struct pan_image_view *iview, - void *out, const struct panfrost_ptr *payload) -{ - const struct pan_image_layout *layout = &iview->image->layout; - enum pipe_format format = iview->format; - unsigned swizzle; - - if (dev->arch == 7 && util_format_is_depth_or_stencil(format)) { - /* v7 doesn't have an _RRRR component order, combine the - * user swizzle with a .XXXX swizzle to emulate that. - */ - static const unsigned char replicate_x[4] = { - PIPE_SWIZZLE_X, PIPE_SWIZZLE_X, - PIPE_SWIZZLE_X, PIPE_SWIZZLE_X, - }; - unsigned char patched_swizzle[4]; - - util_format_compose_swizzles(replicate_x, - iview->swizzle, - patched_swizzle); - swizzle = panfrost_translate_swizzle_4(patched_swizzle); - } else { - swizzle = panfrost_translate_swizzle_4(iview->swizzle); - } - - bool manual_stride = - panfrost_needs_explicit_stride(dev, iview); - - panfrost_emit_texture_payload(dev, iview, format, - manual_stride, - payload->cpu); - - unsigned array_size = iview->last_layer - iview->first_layer + 1; - - if (iview->dim == MALI_TEXTURE_DIMENSION_CUBE) { - assert(iview->first_layer % 6 == 0); - assert(iview->last_layer % 6 == 5); - array_size /= 6; - } - - unsigned width; - - if (iview->buf.size) { - assert(iview->dim == MALI_TEXTURE_DIMENSION_1D); - assert(!iview->first_level && !iview->last_level); - assert(!iview->first_layer && !iview->last_layer); - assert(layout->nr_samples == 1); - assert(layout->height == 1 && layout->depth == 1); - assert(iview->buf.offset + iview->buf.size <= layout->width); - width = iview->buf.size; - } else { - width = u_minify(layout->width, iview->first_level); - } - - if (pan_is_bifrost(dev)) { - pan_pack(out, BIFROST_TEXTURE, cfg) { - cfg.dimension = iview->dim; - cfg.format = dev->formats[format].hw; - cfg.width = width; - cfg.height = u_minify(layout->height, iview->first_level); - if (iview->dim == MALI_TEXTURE_DIMENSION_3D) - cfg.depth = u_minify(layout->depth, iview->first_level); - else - cfg.sample_count = layout->nr_samples; - cfg.swizzle = swizzle; - cfg.texel_ordering = - panfrost_modifier_to_layout(layout->modifier); - cfg.levels = iview->last_level - iview->first_level + 1; - cfg.array_size = array_size; - cfg.surfaces = payload->gpu; - - /* We specify API-level LOD clamps in the sampler descriptor - * and use these clamps simply for bounds checking */ - cfg.minimum_lod = FIXED_16(0, false); - cfg.maximum_lod = FIXED_16(cfg.levels - 1, false); - } - } else { - pan_pack(out, MIDGARD_TEXTURE, cfg) { - cfg.width = width; - cfg.height = u_minify(layout->height, iview->first_level); - if (iview->dim == MALI_TEXTURE_DIMENSION_3D) - cfg.depth = u_minify(layout->depth, iview->first_level); - else - cfg.sample_count = layout->nr_samples; - cfg.array_size = array_size; - cfg.format = panfrost_pipe_format_v6[format].hw; - cfg.dimension = iview->dim; - cfg.texel_ordering = - panfrost_modifier_to_layout(layout->modifier); - cfg.manual_stride = manual_stride; - cfg.levels = iview->last_level - iview->first_level + 1; - cfg.swizzle = swizzle; - }; - } -} - /* Computes sizes for checksumming, which is 8 bytes per 16x16 tile. * Checksumming is believed to be a CRC variant (CRC64 based on the size?). * This feature is also known as "transaction elimination". */ @@ -774,3 +345,418 @@ pan_iview_get_surface(const struct pan_image_view *iview, array_idx, surface_idx); } } + +#else /* ifndef PAN_ARCH */ + +/* A Scalable Texture Compression (ASTC) corresponds to just a few texture type + * in the hardware, but in fact can be parametrized to have various widths and + * heights for the so-called "stretch factor". It turns out these parameters + * are stuffed in the bottom bits of the payload pointers. This functions + * computes these magic stuffing constants based on the ASTC format in use. The + * constant in a given dimension is 3-bits, and two are stored side-by-side for + * each active dimension. + */ + +static unsigned +panfrost_astc_stretch(unsigned dim) +{ + assert(dim >= 4 && dim <= 12); + return MIN2(dim, 11) - 4; +} + +/* Texture addresses are tagged with information about compressed formats. + * AFBC uses a bit for whether the colorspace transform is enabled (RGB and + * RGBA only). + * For ASTC, this is a "stretch factor" encoding the block size. */ + +static unsigned +panfrost_compression_tag(const struct util_format_description *desc, + enum mali_texture_dimension dim, + uint64_t modifier) +{ + if (drm_is_afbc(modifier)) { + unsigned flags = (modifier & AFBC_FORMAT_MOD_YTR) ? + MALI_AFBC_SURFACE_FLAG_YTR : 0; + +#if PAN_ARCH >= 6 + /* Prefetch enable */ + flags |= MALI_AFBC_SURFACE_FLAG_PREFETCH; + + /* Wide blocks (> 16x16) */ + if (panfrost_block_dim(modifier, true, 0) > 16) + flags |= MALI_AFBC_SURFACE_FLAG_WIDE_BLOCK; + + /* Used to make sure AFBC headers don't point outside the AFBC + * body. HW is using the AFBC surface stride to do this check, + * which doesn't work for 3D textures because the surface + * stride does not cover the body. Only supported on v7+. + */ +#endif + +#if PAN_ARCH >= 7 + if (dim != MALI_TEXTURE_DIMENSION_3D) + flags |= MALI_AFBC_SURFACE_FLAG_CHECK_PAYLOAD_RANGE; +#endif + + return flags; + } else if (desc->layout == UTIL_FORMAT_LAYOUT_ASTC) { + return (panfrost_astc_stretch(desc->block.height) << 3) | + panfrost_astc_stretch(desc->block.width); + } else { + return 0; + } +} + +/* Cubemaps have 6 faces as "layers" in between each actual layer. We + * need to fix this up. TODO: logic wrong in the asserted out cases ... + * can they happen, perhaps from cubemap arrays? */ + +static void +panfrost_adjust_cube_dimensions( + unsigned *first_face, unsigned *last_face, + unsigned *first_layer, unsigned *last_layer) +{ + *first_face = *first_layer % 6; + *last_face = *last_layer % 6; + *first_layer /= 6; + *last_layer /= 6; + + assert((*first_layer == *last_layer) || (*first_face == 0 && *last_face == 5)); +} + +/* Following the texture descriptor is a number of pointers. How many? */ + +static unsigned +panfrost_texture_num_elements( + unsigned first_level, unsigned last_level, + unsigned first_layer, unsigned last_layer, + unsigned nr_samples, + bool is_cube, bool manual_stride) +{ + unsigned first_face = 0, last_face = 0; + + if (is_cube) { + panfrost_adjust_cube_dimensions(&first_face, &last_face, + &first_layer, &last_layer); + } + + unsigned levels = 1 + last_level - first_level; + unsigned layers = 1 + last_layer - first_layer; + unsigned faces = 1 + last_face - first_face; + unsigned num_elements = levels * layers * faces * MAX2(nr_samples, 1); + + if (manual_stride) + num_elements *= 2; + + return num_elements; +} + +/* Conservative estimate of the size of the texture payload a priori. + * Average case, size equal to the actual size. Worst case, off by 2x (if + * a manual stride is not needed on a linear texture). Returned value + * must be greater than or equal to the actual size, so it's safe to use + * as an allocation amount */ + +unsigned +GENX(panfrost_estimate_texture_payload_size)(const struct pan_image_view *iview) +{ + /* Assume worst case */ + unsigned manual_stride = PAN_ARCH >= 6 || + (iview->image->layout.modifier == DRM_FORMAT_MOD_LINEAR); + + unsigned elements = + panfrost_texture_num_elements(iview->first_level, iview->last_level, + iview->first_layer, iview->last_layer, + iview->image->layout.nr_samples, + iview->dim == MALI_TEXTURE_DIMENSION_CUBE, + manual_stride); + + return sizeof(mali_ptr) * elements; +} + +struct panfrost_surface_iter { + unsigned layer, last_layer; + unsigned level, first_level, last_level; + unsigned face, first_face, last_face; + unsigned sample, first_sample, last_sample; +}; + +static void +panfrost_surface_iter_begin(struct panfrost_surface_iter *iter, + unsigned first_layer, unsigned last_layer, + unsigned first_level, unsigned last_level, + unsigned first_face, unsigned last_face, + unsigned nr_samples) +{ + iter->layer = first_layer; + iter->last_layer = last_layer; + iter->level = iter->first_level = first_level; + iter->last_level = last_level; + iter->face = iter->first_face = first_face; + iter->last_face = last_face; + iter->sample = iter->first_sample = 0; + iter->last_sample = nr_samples - 1; +} + +static bool +panfrost_surface_iter_end(const struct panfrost_surface_iter *iter) +{ + return iter->layer > iter->last_layer; +} + +static void +panfrost_surface_iter_next(struct panfrost_surface_iter *iter) +{ +#define INC_TEST(field) \ + do { \ + if (iter->field++ < iter->last_ ## field) \ + return; \ + iter->field = iter->first_ ## field; \ + } while (0) + + /* Ordering is different on v7: inner loop is iterating on levels */ + if (PAN_ARCH >= 7) + INC_TEST(level); + + INC_TEST(sample); + INC_TEST(face); + + if (PAN_ARCH < 7) + INC_TEST(level); + + iter->layer++; + +#undef INC_TEST +} + +static void +panfrost_get_surface_strides(const struct pan_image_layout *layout, + unsigned l, + int32_t *row_stride, int32_t *surf_stride) +{ + const struct pan_image_slice_layout *slice = &layout->slices[l]; + + if (drm_is_afbc(layout->modifier)) { + /* Pre v7 don't have a row stride field. This field is + * repurposed as a Y offset which we don't use */ + *row_stride = PAN_ARCH < 7 ? 0 : slice->afbc.row_stride; + *surf_stride = slice->afbc.surface_stride; + } else { + *row_stride = slice->row_stride; + *surf_stride = slice->surface_stride; + } +} + +static mali_ptr +panfrost_get_surface_pointer(const struct pan_image_layout *layout, + enum mali_texture_dimension dim, + mali_ptr base, + unsigned l, unsigned w, unsigned f, unsigned s) +{ + unsigned face_mult = dim == MALI_TEXTURE_DIMENSION_CUBE ? 6 : 1; + unsigned offset; + + if (layout->dim == MALI_TEXTURE_DIMENSION_3D) { + assert(!f && !s); + offset = layout->slices[l].offset + + (w * panfrost_get_layer_stride(layout, l)); + } else { + offset = panfrost_texture_offset(layout, l, (w * face_mult) + f, s); + } + + return base + offset; +} + +static void +panfrost_emit_texture_payload(const struct pan_image_view *iview, + enum pipe_format format, + bool manual_stride, + void *payload) +{ + const struct pan_image_layout *layout = &iview->image->layout; + const struct util_format_description *desc = + util_format_description(format); + + mali_ptr base = iview->image->data.bo->ptr.gpu + iview->image->data.offset; + + if (iview->buf.size) { + assert (iview->dim == MALI_TEXTURE_DIMENSION_1D); + base += iview->buf.offset; + } + + /* panfrost_compression_tag() wants the dimension of the resource, not the + * one of the image view (those might differ). + */ + base |= panfrost_compression_tag(desc, layout->dim, layout->modifier); + + /* Inject the addresses in, interleaving array indices, mip levels, + * cube faces, and strides in that order */ + + unsigned first_layer = iview->first_layer, last_layer = iview->last_layer; + unsigned nr_samples = layout->nr_samples; + unsigned first_face = 0, last_face = 0; + + if (iview->dim == MALI_TEXTURE_DIMENSION_CUBE) { + panfrost_adjust_cube_dimensions(&first_face, &last_face, + &first_layer, &last_layer); + } + + struct panfrost_surface_iter iter; + + for (panfrost_surface_iter_begin(&iter, first_layer, last_layer, + iview->first_level, iview->last_level, + first_face, last_face, nr_samples); + !panfrost_surface_iter_end(&iter); + panfrost_surface_iter_next(&iter)) { + mali_ptr pointer = + panfrost_get_surface_pointer(layout, iview->dim, base, + iter.level, iter.layer, + iter.face, iter.sample); + + if (!manual_stride) { + pan_pack(payload, SURFACE, cfg) { + cfg.pointer = pointer; + } + payload += pan_size(SURFACE); + } else { + pan_pack(payload, SURFACE_WITH_STRIDE, cfg) { + cfg.pointer = pointer; + panfrost_get_surface_strides(layout, iter.level, + &cfg.row_stride, + &cfg.surface_stride); + } + payload += pan_size(SURFACE_WITH_STRIDE); + } + } +} + +/* Check if we need to set a custom stride by computing the "expected" + * stride and comparing it to what the user actually wants. Only applies + * to linear textures, since tiled/compressed textures have strict + * alignment requirements for their strides as it is */ + +static bool +panfrost_needs_explicit_stride(const struct pan_image_view *iview) +{ + /* Stride is explicit on Bifrost */ + if (PAN_ARCH >= 6) + return true; + + if (iview->image->layout.modifier != DRM_FORMAT_MOD_LINEAR) + return false; + + unsigned bytes_per_block = util_format_get_blocksize(iview->format); + unsigned block_w = util_format_get_blockwidth(iview->format); + + for (unsigned l = iview->first_level; l <= iview->last_level; ++l) { + unsigned actual = iview->image->layout.slices[l].line_stride; + unsigned expected = + DIV_ROUND_UP(u_minify(iview->image->layout.width, l), block_w) * + bytes_per_block; + + if (actual != expected) + return true; + } + + return false; +} + +/* Map modifiers to mali_texture_layout for packing in a texture descriptor */ + +static enum mali_texture_layout +panfrost_modifier_to_layout(uint64_t modifier) +{ + if (drm_is_afbc(modifier)) + return MALI_TEXTURE_LAYOUT_AFBC; + else if (modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) + return MALI_TEXTURE_LAYOUT_TILED; + else if (modifier == DRM_FORMAT_MOD_LINEAR) + return MALI_TEXTURE_LAYOUT_LINEAR; + else + unreachable("Invalid modifer"); +} + +void +GENX(panfrost_new_texture)(const struct panfrost_device *dev, + const struct pan_image_view *iview, + void *out, const struct panfrost_ptr *payload) +{ + const struct pan_image_layout *layout = &iview->image->layout; + enum pipe_format format = iview->format; + unsigned swizzle; + + if (PAN_ARCH == 7 && util_format_is_depth_or_stencil(format)) { + /* v7 doesn't have an _RRRR component order, combine the + * user swizzle with a .XXXX swizzle to emulate that. + */ + static const unsigned char replicate_x[4] = { + PIPE_SWIZZLE_X, PIPE_SWIZZLE_X, + PIPE_SWIZZLE_X, PIPE_SWIZZLE_X, + }; + unsigned char patched_swizzle[4]; + + util_format_compose_swizzles(replicate_x, + iview->swizzle, + patched_swizzle); + swizzle = panfrost_translate_swizzle_4(patched_swizzle); + } else { + swizzle = panfrost_translate_swizzle_4(iview->swizzle); + } + + bool manual_stride = + panfrost_needs_explicit_stride(iview); + + panfrost_emit_texture_payload(iview, format, + manual_stride, + payload->cpu); + + unsigned array_size = iview->last_layer - iview->first_layer + 1; + + if (iview->dim == MALI_TEXTURE_DIMENSION_CUBE) { + assert(iview->first_layer % 6 == 0); + assert(iview->last_layer % 6 == 5); + array_size /= 6; + } + + unsigned width; + + if (iview->buf.size) { + assert(iview->dim == MALI_TEXTURE_DIMENSION_1D); + assert(!iview->first_level && !iview->last_level); + assert(!iview->first_layer && !iview->last_layer); + assert(layout->nr_samples == 1); + assert(layout->height == 1 && layout->depth == 1); + assert(iview->buf.offset + iview->buf.size <= layout->width); + width = iview->buf.size; + } else { + width = u_minify(layout->width, iview->first_level); + } + + pan_pack(out, TEXTURE, cfg) { + cfg.dimension = iview->dim; + cfg.format = dev->formats[format].hw; + cfg.width = width; + cfg.height = u_minify(layout->height, iview->first_level); + if (iview->dim == MALI_TEXTURE_DIMENSION_3D) + cfg.depth = u_minify(layout->depth, iview->first_level); + else + cfg.sample_count = layout->nr_samples; + cfg.swizzle = swizzle; + cfg.texel_ordering = + panfrost_modifier_to_layout(layout->modifier); + cfg.levels = iview->last_level - iview->first_level + 1; + cfg.array_size = array_size; + +#if PAN_ARCH >= 6 + cfg.surfaces = payload->gpu; + + /* We specify API-level LOD clamps in the sampler descriptor + * and use these clamps simply for bounds checking */ + cfg.minimum_lod = FIXED_16(0, false); + cfg.maximum_lod = FIXED_16(cfg.levels - 1, false); +#else + cfg.manual_stride = manual_stride; +#endif + } +} +#endif /* ifdef PAN_ARCH */ diff --git a/src/panfrost/lib/pan_texture.h b/src/panfrost/lib/pan_texture.h index 0b797b9fc11..992731791ff 100644 --- a/src/panfrost/lib/pan_texture.h +++ b/src/panfrost/lib/pan_texture.h @@ -28,6 +28,8 @@ #ifndef __PAN_TEXTURE_H #define __PAN_TEXTURE_H +#include "gen_macros.h" + #include #include "drm-uapi/drm_fourcc.h" #include "util/format/u_format.h" @@ -157,15 +159,16 @@ panfrost_afbc_can_ytr(enum pipe_format format); unsigned panfrost_block_dim(uint64_t modifier, bool width, unsigned plane); +#ifdef PAN_ARCH unsigned -panfrost_estimate_texture_payload_size(const struct panfrost_device *dev, - const struct pan_image_view *iview); +GENX(panfrost_estimate_texture_payload_size)(const struct pan_image_view *iview); void -panfrost_new_texture(const struct panfrost_device *dev, - const struct pan_image_view *iview, - void *out, - const struct panfrost_ptr *payload); +GENX(panfrost_new_texture)(const struct panfrost_device *dev, + const struct pan_image_view *iview, + void *out, + const struct panfrost_ptr *payload); +#endif unsigned panfrost_get_layer_stride(const struct pan_image_layout *layout, diff --git a/src/panfrost/vulkan/panvk_vX_image.c b/src/panfrost/vulkan/panvk_vX_image.c index f75f7b21305..232fe270336 100644 --- a/src/panfrost/vulkan/panvk_vX_image.c +++ b/src/panfrost/vulkan/panvk_vX_image.c @@ -127,7 +127,7 @@ panvk_per_arch(CreateImageView)(VkDevice _device, struct panfrost_device *pdev = &device->physical_device->pdev; unsigned bo_size = - panfrost_estimate_texture_payload_size(pdev, &view->pview) + + GENX(panfrost_estimate_texture_payload_size)(&view->pview) + pan_size(TEXTURE); unsigned surf_descs_offset = PAN_ARCH <= 5 ? pan_size(TEXTURE) : 0; @@ -142,7 +142,7 @@ panvk_per_arch(CreateImageView)(VkDevice _device, &view->desc : view->bo->ptr.cpu; STATIC_ASSERT(sizeof(view->desc) >= pan_size(TEXTURE)); - panfrost_new_texture(pdev, &view->pview, tex_desc, &surf_descs); + GENX(panfrost_new_texture)(pdev, &view->pview, tex_desc, &surf_descs); *pView = panvk_image_view_to_handle(view); return VK_SUCCESS;