From 80923e8d58cc6bdcceb8e1b2910737fc76fdc0d3 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Mon, 5 Apr 2021 09:33:21 -0700 Subject: [PATCH] util/format: Add some NEON intrinsics-based u_format_unpack. In looking at the profile of dEQP, GLES3 was spending 5-10% of its time in ReadPixels, and almost all of that is b8g8r8a8_unorm8. It's really slow because we're getting about 47MB/s by doing uncached reads 32 bits at a time in the code-generated unpack. If we use NEON to generate larger bus transactions, we can speed things up to 136MB/s. In comparison, raw ldr/str read/writes with no byte swapping can hit a max of 216MB/sec. Reviewed-by: Jesse Natalie Part-of: --- src/util/Android.mk | 2 + src/util/format/meson.build | 1 + src/util/format/u_format.c | 28 +++++++++ src/util/format/u_format.h | 9 +++ src/util/format/u_format_table.py | 6 +- src/util/format/u_format_unpack_neon.c | 79 ++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 src/util/format/u_format_unpack_neon.c diff --git a/src/util/Android.mk b/src/util/Android.mk index 829699db669..3866610cf81 100644 --- a/src/util/Android.mk +++ b/src/util/Android.mk @@ -50,6 +50,8 @@ LOCAL_C_INCLUDES := \ $(intermediates)/util/format \ $(intermediates) +LOCAL_CFLAGS := -DNO_FORMAT_ASM + # If Android version >=8 MESA should static link libexpat else should dynamic link ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 27; echo $$?), 0) LOCAL_STATIC_LIBRARIES := \ diff --git a/src/util/format/meson.build b/src/util/format/meson.build index e6438629562..b51608f5275 100644 --- a/src/util/format/meson.build +++ b/src/util/format/meson.build @@ -28,6 +28,7 @@ files_mesa_format = [ 'u_format_rgtc.c', 'u_format_s3tc.c', 'u_format_tests.c', + 'u_format_unpack_neon.c', 'u_format_yuv.c', 'u_format_zs.c', ] diff --git a/src/util/format/u_format.c b/src/util/format/u_format.c index 1c3ca57fc76..59bb7d287da 100644 --- a/src/util/format/u_format.c +++ b/src/util/format/u_format.c @@ -34,6 +34,7 @@ #include "util/format/u_format.h" #include "util/format/u_format_s3tc.h" +#include "util/u_cpu_detect.h" #include "util/u_math.h" #include "pipe/p_defines.h" @@ -1130,3 +1131,30 @@ util_format_rgb_to_bgr(enum pipe_format format) return PIPE_FORMAT_NONE; } } + +static const struct util_format_unpack_description *util_format_unpack_table[PIPE_FORMAT_COUNT]; + +static void +util_format_unpack_table_init(void) +{ + for (enum pipe_format format = PIPE_FORMAT_NONE; format < PIPE_FORMAT_COUNT; format++) { +#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM + const struct util_format_unpack_description *unpack = util_format_unpack_description_neon(format); + if (unpack) { + util_format_unpack_table[format] = unpack; + continue; + } +#endif + + util_format_unpack_table[format] = util_format_unpack_description_generic(format); + } +} + +const struct util_format_unpack_description * +util_format_unpack_description(enum pipe_format format) +{ + static once_flag flag = ONCE_FLAG_INIT; + call_once(&flag, util_format_unpack_table_init); + + return util_format_unpack_table[format]; +} diff --git a/src/util/format/u_format.h b/src/util/format/u_format.h index a1b5ec1ecbc..f7b29b407a4 100644 --- a/src/util/format/u_format.h +++ b/src/util/format/u_format.h @@ -415,8 +415,17 @@ util_format_description(enum pipe_format format) ATTRIBUTE_CONST; const struct util_format_pack_description * util_format_pack_description(enum pipe_format format) ATTRIBUTE_CONST; +/* Lookup with CPU detection for choosing optimized paths. */ const struct util_format_unpack_description * util_format_unpack_description(enum pipe_format format) ATTRIBUTE_CONST; + +/* Codegenned table of CPU-agnostic unpack code. */ +const struct util_format_unpack_description * +util_format_unpack_description_generic(enum pipe_format format) ATTRIBUTE_CONST; + +const struct util_format_unpack_description * +util_format_unpack_description_neon(enum pipe_format format) ATTRIBUTE_CONST; + #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/src/util/format/u_format_table.py b/src/util/format/u_format_table.py index b9176956cc3..c76ca583a5c 100644 --- a/src/util/format/u_format_table.py +++ b/src/util/format/u_format_table.py @@ -166,8 +166,11 @@ def write_format_table(formats): print(" },") def generate_table_getter(type): + suffix = "" + if type == "unpack_": + suffix = "_generic" print("const struct util_format_%sdescription *" % type) - print("util_format_%sdescription(enum pipe_format format)" % type) + print("util_format_%sdescription%s(enum pipe_format format)" % (type, suffix)) print("{") print(" if (format >= ARRAY_SIZE(util_format_%sdescriptions))" % (type)) print(" return NULL;") @@ -242,7 +245,6 @@ def write_format_table(formats): print("};") print() generate_table_getter("pack_") - print('static const struct util_format_unpack_description') print('util_format_unpack_descriptions[] = {') for format in formats: diff --git a/src/util/format/u_format_unpack_neon.c b/src/util/format/u_format_unpack_neon.c new file mode 100644 index 00000000000..7456d7aaa88 --- /dev/null +++ b/src/util/format/u_format_unpack_neon.c @@ -0,0 +1,79 @@ +/* + * Copyright © 2021 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include + +#if (defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_ARM)) && !defined NO_FORMAT_ASM + +/* armhf builds default to vfp, not neon, and refuses to compile neon intrinsics + * unless you tell it "no really". + */ +#ifdef PIPE_ARCH_ARM +#pragma GCC target ("fpu=neon") +#endif + +#include +#include "u_format_pack.h" +#include "util/u_cpu_detect.h" + +static void +util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon(uint8_t *restrict dst, const uint8_t *restrict src, unsigned width) +{ + while (width >= 16) { + uint8x16x4_t load = vld4q_u8(src); + uint8x16x4_t swap = { .val = { load.val[2], load.val[1], load.val[0], load.val[3] } }; + vst4q_u8(dst, swap); + width -= 16; + dst += 16 * 4; + src += 16 * 4; + } + if (width) + util_format_b8g8r8a8_unorm_unpack_rgba_8unorm(dst, src, width); +} + +static const struct util_format_unpack_description util_format_unpack_descriptions_neon[] = { + [PIPE_FORMAT_B8G8R8A8_UNORM] = { + .unpack_rgba_8unorm = &util_format_b8g8r8a8_unorm_unpack_rgba_8unorm_neon, + .unpack_rgba = &util_format_b8g8r8a8_unorm_unpack_rgba_float, + }, +}; + +const struct util_format_unpack_description * +util_format_unpack_description_neon(enum pipe_format format) +{ + /* CPU detect for NEON support. On arm64, it's implied. */ +#ifdef PIPE_ARCH_ARM + if (!util_get_cpu_caps()->has_neon) + return NULL; +#endif + + if (format >= ARRAY_SIZE(util_format_unpack_descriptions_neon)) + return NULL; + + if (!util_format_unpack_descriptions_neon[format].unpack_rgba) + return NULL; + + return &util_format_unpack_descriptions_neon[format]; +} + +#endif /* PIPE_ARCH_AARCH64 | PIPE_ARCH_ARM */