From bac77bb30dd178c1c3cb4f6524ba67bf9170d970 Mon Sep 17 00:00:00 2001 From: Antonio Ospite Date: Fri, 7 Feb 2025 15:29:00 +0100 Subject: [PATCH] ci/android: add a job using android-cts instead of deqp-runner We also want to run Android CTS in the Android jobs. Since the Android CTS is quite large, download it and strip it down to only contain the interesting tests, so to reduce the space taken in the container image. Eventually we might want to have android-cts be run via deqp-runner itself, but for now add a proof-of-concept mechanism which calls the android-cts directly and uses an ad-hoc handling of expectations. Part-of: --- .gitlab-ci/android-cts-runner.sh | 44 +++++++++++++++++++ .../container/debian/x86_64_test-android.sh | 32 ++++++++++++++ .gitlab-ci/container/gitlab-ci.yml | 4 ++ .gitlab-ci/cuttlefish-runner.sh | 11 +++-- .gitlab-ci/image-tags.yml | 2 +- .../frontends/lavapipe/ci/gitlab-ci.yml | 12 +++++ .../lvp-android-angle-android-cts-skips.txt | 40 +++++++++++++++++ 7 files changed, 141 insertions(+), 4 deletions(-) create mode 100755 .gitlab-ci/android-cts-runner.sh create mode 100644 src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt diff --git a/.gitlab-ci/android-cts-runner.sh b/.gitlab-ci/android-cts-runner.sh new file mode 100755 index 00000000000..b4fb77fe2f9 --- /dev/null +++ b/.gitlab-ci/android-cts-runner.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2086 # we want word splitting +# shellcheck disable=SC1091 # paths only become valid at runtime + +. "${SCRIPTS_DIR}/setup-test-env.sh" + +export PATH=/android-tools/android-cts/jdk/bin/:/android-tools/build-tools:$PATH +export JAVA_HOME=/android-tools/android-cts/jdk + +# Wait for the appops service to show up +while [ "$($ADB shell dumpsys -l | grep appops)" = "" ] ; do sleep 1; done + +SKIP_FILE="$INSTALL/${GPU_VERSION}-android-cts-skips.txt" + +EXCLUDE_FILTERS="" +if [ -e "$SKIP_FILE" ]; then + EXCLUDE_FILTERS="$(grep -v -E "(^#|^[[:space:]]*$)" "$SKIP_FILE" | sed -s 's/.*/--exclude-filter "\0" /g')" +fi + +INCLUDE_FILE="$INSTALL/${GPU_VERSION}-android-cts-include.txt" + +if [ -e "$INCLUDE_FILE" ]; then + INCLUDE_FILTERS="$(grep -v -E "(^#|^[[:space:]]*$)" "$INCLUDE_FILE" | sed -s 's/.*/--include-filter "\0" /g')" +else + INCLUDE_FILTERS=$(printf -- "--include-filter %s " $ANDROID_CTS_MODULES | sed -e 's/ $//g') +fi + +set +e +eval "/android-tools/android-cts/tools/cts-tradefed" run commandAndExit cts-dev \ + $EXCLUDE_FILTERS \ + $INCLUDE_FILTERS + +[ "$(grep "^FAILED" /android-tools/android-cts/results/latest/invocation_summary.txt | tr -d ' ' | cut -d ':' -f 2)" = "0" ] + +# shellcheck disable=SC2034 # EXIT_CODE is used by the script that sources this one +EXIT_CODE=$? +set -e + +section_switch cuttlefish_results "cuttlefish: gathering the results" + +cp -r "/android-tools/android-cts/results/latest"/* $RESULTS_DIR +cp -r "/android-tools/android-cts/logs/latest"/* $RESULTS_DIR + +section_end cuttlefish_results diff --git a/.gitlab-ci/container/debian/x86_64_test-android.sh b/.gitlab-ci/container/debian/x86_64_test-android.sh index a93085342f9..9dc3c15e196 100755 --- a/.gitlab-ci/container/debian/x86_64_test-android.sh +++ b/.gitlab-ci/container/debian/x86_64_test-android.sh @@ -138,6 +138,38 @@ rm -rf android-cuttlefish addgroup --system kvm usermod -a -G kvm,cvdnetwork root +############### Downloading Android CTS - Build tools - Platform tools ... + +ANDROID_CTS_VERSION="${ANDROID_VERSION}_r1" +ANDROID_CTS_DEVICE_ARCH="x86" + +mkdir /android-tools +pushd /android-tools + +curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" \ + "https://dl.google.com/dl/android/cts/android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" +unzip "android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" +rm "android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" + +# Keep only the interesting tests to save space +# shellcheck disable=SC2086 # we want word splitting +ANDROID_CTS_MODULES_KEEP_EXPRESSION=$(printf "%s|" $ANDROID_CTS_MODULES | sed -e 's/|$//g') +find android-cts/testcases/ -mindepth 1 -type d | grep -v -E "$ANDROID_CTS_MODULES_KEEP_EXPRESSION" | xargs rm -rf + +curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "build-tools_r${ANDROID_SDK_VERSION}-linux.zip" "https://dl.google.com/android/repository/build-tools_r${ANDROID_SDK_VERSION}-linux.zip" +unzip "build-tools_r${ANDROID_SDK_VERSION}-linux.zip" +rm "build-tools_r${ANDROID_SDK_VERSION}-linux.zip" +mv "android-$ANDROID_VERSION" build-tools + +curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" "https://dl.google.com/android/repository/platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" +unzip "platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" +rm "platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" + +popd + ############### Uninstall the build software rm -rf "/${ndk:?}" diff --git a/.gitlab-ci/container/gitlab-ci.yml b/.gitlab-ci/container/gitlab-ci.yml index 97a981f5f60..d85656c74e7 100644 --- a/.gitlab-ci/container/gitlab-ci.yml +++ b/.gitlab-ci/container/gitlab-ci.yml @@ -193,8 +193,11 @@ debian/s390x_build: extends: - .container-builds-android variables: + ANDROID_VERSION: 14 ANDROID_NDK_VERSION: "r27c" ANDROID_SDK_VERSION: 34 + # Space-separated list of interesting CTS modules + ANDROID_CTS_MODULES: CtsGraphicsTestCases ANDROID_LLVM_VERSION: llvmorg-18.1.8 ANDROID_LLVM_ARTIFACT_NAME: android-x86_64-llvm-20250103 # This can be confusing: LLVM_VERSION refers to the host LLVM toolchain @@ -419,6 +422,7 @@ debian/x86_64_test-android: .use-debian/x86_64_test-android: extends: + - .android-variables - .set-image-base-tag variables: MESA_BASE_TAG: *debian-x86_64_test-base diff --git a/.gitlab-ci/cuttlefish-runner.sh b/.gitlab-ci/cuttlefish-runner.sh index 76ea6e13674..5658b8b09fd 100755 --- a/.gitlab-ci/cuttlefish-runner.sh +++ b/.gitlab-ci/cuttlefish-runner.sh @@ -13,7 +13,7 @@ if [ -n "$ANGLE_TAG" ]; then ci_tag_test_time_check "ANGLE_TAG" fi -export PATH=/cuttlefish/bin:$PATH +export PATH=/cuttlefish/bin:/android-tools/platform-tools:$PATH export LD_LIBRARY_PATH=/cuttlefish/lib64:${CI_PROJECT_DIR}/install/lib:$LD_LIBRARY_PATH export EGL_PLATFORM=surfaceless @@ -156,7 +156,12 @@ else fi fi -# The script sets EXIT_CODE -. "$(dirname "$0")/android-deqp-runner.sh" +if [ -n "$USE_ANDROID_CTS" ]; then + # The script sets EXIT_CODE + . "$(dirname "$0")/android-cts-runner.sh" +else + # The script sets EXIT_CODE + . "$(dirname "$0")/android-deqp-runner.sh" +fi exit $EXIT_CODE diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 1209c83d04a..e77ee3f3aee 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -37,7 +37,7 @@ variables: DEBIAN_ARM64_TEST_IMAGE_VK_PATH: "debian/arm64_test-vk" DEBIAN_X86_64_TEST_ANDROID_IMAGE_PATH: "debian/x86_64_test-android" - DEBIAN_TEST_ANDROID_TAG: "20250218-cuttlefish" + DEBIAN_TEST_ANDROID_TAG: "20250310-android-cts" DEBIAN_TEST_GL_TAG: "20250307-piglit-70" DEBIAN_TEST_VK_TAG: "20250307-piglit-70" KERNEL_ROOTFS_TAG: "20250307-piglit-70" diff --git a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml index ff65ff0293a..73ff5dd5f31 100644 --- a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml +++ b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml @@ -68,3 +68,15 @@ android-angle-lavapipe: - .test-android - .lavapipe-rules - .test-angle + +android-angle-lavapipe-cts: + variables: + USE_ANDROID_CTS: 1 + ANDROID_GPU_MODE: mesa_swrast_guest_angle + GPU_VERSION: lvp-android-angle + MESA_VK_IGNORE_CONFORMANCE_WARNING: 1 + timeout: 15m + extends: + - .test-android + - .lavapipe-rules + - .test-angle diff --git a/src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt b/src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt new file mode 100644 index 00000000000..9202b8e987d --- /dev/null +++ b/src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt @@ -0,0 +1,40 @@ +# Crashes +x86_64 CtsGraphicsTestCases android.graphics.cts.BitmapTest#testCreateBitmap_Picture_immutable +x86_64 CtsGraphicsTestCases android.graphics.cts.BitmapTest#testDrawingHardwareBitmapNotLeaking +x86_64 CtsGraphicsTestCases android.graphics.cts.CameraGpuTest#testCameraImageCaptureAndRendering +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppDisplayModeGetRefreshRateDisplayModeReturnsPhysicalRefreshRateEnabled +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalBackpressure +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppBackpressure +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalDisplayGetRefreshRate +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppDisplayGetRefreshRate +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppChoreographer +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalDisplayModeGetRefreshRateDisplayModeReturnsPhysicalRefreshRateEnabled +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalChoreographer +x86_64 CtsGraphicsTestCases android.graphics.cts.MatchContentFrameRateTest#testMatchContentFramerate_Always +x86_64 CtsGraphicsTestCases android.graphics.cts.MatchContentFrameRateTest#testMatchContentFramerate_Auto +x86_64 CtsGraphicsTestCases android.graphics.cts.MatchContentFrameRateTest#testMatchContentFramerate_None +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testClearFrameRate +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testExactFrameRateMatch_Seamless +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testFixedSource_NonSeamless +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testFixedSource_Seamless +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testInvalidParams +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testExactFrameRateMatch_NonSeamless +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanPreTransformTest#testVulkanPreTransformNotSetToMatchCurrentTransform +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanPreTransformTest#testVulkanPreTransformSetToMatchCurrentTransform +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimatedImageDrawableTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimatedVectorDrawableParameterizedTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimatedVectorDrawableTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimationDrawableTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.IconTest + +# Failures +x86_64 CtsGraphicsTestCases android.graphics.cts.BitmapTest#testHardwareBitmapNotLeaking +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanRequiredExtensions +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanApplicationBinaryInterfaceRequirements +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanApiForEachDevice +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanHardwareFeatures +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanVariantSupport +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testAndroidBaselineProfile2021Support +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanVersionForVrHighPerformance +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanBlockedExtensions +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkan1_1Requirements