diff --git a/.gitlab-ci/android-cts-runner.sh b/.gitlab-ci/android-cts-runner.sh new file mode 100755 index 00000000000..b4fb77fe2f9 --- /dev/null +++ b/.gitlab-ci/android-cts-runner.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2086 # we want word splitting +# shellcheck disable=SC1091 # paths only become valid at runtime + +. "${SCRIPTS_DIR}/setup-test-env.sh" + +export PATH=/android-tools/android-cts/jdk/bin/:/android-tools/build-tools:$PATH +export JAVA_HOME=/android-tools/android-cts/jdk + +# Wait for the appops service to show up +while [ "$($ADB shell dumpsys -l | grep appops)" = "" ] ; do sleep 1; done + +SKIP_FILE="$INSTALL/${GPU_VERSION}-android-cts-skips.txt" + +EXCLUDE_FILTERS="" +if [ -e "$SKIP_FILE" ]; then + EXCLUDE_FILTERS="$(grep -v -E "(^#|^[[:space:]]*$)" "$SKIP_FILE" | sed -s 's/.*/--exclude-filter "\0" /g')" +fi + +INCLUDE_FILE="$INSTALL/${GPU_VERSION}-android-cts-include.txt" + +if [ -e "$INCLUDE_FILE" ]; then + INCLUDE_FILTERS="$(grep -v -E "(^#|^[[:space:]]*$)" "$INCLUDE_FILE" | sed -s 's/.*/--include-filter "\0" /g')" +else + INCLUDE_FILTERS=$(printf -- "--include-filter %s " $ANDROID_CTS_MODULES | sed -e 's/ $//g') +fi + +set +e +eval "/android-tools/android-cts/tools/cts-tradefed" run commandAndExit cts-dev \ + $EXCLUDE_FILTERS \ + $INCLUDE_FILTERS + +[ "$(grep "^FAILED" /android-tools/android-cts/results/latest/invocation_summary.txt | tr -d ' ' | cut -d ':' -f 2)" = "0" ] + +# shellcheck disable=SC2034 # EXIT_CODE is used by the script that sources this one +EXIT_CODE=$? +set -e + +section_switch cuttlefish_results "cuttlefish: gathering the results" + +cp -r "/android-tools/android-cts/results/latest"/* $RESULTS_DIR +cp -r "/android-tools/android-cts/logs/latest"/* $RESULTS_DIR + +section_end cuttlefish_results diff --git a/.gitlab-ci/container/debian/x86_64_test-android.sh b/.gitlab-ci/container/debian/x86_64_test-android.sh index a93085342f9..9dc3c15e196 100755 --- a/.gitlab-ci/container/debian/x86_64_test-android.sh +++ b/.gitlab-ci/container/debian/x86_64_test-android.sh @@ -138,6 +138,38 @@ rm -rf android-cuttlefish addgroup --system kvm usermod -a -G kvm,cvdnetwork root +############### Downloading Android CTS - Build tools - Platform tools ... + +ANDROID_CTS_VERSION="${ANDROID_VERSION}_r1" +ANDROID_CTS_DEVICE_ARCH="x86" + +mkdir /android-tools +pushd /android-tools + +curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" \ + "https://dl.google.com/dl/android/cts/android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" +unzip "android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" +rm "android-cts-${ANDROID_CTS_VERSION}-linux_x86-${ANDROID_CTS_DEVICE_ARCH}.zip" + +# Keep only the interesting tests to save space +# shellcheck disable=SC2086 # we want word splitting +ANDROID_CTS_MODULES_KEEP_EXPRESSION=$(printf "%s|" $ANDROID_CTS_MODULES | sed -e 's/|$//g') +find android-cts/testcases/ -mindepth 1 -type d | grep -v -E "$ANDROID_CTS_MODULES_KEEP_EXPRESSION" | xargs rm -rf + +curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "build-tools_r${ANDROID_SDK_VERSION}-linux.zip" "https://dl.google.com/android/repository/build-tools_r${ANDROID_SDK_VERSION}-linux.zip" +unzip "build-tools_r${ANDROID_SDK_VERSION}-linux.zip" +rm "build-tools_r${ANDROID_SDK_VERSION}-linux.zip" +mv "android-$ANDROID_VERSION" build-tools + +curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" "https://dl.google.com/android/repository/platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" +unzip "platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" +rm "platform-tools_r${ANDROID_SDK_VERSION}.0.0-linux.zip" + +popd + ############### Uninstall the build software rm -rf "/${ndk:?}" diff --git a/.gitlab-ci/container/gitlab-ci.yml b/.gitlab-ci/container/gitlab-ci.yml index 97a981f5f60..d85656c74e7 100644 --- a/.gitlab-ci/container/gitlab-ci.yml +++ b/.gitlab-ci/container/gitlab-ci.yml @@ -193,8 +193,11 @@ debian/s390x_build: extends: - .container-builds-android variables: + ANDROID_VERSION: 14 ANDROID_NDK_VERSION: "r27c" ANDROID_SDK_VERSION: 34 + # Space-separated list of interesting CTS modules + ANDROID_CTS_MODULES: CtsGraphicsTestCases ANDROID_LLVM_VERSION: llvmorg-18.1.8 ANDROID_LLVM_ARTIFACT_NAME: android-x86_64-llvm-20250103 # This can be confusing: LLVM_VERSION refers to the host LLVM toolchain @@ -419,6 +422,7 @@ debian/x86_64_test-android: .use-debian/x86_64_test-android: extends: + - .android-variables - .set-image-base-tag variables: MESA_BASE_TAG: *debian-x86_64_test-base diff --git a/.gitlab-ci/cuttlefish-runner.sh b/.gitlab-ci/cuttlefish-runner.sh index 76ea6e13674..5658b8b09fd 100755 --- a/.gitlab-ci/cuttlefish-runner.sh +++ b/.gitlab-ci/cuttlefish-runner.sh @@ -13,7 +13,7 @@ if [ -n "$ANGLE_TAG" ]; then ci_tag_test_time_check "ANGLE_TAG" fi -export PATH=/cuttlefish/bin:$PATH +export PATH=/cuttlefish/bin:/android-tools/platform-tools:$PATH export LD_LIBRARY_PATH=/cuttlefish/lib64:${CI_PROJECT_DIR}/install/lib:$LD_LIBRARY_PATH export EGL_PLATFORM=surfaceless @@ -156,7 +156,12 @@ else fi fi -# The script sets EXIT_CODE -. "$(dirname "$0")/android-deqp-runner.sh" +if [ -n "$USE_ANDROID_CTS" ]; then + # The script sets EXIT_CODE + . "$(dirname "$0")/android-cts-runner.sh" +else + # The script sets EXIT_CODE + . "$(dirname "$0")/android-deqp-runner.sh" +fi exit $EXIT_CODE diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 1209c83d04a..e77ee3f3aee 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -37,7 +37,7 @@ variables: DEBIAN_ARM64_TEST_IMAGE_VK_PATH: "debian/arm64_test-vk" DEBIAN_X86_64_TEST_ANDROID_IMAGE_PATH: "debian/x86_64_test-android" - DEBIAN_TEST_ANDROID_TAG: "20250218-cuttlefish" + DEBIAN_TEST_ANDROID_TAG: "20250310-android-cts" DEBIAN_TEST_GL_TAG: "20250307-piglit-70" DEBIAN_TEST_VK_TAG: "20250307-piglit-70" KERNEL_ROOTFS_TAG: "20250307-piglit-70" diff --git a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml index ff65ff0293a..73ff5dd5f31 100644 --- a/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml +++ b/src/gallium/frontends/lavapipe/ci/gitlab-ci.yml @@ -68,3 +68,15 @@ android-angle-lavapipe: - .test-android - .lavapipe-rules - .test-angle + +android-angle-lavapipe-cts: + variables: + USE_ANDROID_CTS: 1 + ANDROID_GPU_MODE: mesa_swrast_guest_angle + GPU_VERSION: lvp-android-angle + MESA_VK_IGNORE_CONFORMANCE_WARNING: 1 + timeout: 15m + extends: + - .test-android + - .lavapipe-rules + - .test-angle diff --git a/src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt b/src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt new file mode 100644 index 00000000000..9202b8e987d --- /dev/null +++ b/src/gallium/frontends/lavapipe/ci/lvp-android-angle-android-cts-skips.txt @@ -0,0 +1,40 @@ +# Crashes +x86_64 CtsGraphicsTestCases android.graphics.cts.BitmapTest#testCreateBitmap_Picture_immutable +x86_64 CtsGraphicsTestCases android.graphics.cts.BitmapTest#testDrawingHardwareBitmapNotLeaking +x86_64 CtsGraphicsTestCases android.graphics.cts.CameraGpuTest#testCameraImageCaptureAndRendering +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppDisplayModeGetRefreshRateDisplayModeReturnsPhysicalRefreshRateEnabled +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalBackpressure +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppBackpressure +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalDisplayGetRefreshRate +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppDisplayGetRefreshRate +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testAppChoreographer +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalDisplayModeGetRefreshRateDisplayModeReturnsPhysicalRefreshRateEnabled +x86_64 CtsGraphicsTestCases android.graphics.cts.FrameRateOverrideTest#testGlobalChoreographer +x86_64 CtsGraphicsTestCases android.graphics.cts.MatchContentFrameRateTest#testMatchContentFramerate_Always +x86_64 CtsGraphicsTestCases android.graphics.cts.MatchContentFrameRateTest#testMatchContentFramerate_Auto +x86_64 CtsGraphicsTestCases android.graphics.cts.MatchContentFrameRateTest#testMatchContentFramerate_None +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testClearFrameRate +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testExactFrameRateMatch_Seamless +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testFixedSource_NonSeamless +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testFixedSource_Seamless +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testInvalidParams +x86_64 CtsGraphicsTestCases android.graphics.cts.SetFrameRateTest#testExactFrameRateMatch_NonSeamless +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanPreTransformTest#testVulkanPreTransformNotSetToMatchCurrentTransform +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanPreTransformTest#testVulkanPreTransformSetToMatchCurrentTransform +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimatedImageDrawableTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimatedVectorDrawableParameterizedTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimatedVectorDrawableTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.AnimationDrawableTest +x86_64 CtsGraphicsTestCases android.graphics.drawable.cts.IconTest + +# Failures +x86_64 CtsGraphicsTestCases android.graphics.cts.BitmapTest#testHardwareBitmapNotLeaking +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanRequiredExtensions +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanApplicationBinaryInterfaceRequirements +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanApiForEachDevice +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanHardwareFeatures +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanVariantSupport +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testAndroidBaselineProfile2021Support +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanVersionForVrHighPerformance +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkanBlockedExtensions +x86_64 CtsGraphicsTestCases android.graphics.cts.VulkanFeaturesTest#testVulkan1_1Requirements