From 05021a178662c11075a31b76bde9d487d86a55c1 Mon Sep 17 00:00:00 2001 From: Valentine Burley Date: Mon, 10 Feb 2025 08:06:29 +0100 Subject: [PATCH] ci/angle: Rework building ANGLE (again) Make setting the ANGLE_ARCH variable optional by providing a default based on DEBIAN_ARCH, while keeping it possible to override it, which is expected to be necessary for the Android-arm64 build. Exclude unnecessary third party dependencies in the .gclient file, which allows us to delete our first local patch. Thanks to Yuly Novikov for the suggestion. Use -no-history for gclient sync, which is equivalent to git's --depth=1 argument. This greatly speeds up the process of fetching sources. Thanks to this speedup fetching third_party/catapult is no longer an issue, allowing us to remove our second local patch. Since we're no longer applying local patches, use ANGLE_REV and /angle/version as the base for our version check on Android. Signed-off-by: Valentine Burley Reviewed-by: Antonio Ospite Part-of: --- .gitlab-ci/container/build-angle.sh | 57 +++---- .../container/debian/x86_64_test-android.sh | 1 - .gitlab-ci/container/lava_build.sh | 1 - ...ild-angle-build_deps_fix-build-error.patch | 30 ---- ...e_deps_Make-more-sources-conditional.patch | 144 ------------------ .gitlab-ci/cuttlefish-runner.sh | 2 +- .gitlab-ci/image-tags.yml | 4 +- 7 files changed, 24 insertions(+), 215 deletions(-) delete mode 100644 .gitlab-ci/container/patches/build-angle-build_deps_fix-build-error.patch delete mode 100644 .gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch diff --git a/.gitlab-ci/container/build-angle.sh b/.gitlab-ci/container/build-angle.sh index 82443576cc9..427e44470b5 100755 --- a/.gitlab-ci/container/build-angle.sh +++ b/.gitlab-ci/container/build-angle.sh @@ -7,11 +7,17 @@ set -uex -uncollapsed_section_start angle "Building angle" +uncollapsed_section_start angle "Building ANGLE" ANGLE_REV="76025caa1a059f464a2b0e8f879dbd4746f092b9" -SCRIPTS_DIR="$(pwd)/.gitlab-ci" -ANGLE_PATCH_DIR="${SCRIPTS_DIR}/container/patches" + +# Set ANGLE_ARCH based on DEBIAN_ARCH if it hasn't been explicitly defined +if [[ -z "${ANGLE_ARCH:-}" ]]; then + case "$DEBIAN_ARCH" in + amd64) ANGLE_ARCH=x64;; + arm64) ANGLE_ARCH=arm64;; + esac +fi # DEPOT tools git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git /depot-tools @@ -26,33 +32,15 @@ git remote add origin https://chromium.googlesource.com/angle/angle.git git fetch --depth 1 origin "$ANGLE_REV" git checkout FETCH_HEAD -angle_patch_files=( - build-angle_deps_Make-more-sources-conditional.patch -) -for patch in "${angle_patch_files[@]}"; do - echo "Apply patch to ANGLE from ${patch}" - GIT_COMMITTER_DATE="$(LC_TIME=C date -d@0)" git am < "${ANGLE_PATCH_DIR}/${patch}" -done - -{ - echo "ANGLE base version $ANGLE_REV" - echo "The following local patches are applied on top:" - git log --reverse --oneline $ANGLE_REV.. --format='- %s' -} > /angle/version - -# Save the current git hash to /angle/hash after applying local patches -git rev-parse HEAD > /angle/hash +echo "$ANGLE_REV" > /angle/version GCLIENT_CUSTOM_VARS=() GCLIENT_CUSTOM_VARS+=('--custom-var=angle_enable_cl=False') GCLIENT_CUSTOM_VARS+=('--custom-var=angle_enable_cl_testing=False') GCLIENT_CUSTOM_VARS+=('--custom-var=angle_enable_vulkan_validation_layers=False') GCLIENT_CUSTOM_VARS+=('--custom-var=angle_enable_wgpu=False') -GCLIENT_CUSTOM_VARS+=('--custom-var=build_allow_regenerate=False') GCLIENT_CUSTOM_VARS+=('--custom-var=build_angle_deqp_tests=False') GCLIENT_CUSTOM_VARS+=('--custom-var=build_angle_perftests=False') -GCLIENT_CUSTOM_VARS+=('--custom-var=build_with_catapult=False') -GCLIENT_CUSTOM_VARS+=('--custom-var=build_with_swiftshader=False') if [[ "$ANGLE_TARGET" == "android" ]]; then GCLIENT_CUSTOM_VARS+=('--custom-var=checkout_android=True') fi @@ -62,20 +50,17 @@ gclient config --name REPLACE-WITH-A-DOT --unmanaged \ "${GCLIENT_CUSTOM_VARS[@]}" \ https://chromium.googlesource.com/angle/angle.git sed -e 's/REPLACE-WITH-A-DOT/./;' -i .gclient -gclient sync -j"${FDO_CI_CONCURRENT:-4}" - -# TODO: The following patch seems necessary to avoid some errors when running -# `gn gen out/Release` for Android, investigate and see if there is a better -# way to handle this. -angle_build_patch_files=( - build-angle-build_deps_fix-build-error.patch -) -pushd build/ -for patch in "${angle_build_patch_files[@]}"; do - echo "Apply patch to ANGLE build from ${patch}" - GIT_COMMITTER_DATE="$(LC_TIME=C date -d@0)" git am < "${ANGLE_PATCH_DIR}/${patch}" -done -popd +sed -e 's|"custom_deps" : {|"custom_deps" : {\ + "third_party/clspv/src": None,\ + "third_party/dawn": None,\ + "third_party/glmark2/src": None,\ + "third_party/libjpeg_turbo": None,\ + "third_party/llvm/src": None,\ + "third_party/OpenCL-CTS/src": None,\ + "third_party/SwiftShader": None,\ + "third_party/VK-GL-CTS/src": None,\ + "third_party/vulkan-validation-layers/src": None,|' -i .gclient +gclient sync --no-history -j"${FDO_CI_CONCURRENT:-4}" mkdir -p out/Release cat > out/Release/args.gn < -Date: Mon, 20 Jan 2025 09:38:46 +0100 -Subject: [PATCH] Fix building on Android with minimal dependencies - -The fixes the following error: - -+ gn gen out/Android -ERROR at //build/android/BUILD.gn:233:5: Unable to load "/angle-android-build/third_party/catapult/tracing/BUILD.gn". - "//third_party/catapult/tracing:convert_chart_json", - ^-------------------------------------------------- ---- - android/BUILD.gn | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/android/BUILD.gn b/android/BUILD.gn -index 3c6fc6ab5..db25974a6 100644 ---- a/android/BUILD.gn -+++ b/android/BUILD.gn -@@ -230,7 +230,6 @@ python_library("resource_sizes_py") { - pydeps_file = "resource_sizes.pydeps" - data_deps = [ - ":devil_chromium_py", -- "//third_party/catapult/tracing:convert_chart_json", - ] - - data = [ --- -2.47.1 - diff --git a/.gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch b/.gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch deleted file mode 100644 index 50a5532c4f7..00000000000 --- a/.gitlab-ci/container/patches/build-angle_deps_Make-more-sources-conditional.patch +++ /dev/null @@ -1,144 +0,0 @@ -From 2f4a38ecfde470abcd5d3c0ae7337bf780343469 Mon Sep 17 00:00:00 2001 -From: Daniel Stone -Date: Tue, 15 Oct 2024 16:02:26 +0100 -Subject: [PATCH] deps: Make more sources conditional - -Fetching all the dependent sources - including at least one copy of LLVM -- can take a surprising amount of time. Mesa needs to build ANGLE as -part of CI, and the cost of downloading all the sources all of the time -is not OK for the number of dependencies we don't need during the build. ---- - DEPS | 33 +++++++++++++++++++++++---------- - 1 file changed, 23 insertions(+), 10 deletions(-) - -Submitted upstream at: -https://chromium-review.googlesource.com/c/angle/angle/+/5937820 - -diff --git a/DEPS b/DEPS -index 61263fb7af..0cff8c3126 100644 ---- a/DEPS -+++ b/DEPS -@@ -17,6 +17,17 @@ gclient_gn_args = [ - ] - - vars = { -+ 'angle_enable_cl': True, -+ 'angle_enable_cl_testing': False, -+ 'angle_enable_vulkan': True, -+ 'angle_enable_vulkan_validation_layers': True, -+ 'angle_enable_wgpu': True, -+ 'build_angle_deqp_tests': True, -+ 'build_angle_perftests': True, -+ 'build_with_swiftshader': True, -+ 'use_custom_libcxx': True, -+ 'export_libcxxapi_from_executables': True, -+ - 'android_git': 'https://android.googlesource.com', - 'chromium_git': 'https://chromium.googlesource.com', - 'chrome_internal_git': 'https://chrome-internal.googlesource.com', -@@ -673,7 +684,7 @@ deps = { - - 'third_party/catapult': { - 'url': Var('chromium_git') + '/catapult.git' + '@' + Var('catapult_revision'), -- 'condition': 'not build_with_chromium', -+ 'condition': 'build_with_catapult and not build_with_chromium', - }, - - # Cherry is a dEQP/VK-GL-CTS management GUI written in Go. We use it for viewing test results. -@@ -689,7 +700,7 @@ deps = { - - 'third_party/clspv/src': { - 'url': Var('chromium_git') + '/external/github.com/google/clspv@a173c052455434a422bcfe5c12ffe44d574fd6e1', -- 'condition': 'not build_with_chromium', -+ 'condition': 'angle_enable_cl and angle_enable_vulkan and not build_with_chromium', - }, - - 'third_party/cpu_features/src': { -@@ -700,7 +711,7 @@ deps = { - - 'third_party/dawn': { - 'url': Var('dawn_git') + '/dawn.git' + '@' + Var('dawn_revision'), -- 'condition': 'not build_with_chromium' -+ 'condition': 'angle_enable_wgpu and not build_with_chromium' - }, - - 'third_party/depot_tools': { -@@ -745,6 +756,7 @@ deps = { - # glmark2 is a GPL3-licensed OpenGL ES 2.0 benchmark. We use it for testing. - 'third_party/glmark2/src': { - 'url': Var('chromium_git') + '/external/github.com/glmark2/glmark2@ca8de51fedb70bace5351c6b002eb952c747e889', -+ 'condition': 'build_angle_perftests', - }, - - 'third_party/googletest': { -@@ -777,7 +789,7 @@ deps = { - # libjpeg_turbo is used by glmark2. - 'third_party/libjpeg_turbo': { - 'url': Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git@927aabfcd26897abb9776ecf2a6c38ea5bb52ab6', -- 'condition': 'not build_with_chromium', -+ 'condition': 'build_angle_perftests and not build_with_chromium', - }, - - 'third_party/libpng/src': { -@@ -787,7 +799,7 @@ deps = { - - 'third_party/llvm/src': { - 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project@d222fa4521531cc4ac14b8e157d231c108c003be', -- 'condition': 'not build_with_chromium', -+ 'condition': '(build_with_swiftshader or (angle_enable_cl and angle_enable_vulkan)) and not build_with_chromium', - }, - - 'third_party/jdk': { -@@ -824,12 +836,12 @@ deps = { - - 'third_party/libc++/src': { - 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxx.git@6a68fd412b9aecd515a20a7cf84d11b598bfaf96', -- 'condition': 'not build_with_chromium', -+ 'condition': 'use_custom_libcxx and not build_with_chromium', - }, - - 'third_party/libc++abi/src': { - 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxxabi.git@9a1d90c3b412d5ebeb97a6e33d98e1d0dd923221', -- 'condition': 'not build_with_chromium', -+ 'condition': 'export_libcxxapi_from_executables and not build_with_chromium', - }, - - 'third_party/libunwind/src': { -@@ -872,7 +884,7 @@ deps = { - - 'third_party/OpenCL-CTS/src': { - 'url': Var('chromium_git') + '/external/github.com/KhronosGroup/OpenCL-CTS@e0a31a03fc8f816d59fd8b3051ac6a61d3fa50c6', -- 'condition': 'not build_with_chromium', -+ 'condition': 'angle_enable_cl_testing and not build_with_chromium', - }, - - 'third_party/OpenCL-Docs/src': { -@@ -968,7 +980,7 @@ deps = { - - 'third_party/SwiftShader': { - 'url': Var('swiftshader_git') + '/SwiftShader@7a9a492a38b7c701f7c96a15a76046aed8f8c0c3', -- 'condition': 'not build_with_chromium', -+ 'condition': 'build_with_swiftshader and not build_with_chromium', - }, - - 'third_party/turbine/cipd': { -@@ -984,6 +996,7 @@ deps = { - - 'third_party/VK-GL-CTS/src': { - 'url': Var('chromium_git') + '/external/github.com/KhronosGroup/VK-GL-CTS' + '@' + Var('vk_gl_cts_revision'), -+ 'condition': 'build_angle_deqp_tests', - }, - - 'third_party/vulkan-deps': { -@@ -1038,7 +1051,7 @@ deps = { - - 'third_party/vulkan-validation-layers/src': { - 'url': '{chromium_git}/external/github.com/KhronosGroup/Vulkan-ValidationLayers@b63e9bd51fbd7bf8fea161a4f7c06994abc24b75', -- 'condition': 'not build_with_chromium', -+ 'condition': 'angle_enable_vulkan_validation_layers and not build_with_chromium', - }, - - 'third_party/vulkan_memory_allocator': { --- -2.46.2 - diff --git a/.gitlab-ci/cuttlefish-runner.sh b/.gitlab-ci/cuttlefish-runner.sh index 088cc3aed45..50fd8d9b9b6 100755 --- a/.gitlab-ci/cuttlefish-runner.sh +++ b/.gitlab-ci/cuttlefish-runner.sh @@ -172,7 +172,7 @@ while [ "$($ADB shell dumpsys SurfaceFlinger | grep GLES:)" = "" ] ; do sleep 1; MESA_RUNTIME_VERSION="$($ADB shell dumpsys SurfaceFlinger | grep GLES:)" if [ "$USE_ANGLE" = 1 ]; then - ANGLE_HASH=$(head -c 12 /angle/hash) + ANGLE_HASH=$(head -c 12 /angle/version) if ! printf "%s" "$MESA_RUNTIME_VERSION" | grep --quiet "${ANGLE_HASH}"; then echo "Fatal: Android is loading a wrong version of the ANGLE libs: ${ANGLE_HASH}" 1>&2 exit 1 diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index e22722f97f8..2a6e25c5b2c 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -30,10 +30,10 @@ variables: DEBIAN_ARM64_TEST_IMAGE_VK_PATH: "debian/arm64_test-vk" DEBIAN_X86_64_TEST_ANDROID_IMAGE_PATH: "debian/x86_64_test-android" - DEBIAN_TEST_ANDROID_TAG: "20250210-libdrm" + DEBIAN_TEST_ANDROID_TAG: "20250212-re-angle" DEBIAN_TEST_GL_TAG: "20250204-fix-deqp" DEBIAN_TEST_VK_TAG: "20250204-fix-deqp" - KERNEL_ROOTFS_TAG: "20250212-angular" + KERNEL_ROOTFS_TAG: "20250212-re-angle" DEBIAN_PYUTILS_IMAGE: "debian/x86_64_pyutils" DEBIAN_PYUTILS_TAG: "20250129-lavacli"