From a3fb667b1d4bda3ca10d4fa20924f61d6c28af4d Mon Sep 17 00:00:00 2001 From: Valentine Burley Date: Mon, 7 Apr 2025 11:05:17 +0200 Subject: [PATCH] ci: Include Fluster vectors in the rootfs Downloading them at runtime proved too slow and unreliable. Keep the logic to upload the vectors to S3 as it's faster, the build job takes around 5-10 minutes when downloading from S3, but 30+ minutes to using Fluster to download the vectors. Move this to the build-fluster.sh script to allow re-using it in the test-video container. Signed-off-by: Valentine Burley Part-of: --- .../common/export-gitlab-job-env-for-dut.sh | 3 -- .gitlab-ci/container/build-fluster.sh | 38 ++++++++++++++----- .gitlab-ci/container/lava_build.sh | 14 ------- .gitlab-ci/fluster/fluster-runner.sh | 13 ------- .gitlab-ci/lava/lava-gitlab-ci.yml | 2 - 5 files changed, 28 insertions(+), 42 deletions(-) diff --git a/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh b/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh index dbf6269c5b1..de6ddede4b0 100755 --- a/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh +++ b/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh @@ -49,7 +49,6 @@ VARS=( FLAKES_CHANNEL FLUSTER_CODECS FLUSTER_FRACTION - FLUSTER_VECTORS_VERSION FREEDRENO_HANGCHECK_MS GALLIUM_DRIVER GALLIVM_PERF @@ -121,8 +120,6 @@ VARS=( S3_RESULTS_UPLOAD SKQP_ASSETS_DIR SKQP_BACKENDS - STORAGE_FORK_HOST_PATH - STORAGE_MAINLINE_HOST_PATH TU_DEBUG VIRGL_HOST_API VIRGL_RENDER_SERVER diff --git a/.gitlab-ci/container/build-fluster.sh b/.gitlab-ci/container/build-fluster.sh index 7afe0ad4541..91bd9ff57fb 100644 --- a/.gitlab-ci/container/build-fluster.sh +++ b/.gitlab-ci/container/build-fluster.sh @@ -10,21 +10,40 @@ # Install fluster in /usr/local. +set -uex + section_start fluster "Install fluster" FLUSTER_REVISION="e997402978f62428fffc8e5a4a709690d9ca9bc5" git clone https://github.com/fluendo/fluster.git --single-branch --no-checkout +export SKIP_UPDATE_FLUSTER_VECTORS=false + +check_fluster() +{ + S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}/vectors.tar.zst" + if curl -L --retry 4 -f --retry-connrefused --retry-delay 30 -s --head \ + "https://${S3_PATH_FLUSTER}"; then + echo "Fluster vectors are up-to-date, skip rebuilding them." + export SKIP_UPDATE_FLUSTER_VECTORS=true + fi +} + +check_fluster "${FDO_UPSTREAM_REPO}" +if ! $SKIP_UPDATE_FLUSTER_VECTORS; then + check_fluster "${CI_PROJECT_PATH}" +fi + pushd fluster || exit -git checkout ${FLUSTER_REVISION} +git checkout "${FLUSTER_REVISION}" popd || exit -if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then +if ! $SKIP_UPDATE_FLUSTER_VECTORS; then # Download the necessary vectors: H264, H265 and VP9 # When updating FLUSTER_REVISION, make sure to update the vectors if necessary or # fluster-runner will report Missing results. - fluster/fluster.py download \ + fluster/fluster.py download -j ${FDO_CI_CONCURRENT:-4} \ JVT-AVC_V1 JVT-FR-EXT JVT-MVC JVT-SVC_V1 \ JCT-VC-3D-HEVC JCT-VC-HEVC_V1 JCT-VC-MV-HEVC JCT-VC-RExt JCT-VC-SCC JCT-VC-SHVC \ VP9-TEST-VECTORS-HIGH VP9-TEST-VECTORS @@ -32,15 +51,14 @@ if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then # Build fluster vectors archive and upload it tar --zstd -cf "vectors.tar.zst" fluster/resources/ ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \ - "https://${S3_PATH_FLUSTER}/vectors.tar.zst" - - touch /lava-files/done - ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done" - - # Don't include the vectors in the rootfs - rm -fr fluster/resources/* + "https://${S3_PATH_FLUSTER}/vectors.tar.zst" fi mv fluster/ /usr/local/ +if $SKIP_UPDATE_FLUSTER_VECTORS; then + curl -L --retry 4 -f --retry-connrefused --retry-delay 30 \ + "${FDO_HTTP_CACHE_URI:-}https://${S3_PATH_FLUSTER}" | tar --zstd -x -C /usr/local/ +fi + section_end fluster diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index f3838ac26b9..4814e8cb2ed 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -20,7 +20,6 @@ set -o xtrace export DEBIAN_FRONTEND=noninteractive : "${LLVM_VERSION:?llvm version not set!}" export FIRMWARE_FILES="${FIRMWARE_FILES}" -export SKIP_UPDATE_FLUSTER_VECTORS=0 check_minio() { @@ -32,22 +31,9 @@ check_minio() fi } -check_fluster() -{ - S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" - if curl -L --retry 4 -f --retry-delay 60 -s \ - "https://${S3_PATH_FLUSTER}/done"; then - echo "Fluster vectors are up-to-date, skip downloading them." - export SKIP_UPDATE_FLUSTER_VECTORS=1 - fi -} - check_minio "${FDO_UPSTREAM_REPO}" check_minio "${CI_PROJECT_PATH}" -check_fluster "${FDO_UPSTREAM_REPO}" -check_fluster "${CI_PROJECT_PATH}" - . .gitlab-ci/container/container_pre_build.sh # Install rust, which we'll be using for deqp-runner. It will be cleaned up at the end. diff --git a/.gitlab-ci/fluster/fluster-runner.sh b/.gitlab-ci/fluster/fluster-runner.sh index 65a8e7fa7c9..e7ccaaa7d1a 100755 --- a/.gitlab-ci/fluster/fluster-runner.sh +++ b/.gitlab-ci/fluster/fluster-runner.sh @@ -13,19 +13,6 @@ if [ -z "$FLUSTER_CODECS" ]; then exit 1 fi -# Check which fluster vectors to get -FLUSTER_VECTORS_HOST_PATH="${STORAGE_MAINLINE_HOST_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" -if [ "$CI_PROJECT_PATH" != "$FDO_UPSTREAM_REPO" ]; then - if ! curl -s -L --retry 4 -f --retry-delay 60 "${FDO_HTTP_CACHE_URI:-}https://${FLUSTER_VECTORS_HOST_PATH}/done"; then - echo "Using Fluster vectors from the fork, cached from mainline is unavailable." - FLUSTER_VECTORS_HOST_PATH="${STORAGE_FORK_HOST_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" - else - echo "Using the cached Fluster vectors." - fi -fi - -curl -L --retry 4 -f --retry-all-errors --retry-delay 60 "${FDO_HTTP_CACHE_URI:-}https://${FLUSTER_VECTORS_HOST_PATH}/vectors.tar.zst" | tar --zstd -x -C /usr/local/ - INSTALL="$PWD/install" # Set up the driver environment. diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml index 11321d46431..5a53e8dd58c 100644 --- a/.gitlab-ci/lava/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava/lava-gitlab-ci.yml @@ -36,8 +36,6 @@ variables: S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}" PIGLIT_NO_WINDOW: 1 VISIBILITY_GROUP: "Collabora+fdo" - STORAGE_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DATA_STORAGE_PATH}" - STORAGE_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DATA_STORAGE_PATH}" before_script: - !reference [.download_s3, before_script] script: