diff --git a/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh b/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh index dbf6269c5b1..de6ddede4b0 100755 --- a/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh +++ b/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh @@ -49,7 +49,6 @@ VARS=( FLAKES_CHANNEL FLUSTER_CODECS FLUSTER_FRACTION - FLUSTER_VECTORS_VERSION FREEDRENO_HANGCHECK_MS GALLIUM_DRIVER GALLIVM_PERF @@ -121,8 +120,6 @@ VARS=( S3_RESULTS_UPLOAD SKQP_ASSETS_DIR SKQP_BACKENDS - STORAGE_FORK_HOST_PATH - STORAGE_MAINLINE_HOST_PATH TU_DEBUG VIRGL_HOST_API VIRGL_RENDER_SERVER diff --git a/.gitlab-ci/container/build-fluster.sh b/.gitlab-ci/container/build-fluster.sh index 7afe0ad4541..91bd9ff57fb 100644 --- a/.gitlab-ci/container/build-fluster.sh +++ b/.gitlab-ci/container/build-fluster.sh @@ -10,21 +10,40 @@ # Install fluster in /usr/local. +set -uex + section_start fluster "Install fluster" FLUSTER_REVISION="e997402978f62428fffc8e5a4a709690d9ca9bc5" git clone https://github.com/fluendo/fluster.git --single-branch --no-checkout +export SKIP_UPDATE_FLUSTER_VECTORS=false + +check_fluster() +{ + S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}/vectors.tar.zst" + if curl -L --retry 4 -f --retry-connrefused --retry-delay 30 -s --head \ + "https://${S3_PATH_FLUSTER}"; then + echo "Fluster vectors are up-to-date, skip rebuilding them." + export SKIP_UPDATE_FLUSTER_VECTORS=true + fi +} + +check_fluster "${FDO_UPSTREAM_REPO}" +if ! $SKIP_UPDATE_FLUSTER_VECTORS; then + check_fluster "${CI_PROJECT_PATH}" +fi + pushd fluster || exit -git checkout ${FLUSTER_REVISION} +git checkout "${FLUSTER_REVISION}" popd || exit -if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then +if ! $SKIP_UPDATE_FLUSTER_VECTORS; then # Download the necessary vectors: H264, H265 and VP9 # When updating FLUSTER_REVISION, make sure to update the vectors if necessary or # fluster-runner will report Missing results. - fluster/fluster.py download \ + fluster/fluster.py download -j ${FDO_CI_CONCURRENT:-4} \ JVT-AVC_V1 JVT-FR-EXT JVT-MVC JVT-SVC_V1 \ JCT-VC-3D-HEVC JCT-VC-HEVC_V1 JCT-VC-MV-HEVC JCT-VC-RExt JCT-VC-SCC JCT-VC-SHVC \ VP9-TEST-VECTORS-HIGH VP9-TEST-VECTORS @@ -32,15 +51,14 @@ if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then # Build fluster vectors archive and upload it tar --zstd -cf "vectors.tar.zst" fluster/resources/ ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \ - "https://${S3_PATH_FLUSTER}/vectors.tar.zst" - - touch /lava-files/done - ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done" - - # Don't include the vectors in the rootfs - rm -fr fluster/resources/* + "https://${S3_PATH_FLUSTER}/vectors.tar.zst" fi mv fluster/ /usr/local/ +if $SKIP_UPDATE_FLUSTER_VECTORS; then + curl -L --retry 4 -f --retry-connrefused --retry-delay 30 \ + "${FDO_HTTP_CACHE_URI:-}https://${S3_PATH_FLUSTER}" | tar --zstd -x -C /usr/local/ +fi + section_end fluster diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index f3838ac26b9..4814e8cb2ed 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -20,7 +20,6 @@ set -o xtrace export DEBIAN_FRONTEND=noninteractive : "${LLVM_VERSION:?llvm version not set!}" export FIRMWARE_FILES="${FIRMWARE_FILES}" -export SKIP_UPDATE_FLUSTER_VECTORS=0 check_minio() { @@ -32,22 +31,9 @@ check_minio() fi } -check_fluster() -{ - S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" - if curl -L --retry 4 -f --retry-delay 60 -s \ - "https://${S3_PATH_FLUSTER}/done"; then - echo "Fluster vectors are up-to-date, skip downloading them." - export SKIP_UPDATE_FLUSTER_VECTORS=1 - fi -} - check_minio "${FDO_UPSTREAM_REPO}" check_minio "${CI_PROJECT_PATH}" -check_fluster "${FDO_UPSTREAM_REPO}" -check_fluster "${CI_PROJECT_PATH}" - . .gitlab-ci/container/container_pre_build.sh # Install rust, which we'll be using for deqp-runner. It will be cleaned up at the end. diff --git a/.gitlab-ci/fluster/fluster-runner.sh b/.gitlab-ci/fluster/fluster-runner.sh index 65a8e7fa7c9..e7ccaaa7d1a 100755 --- a/.gitlab-ci/fluster/fluster-runner.sh +++ b/.gitlab-ci/fluster/fluster-runner.sh @@ -13,19 +13,6 @@ if [ -z "$FLUSTER_CODECS" ]; then exit 1 fi -# Check which fluster vectors to get -FLUSTER_VECTORS_HOST_PATH="${STORAGE_MAINLINE_HOST_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" -if [ "$CI_PROJECT_PATH" != "$FDO_UPSTREAM_REPO" ]; then - if ! curl -s -L --retry 4 -f --retry-delay 60 "${FDO_HTTP_CACHE_URI:-}https://${FLUSTER_VECTORS_HOST_PATH}/done"; then - echo "Using Fluster vectors from the fork, cached from mainline is unavailable." - FLUSTER_VECTORS_HOST_PATH="${STORAGE_FORK_HOST_PATH}/fluster/${FLUSTER_VECTORS_VERSION}" - else - echo "Using the cached Fluster vectors." - fi -fi - -curl -L --retry 4 -f --retry-all-errors --retry-delay 60 "${FDO_HTTP_CACHE_URI:-}https://${FLUSTER_VECTORS_HOST_PATH}/vectors.tar.zst" | tar --zstd -x -C /usr/local/ - INSTALL="$PWD/install" # Set up the driver environment. diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml index 11321d46431..5a53e8dd58c 100644 --- a/.gitlab-ci/lava/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava/lava-gitlab-ci.yml @@ -36,8 +36,6 @@ variables: S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}" PIGLIT_NO_WINDOW: 1 VISIBILITY_GROUP: "Collabora+fdo" - STORAGE_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DATA_STORAGE_PATH}" - STORAGE_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DATA_STORAGE_PATH}" before_script: - !reference [.download_s3, before_script] script: