ci: Include Fluster vectors in the rootfs

Downloading them at runtime proved too slow and unreliable.
Keep the logic to upload the vectors to S3 as it's faster, the build job
takes around 5-10 minutes when downloading from S3, but 30+ minutes to
using Fluster to download the vectors.
Move this to the build-fluster.sh script to allow re-using it in the
test-video container.

Signed-off-by: Valentine Burley <valentine.burley@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34451>

.gitlab-ci/common/export-gitlab-job-env-for-dut.sh

@@ -49,7 +49,6 @@ VARS=(
     FLAKES_CHANNEL
     FLUSTER_CODECS
     FLUSTER_FRACTION
-    FLUSTER_VECTORS_VERSION
     FREEDRENO_HANGCHECK_MS
     GALLIUM_DRIVER
     GALLIVM_PERF
@@ -121,8 +120,6 @@ VARS=(
     S3_RESULTS_UPLOAD
     SKQP_ASSETS_DIR
     SKQP_BACKENDS
-    STORAGE_FORK_HOST_PATH
-    STORAGE_MAINLINE_HOST_PATH
     TU_DEBUG
     VIRGL_HOST_API
     VIRGL_RENDER_SERVER

.gitlab-ci/container/build-fluster.sh

@@ -10,21 +10,40 @@
 
 # Install fluster in /usr/local.
 
+set -uex
+
 section_start fluster "Install fluster"
 
 FLUSTER_REVISION="e997402978f62428fffc8e5a4a709690d9ca9bc5"
 
 git clone https://github.com/fluendo/fluster.git --single-branch --no-checkout
 
+export SKIP_UPDATE_FLUSTER_VECTORS=false
+
+check_fluster()
+{
+    S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}/vectors.tar.zst"
+    if curl -L --retry 4 -f --retry-connrefused --retry-delay 30 -s --head \
+      "https://${S3_PATH_FLUSTER}"; then
+        echo "Fluster vectors are up-to-date, skip rebuilding them."
+        export SKIP_UPDATE_FLUSTER_VECTORS=true
+    fi
+}
+
+check_fluster "${FDO_UPSTREAM_REPO}"
+if ! $SKIP_UPDATE_FLUSTER_VECTORS; then
+    check_fluster "${CI_PROJECT_PATH}"
+fi
+
 pushd fluster || exit
-git checkout ${FLUSTER_REVISION}
+git checkout "${FLUSTER_REVISION}"
 popd || exit
 
-if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then
+if ! $SKIP_UPDATE_FLUSTER_VECTORS; then
     # Download the necessary vectors: H264, H265 and VP9
     # When updating FLUSTER_REVISION, make sure to update the vectors if necessary or
     # fluster-runner will report Missing results.
-    fluster/fluster.py download \
+    fluster/fluster.py download -j ${FDO_CI_CONCURRENT:-4} \
 	JVT-AVC_V1 JVT-FR-EXT JVT-MVC JVT-SVC_V1 \
 	JCT-VC-3D-HEVC JCT-VC-HEVC_V1 JCT-VC-MV-HEVC JCT-VC-RExt JCT-VC-SCC JCT-VC-SHVC \
 	VP9-TEST-VECTORS-HIGH VP9-TEST-VECTORS
@@ -32,15 +51,14 @@ if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then
     # Build fluster vectors archive and upload it
     tar --zstd -cf "vectors.tar.zst" fluster/resources/
     ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \
-          "https://${S3_PATH_FLUSTER}/vectors.tar.zst"
-
-    touch /lava-files/done
-    ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done"
-
-    # Don't include the vectors in the rootfs
-    rm -fr fluster/resources/*
+      "https://${S3_PATH_FLUSTER}/vectors.tar.zst"
 fi
 
 mv fluster/ /usr/local/
 
+if $SKIP_UPDATE_FLUSTER_VECTORS; then
+    curl -L --retry 4 -f --retry-connrefused --retry-delay 30 \
+      "${FDO_HTTP_CACHE_URI:-}https://${S3_PATH_FLUSTER}" | tar --zstd -x -C /usr/local/
+fi
+
 section_end fluster

.gitlab-ci/container/lava_build.sh

@@ -20,7 +20,6 @@ set -o xtrace
 export DEBIAN_FRONTEND=noninteractive
 : "${LLVM_VERSION:?llvm version not set!}"
 export FIRMWARE_FILES="${FIRMWARE_FILES}"
-export SKIP_UPDATE_FLUSTER_VECTORS=0
 
 check_minio()
 {
@@ -32,22 +31,9 @@ check_minio()
     fi
 }
 
-check_fluster()
-{
-    S3_PATH_FLUSTER="${S3_HOST}/${S3_KERNEL_BUCKET}/$1/${DATA_STORAGE_PATH}/fluster/${FLUSTER_VECTORS_VERSION}"
-    if curl -L --retry 4 -f --retry-delay 60 -s \
-      "https://${S3_PATH_FLUSTER}/done"; then
-        echo "Fluster vectors are up-to-date, skip downloading them."
-        export SKIP_UPDATE_FLUSTER_VECTORS=1
-    fi
-}
-
 check_minio "${FDO_UPSTREAM_REPO}"
 check_minio "${CI_PROJECT_PATH}"
 
-check_fluster "${FDO_UPSTREAM_REPO}"
-check_fluster "${CI_PROJECT_PATH}"
-
 . .gitlab-ci/container/container_pre_build.sh
 
 # Install rust, which we'll be using for deqp-runner.  It will be cleaned up at the end.

.gitlab-ci/fluster/fluster-runner.sh

@@ -13,19 +13,6 @@ if [ -z "$FLUSTER_CODECS" ]; then
    exit 1
 fi
 
-# Check which fluster vectors to get
-FLUSTER_VECTORS_HOST_PATH="${STORAGE_MAINLINE_HOST_PATH}/fluster/${FLUSTER_VECTORS_VERSION}"
-if [ "$CI_PROJECT_PATH" != "$FDO_UPSTREAM_REPO" ]; then
-  if ! curl -s -L --retry 4 -f --retry-delay 60 "${FDO_HTTP_CACHE_URI:-}https://${FLUSTER_VECTORS_HOST_PATH}/done"; then
-    echo "Using Fluster vectors from the fork, cached from mainline is unavailable."
-    FLUSTER_VECTORS_HOST_PATH="${STORAGE_FORK_HOST_PATH}/fluster/${FLUSTER_VECTORS_VERSION}"
-  else
-    echo "Using the cached Fluster vectors."
-  fi
-fi
-
-curl -L --retry 4 -f --retry-all-errors --retry-delay 60 "${FDO_HTTP_CACHE_URI:-}https://${FLUSTER_VECTORS_HOST_PATH}/vectors.tar.zst" | tar --zstd -x -C /usr/local/
-
 INSTALL="$PWD/install"
 
 # Set up the driver environment.

.gitlab-ci/lava/lava-gitlab-ci.yml

@@ -36,8 +36,6 @@ variables:
     S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}"
     PIGLIT_NO_WINDOW: 1
     VISIBILITY_GROUP: "Collabora+fdo"
-    STORAGE_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DATA_STORAGE_PATH}"
-    STORAGE_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DATA_STORAGE_PATH}"
   before_script:
     - !reference [.download_s3, before_script]
   script: