ci: Build gpu-trace-perf and include a script to use it.

The build script is just copy and paste of deqp-runner's.

This will be used to replace piglit's trace replay (and I have plans for
better gitlab CI-based performance testing as well)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/40959>

.gitlab-ci/container/build-gpu-trace-perf.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2086 # we want word splitting
+
+# When changing this file, you need to bump the following
+# .gitlab-ci/image-tags.yml tags:
+# DEBIAN_TEST_BASE_TAG
+
+set -uex
+
+section_start gpu-trace-perf "Building gpu-trace-perf"
+
+GPU_TRACE_PERF_VERSION=1.8.2
+
+commits_to_backport=(
+)
+
+patch_files=(
+)
+
+GPU_TRACE_PERF_GIT_URL="${GPU_TRACE_PERF_GIT_URL:-https://gitlab.freedesktop.org/anholt/gpu-trace-perf.git}"
+if [ -n "${GPU_TRACE_PERF_GIT_TAG:-}" ]; then
+    GPU_TRACE_PERF_GIT_CHECKOUT="$GPU_TRACE_PERF_GIT_TAG"
+elif [ -n "${GPU_TRACE_PERF_GIT_REV:-}" ]; then
+    GPU_TRACE_PERF_GIT_CHECKOUT="$GPU_TRACE_PERF_GIT_REV"
+else
+    GPU_TRACE_PERF_GIT_CHECKOUT="v$GPU_TRACE_PERF_VERSION"
+fi
+
+BASE_PWD=$PWD
+
+mkdir -p /gpu-trace-perf
+pushd /gpu-trace-perf
+mkdir gpu-trace-perf-git
+pushd gpu-trace-perf-git
+git init
+git remote add origin "$GPU_TRACE_PERF_GIT_URL"
+git fetch --depth 1 origin "$GPU_TRACE_PERF_GIT_CHECKOUT"
+git checkout FETCH_HEAD
+
+for commit in "${commits_to_backport[@]}"
+do
+  PATCH_URL="https://gitlab.freedesktop.org/anholt/gpu-trace-perf/-/commit/$commit.patch"
+  echo "Backport gpu-trace-perf commit $commit from $PATCH_URL"
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 $PATCH_URL | git am
+done
+
+for patch in "${patch_files[@]}"
+do
+  echo "Apply patch to gpu-trace-perf from $patch"
+  git am "$BASE_PWD/.gitlab-ci/container/patches/$patch"
+done
+
+if [ -z "${RUST_TARGET:-}" ]; then
+    RUST_TARGET=""
+fi
+
+if [[ "$RUST_TARGET" != *-android ]]; then
+    # When CC (/usr/lib/ccache/gcc) variable is set, the rust compiler uses
+    # this variable when cross-compiling arm32 and build fails for zsys-sys.
+    # So unset the CC variable when cross-compiling for arm32.
+    SAVEDCC=${CC:-}
+    if [ "$RUST_TARGET" = "armv7-unknown-linux-gnueabihf" ]; then
+        unset CC
+    fi
+    cargo install --locked  \
+        -j ${FDO_CI_CONCURRENT:-4} \
+        --root /usr/local \
+        ${EXTRA_CARGO_ARGS:-} \
+        --path .
+    CC=$SAVEDCC
+else
+    cargo install --locked  \
+        -j ${FDO_CI_CONCURRENT:-4} \
+        --root /usr/local --version 2.10.0 \
+        cargo-ndk
+
+    rustup target add $RUST_TARGET
+    RUSTFLAGS='-C target-feature=+crt-static' cargo ndk --target $RUST_TARGET build --release
+
+    mv target/$RUST_TARGET/release/gpu-trace-perf /gpu-trace-perf
+
+    cargo uninstall --locked  \
+        --root /usr/local \
+        cargo-ndk
+fi
+
+popd
+rm -rf gpu-trace-perf-git
+popd
+
+section_end gpu-trace-perf

.gitlab-ci/container/debian/test-base.sh

@@ -232,10 +232,14 @@ if [ "$DEBIAN_ARCH" != "armhf" ]; then
   . .gitlab-ci/container/build-crosvm.sh
 fi
 
-############### Build dEQP runner
+############### Build dEQP runner and gpu-trace-perf
 
 . .gitlab-ci/container/build-deqp-runner.sh
 
+############### Build gpu-trace-perf
+
+. .gitlab-ci/container/build-gpu-trace-perf.sh
+
 ############### Build apitrace
 
 . .gitlab-ci/container/build-apitrace.sh

.gitlab-ci/gpu-trace-replay.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2086 # we want word splitting
+# shellcheck disable=SC1091 # paths only become valid at runtime
+
+. "${SCRIPTS_DIR}/setup-test-env.sh"
+
+section_start traces_prepare "traces: preparing test setup"
+
+set -ex
+
+# Our rootfs may not have "less", which apitrace uses during apitrace dump
+export PAGER=cat  # FIXME: export everywhere
+
+INSTALL=$(realpath -s "$PWD"/install)
+
+if [ -n "${LAVA_HTTP_CACHE_URI:-}" ]; then
+  export EXTRA_ARGS="${EXTRA_ARGS} --download-caching-proxy=${LAVA_HTTP_CACHE_URI}"
+elif [ -n "${CI_TRON_JOB_HTTP_SERVER:-}" ]; then
+  # The caching proxy doesn't appear to be working.
+  # export EXTRA_ARGS="${EXTRA_ARGS} --download-caching-proxy=${CI_TRON_JOB_HTTP_SERVER}/caching_proxy/"
+  true
+elif [ -n "${FDO_HTTP_CACHE_URI:-}" ]; then
+  # FIXME: remove when there is no baremetal traces job anymore.
+  export EXTRA_ARGS="${EXTRA_ARGS} --download-caching-proxy=${FDO_HTTP_CACHE_URI}"
+fi
+
+if [ $GITLAB_USER_LOGIN == "marge-bot" ]; then
+    # When merging the MR, uploading to the permanent storage for .pngs must
+    # succeed.
+    export EXTRA_ARGS="${EXTRA_ARGS} --snapshot-url-must-work"
+fi
+
+# Set up the environment.
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$INSTALL/lib/"
+if [ -n "${VK_DRIVER}" ]; then
+  ARCH=$(uname -m)
+  export VK_DRIVER_FILES="$INSTALL/share/vulkan/icd.d/${VK_DRIVER}_icd.$ARCH.json"
+fi
+
+MESA_VERSION=$(head -1 "$INSTALL/VERSION" | sed 's/\./\\./g')
+
+# Set environment for replay tool executables.
+export PATH="/apitrace/build:/gfxreconstruct/build/bin:$PATH"
+
+echo "Version:"
+apitrace version 2>/dev/null || echo "apitrace not found (Linux)"
+
+if [ "$GALLIUM_DRIVER" = "virpipe" ]; then
+    # replay is to use virpipe, and virgl_test_server llvmpipe
+    export GALLIUM_DRIVER="$GALLIUM_DRIVER"
+
+    GALLIUM_DRIVER=llvmpipe \
+    VTEST_USE_EGL_SURFACELESS=1 \
+    VTEST_USE_GLES=1 \
+    virgl_test_server >"$RESULTS_DIR"/vtest-log.txt 2>&1 &
+
+    sleep 1
+fi
+
+cd $RESULTS_DIR && rm -rf ..?* .[!.]* ./*
+
+if [ -n "$WINE_TAG" ]; then
+  # Are we using the right wine version?
+  ci_tag_test_time_check "WINE_TAG"
+
+  # Set environment for Wine.
+  export WINEDEBUG="-all"
+  export WINEPREFIX="/wineprefix"
+  export WINEESYNC=1
+  export WINEPATH="/apitrace-msvc-win64/bin"
+
+  # This may be useful if you're debugging DXVK loading.
+  #export WINEDEBUG="+loaddll,+module"
+fi
+
+# Disable using fast-linked GPL shaders in DXVK.  Otherwise, we may end up with
+# (subtly, hopefully) flaky rendering when the optimized pipeline gets swapped
+# in.
+export DXVK_CONFIG="dxvk.enableGraphicsPipelineLibrary=False"
+
+# ANGLE: download compiled ANGLE runtime and the compiled restricted traces
+# (all-in-one package).
+if [ -n "$ANGLE_TRACE_FILES_TAG" ]; then
+  ANGLE_DIR="${INSTALL}/traces-db/angle"
+  mkdir -p "${ANGLE_DIR}"
+
+  if [ "$(uname -m)" = "aarch64" ]; then
+    ANGLE_ARCH=arm64
+  else
+    ANGLE_ARCH=x64
+  fi
+
+  FILE="angle-bin-${ANGLE_ARCH}-${ANGLE_TRACE_FILES_TAG}.tar.zst"
+  curl --location --fail --retry-all-errors --retry 4 --retry-delay 60 \
+    --header "Authorization: Bearer $(cat "${S3_JWT_FILE}")" \
+    "https://s3.freedesktop.org/mesa-tracie-private/${FILE}" --output "${FILE}"
+  tar --zstd -xf ${FILE} -C "${ANGLE_DIR}"
+  rm ${FILE}
+
+  EXTRA_ARGS="${EXTRA_ARGS} --traces-db ${INSTALL}/traces-db"
+fi
+
+# Sanity check to ensure that our environment is sufficient to make our tests
+# run against the Mesa built by CI, rather than any installed distro version.
+if [ -z "${VK_DRIVER}" ]; then
+  wflinfo -a gles2 -p wayland | tee /tmp/version.txt | grep "Mesa $MESA_VERSION\(\s\|$\)"
+else
+  vulkaninfo | grep driverInfo | tee /tmp/version.txt | grep "Mesa $MESA_VERSION\(\s\|$\)"
+fi
+
+uncollapsed_section_switch traces "traces: run traces"
+
+# This gets lost in uncollapsed_section_switch.
+set -x
+
+# wrapper to supress +x to avoid spamming the log
+quiet() {
+    set +x
+    "$@"
+    set -x
+}
+
+report_failure() {
+    echo "Review the image changes and get a checksums patch at: ${ARTIFACTS_BASE_URL}/results/index.html"
+    echo "If the new traces look correct to you, you can update the checksums"
+    echo "locally by running:"
+    echo "    ./bin/ci/update_traces_checksum.sh"
+    echo "and resubmit this merge request."
+    exit 1
+}
+report_success() {
+    echo "All image checksums matched.  Results can be viewed at ${ARTIFACTS_BASE_URL}/results/index.html"
+}
+
+if gpu-trace-perf replay \
+  -j ${FDO_CI_CONCURRENT:-4} \
+  --fraction-start ${CI_NODE_INDEX:-1} \
+  --fraction ${CI_NODE_TOTAL} \
+  --output $RESULTS_DIR \
+  --cache-dir $CACHE_DIR \
+  --config $INSTALL/$REPLAY_CONFIG \
+  --device $GPU_VERSION \
+  --jwt "${S3_JWT_FILE}" \
+  --snapshots-url "https://$PIGLIT_REPLAY_REFERENCE_IMAGES_BASE/" \
+  --job-url "https://$JOB_ARTIFACTS_BASE/" \
+  $EXTRA_ARGS \
+  ; then
+    quiet report_success
+else
+    quiet report_failure
+fi

.gitlab-ci/prepare-artifacts.sh

@@ -46,6 +46,7 @@ for f in \
   .gitlab-ci/piglit \
   .gitlab-ci/fossils.yml \
   .gitlab-ci/fossils \
+  .gitlab-ci/gpu-trace-replay.sh \
   .gitlab-ci/crosvm-init.sh \
   .gitlab-ci/*.txt \
   .gitlab-ci/report-flakes.py \

.gitlab-ci/test/gitlab-ci-inc.yml

@@ -124,6 +124,27 @@
   variables:
     WINE_TAG: ${CONDITIONAL_BUILD_WINE_TAG}
 
+.traces-test:
+  artifacts:
+    # We don't specify on:failure, because it's useful for devs to be able to go
+    # look at the actual rendering results of other drivers.
+    name: "${CI_PROJECT_NAME}_${CI_JOB_NAME_SLUG}"
+    paths:
+      - results/
+  variables:
+    CACHE_DIR: $CI_PROJECT_DIR/traces-db/
+    HWCI_TEST_SCRIPT: "install/gpu-trace-replay.sh"
+    # For downloading traces:
+    CI_TRON_HTTP_ARTIFACT__TRACIE_PUBLIC__PATH: /caching_proxy/https://s3.freedesktop.org/mesa-tracie-public/(.*)
+    CI_TRON_HTTP_ARTIFACT__TRACIE_PUBLIC__URL: https://s3.freedesktop.org/mesa-tracie-public/\1
+    CI_TRON_HTTP_ARTIFACT__TRACIE_PRIVATE__PATH: /caching_proxy/https://s3.freedesktop.org/mesa-tracie-private/(.*)
+    CI_TRON_HTTP_ARTIFACT__TRACIE_PRIVATE__URL: https://s3.freedesktop.org/mesa-tracie-private/\1
+    # For uploading renders:
+    # s3.fd.o redirects to hetzner, so we need that too
+    CI_TRON_PROXY__ALLOWED_ENDPOINTS_CATEGORY__S3__FDO: s3.freedesktop.org:443
+    CI_TRON_PROXY__ALLOWED_ENDPOINTS_CATEGORY__S3__BACKEND: fsn1.your-objectstorage.com:443
+
+
 .piglit-traces-test:
   artifacts:
     when: on_failure