ci: Forward all environment variables to DUTs and crosvm

Instead of the current allowlist in export-gitlab-job-env-for-dut.sh,
filter out unwanted environment variables and forward the rest to
bare-metal and LAVA DUTs, as well as crosvm.

Signed-off-by: Valentine Burley <valentine.burley@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35051>

.gitlab-ci.yml

@@ -127,6 +127,8 @@ variables:
   JOB_PRIORITY: 50
   DATA_STORAGE_PATH: data_storage
   KERNEL_IMAGE_BASE: "https://$S3_HOST/$S3_KERNEL_BUCKET/$KERNEL_REPO/$KERNEL_TAG"
+  # Mesa-specific variables that shouldn't be forwarded to DUTs and crosvm
+  CI_EXCLUDE_ENV_VAR_REGEX: 'SCRIPTS_DIR|RESULTS_DIR'
 
   CI_TRON_JOB_TEMPLATE_PROJECT: &ci-tron-template-project gfx-ci/ci-tron
   CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit ddadab0006e43f1365cd30779f565b444a6538ee

.gitlab-ci/bare-metal/rootfs-setup.sh

@@ -23,7 +23,7 @@ set +x
 
 # Pass through relevant env vars from the gitlab job to the baremetal init script
 echo "Variables passed through:"
-"$CI_COMMON"/export-gitlab-job-env-for-dut.sh | tee $rootfs_dst/set-job-env-vars.sh
+filter_env_vars | tee $rootfs_dst/set-job-env-vars.sh
 
 set -x
 

.gitlab-ci/common/export-gitlab-job-env-for-dut.sh

@@ -1,138 +0,0 @@
-#!/bin/bash
-
-VARS=(
-    ANGLE_TAG
-    ANGLE_TRACE_FILES_TAG
-    ANV_DEBUG
-    ARTIFACTS_BASE_URL
-    ASAN_OPTIONS
-    BASE_SYSTEM_FORK_HOST_PREFIX
-    BASE_SYSTEM_MAINLINE_HOST_PREFIX
-    CI_COMMIT_BRANCH
-    CI_COMMIT_REF_NAME
-    CI_COMMIT_TITLE
-    CI_JOB_ID
-    CI_JOB_NAME
-    CI_JOB_STARTED_AT
-    CI_JOB_URL
-    CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-    CI_MERGE_REQUEST_TITLE
-    CI_NODE_INDEX
-    CI_NODE_TOTAL
-    CI_PAGES_DOMAIN
-    CI_PIPELINE_ID
-    CI_PIPELINE_URL
-    CI_PROJECT_DIR
-    CI_PROJECT_NAME
-    CI_PROJECT_PATH
-    CI_PROJECT_ROOT_NAMESPACE
-    CI_RUNNER_DESCRIPTION
-    CI_SERVER_URL
-    CROSVM_GALLIUM_DRIVER
-    CROSVM_GPU_ARGS
-    CROSVM_TAG
-    CURRENT_SECTION
-    DEQP_BIN_DIR
-    DEQP_FORCE_ASAN
-    DEQP_FRACTION
-    DEQP_RUNNER_MAX_FAILS
-    DEQP_SUITE
-    DEQP_TEMP_DIR
-    DEVICE_NAME
-    DRIVER_NAME
-    EGL_PLATFORM
-    ETNA_MESA_DEBUG
-    FDO_CI_CONCURRENT
-    FDO_HTTP_CACHE_URI
-    FDO_UPSTREAM_REPO
-    FD_MESA_DEBUG
-    FLAKES_CHANNEL
-    FLUSTER_CODECS
-    FLUSTER_FRACTION
-    FLUSTER_TAG
-    FREEDRENO_HANGCHECK_MS
-    GALLIUM_DRIVER
-    GALLIVM_PERF
-    GPU_VERSION
-    GTEST
-    GTEST_FAILS
-    GTEST_FRACTION
-    GTEST_RUNNER_OPTIONS
-    GTEST_SKIPS
-    HWCI_ENABLE_X86_KVM
-    HWCI_FREQ_MAX
-    HWCI_KERNEL_MODULES
-    HWCI_START_WESTON
-    HWCI_START_XORG
-    HWCI_TEST_ARGS
-    HWCI_TEST_SCRIPT
-    INTEL_XE_IGNORE_EXPERIMENTAL_WARNING
-    IR3_SHADER_DEBUG
-    JOB_ARTIFACTS_BASE
-    JOB_RESULTS_PATH
-    JOB_ROOTFS_OVERLAY_PATH
-    KERNEL_IMAGE_BASE
-    KERNEL_IMAGE_NAME
-    LD_LIBRARY_PATH
-    LIBGL_ALWAYS_SOFTWARE
-    LP_NUM_THREADS
-    LVP_POISON_MEMORY
-    MESA_BASE_TAG
-    MESA_BUILD_PATH
-    MESA_DEBUG
-    MESA_GLES_VERSION_OVERRIDE
-    MESA_GLSL_VERSION_OVERRIDE
-    MESA_GL_VERSION_OVERRIDE
-    MESA_IMAGE
-    MESA_IMAGE_PATH
-    MESA_IMAGE_TAG
-    MESA_LOADER_DRIVER_OVERRIDE
-    MESA_SPIRV_LOG_LEVEL
-    MESA_TEMPLATES_COMMIT
-    MESA_VK_ABORT_ON_DEVICE_LOSS
-    MESA_VK_IGNORE_CONFORMANCE_WARNING
-    NIR_DEBUG
-    PANVK_DEBUG
-    PAN_I_WANT_A_BROKEN_VULKAN_DRIVER
-    PAN_MESA_DEBUG
-    PIGLIT_FRACTION
-    PIGLIT_NO_WINDOW
-    PIGLIT_OPTIONS
-    PIGLIT_PLATFORM
-    PIGLIT_REPLAY_ANGLE_ARCH
-    PIGLIT_REPLAY_ARTIFACTS_BASE_URL
-    PIGLIT_REPLAY_DEVICE_NAME
-    PIGLIT_REPLAY_EXTRA_ARGS
-    PIGLIT_REPLAY_LOOP_TIMES
-    PIGLIT_REPLAY_REFERENCE_IMAGES_BASE
-    PIGLIT_REPLAY_SUBCOMMAND
-    PIGLIT_RESULTS
-    PIGLIT_RUNNER_OPTIONS
-    PIGLIT_TAG
-    PIGLIT_TESTS
-    PIGLIT_TRACES_FILE
-    PIPELINE_ARTIFACTS_BASE
-    RADEON_DEBUG
-    S3_HOST
-    S3_JWT_FILE
-    S3_RESULTS_UPLOAD
-    SKQP_ASSETS_DIR
-    SKQP_BACKENDS
-    TU_DEBUG
-    VIRGL_HOST_API
-    VIRGL_RENDER_SERVER
-    VK_DRIVER
-    WAFFLE_PLATFORM
-    ZINK_DEBUG
-    ZINK_DESCRIPTORS
-
-    # Dead code within Mesa CI, but required by virglrender CI
-    # (because they include our files in their CI)
-    VK_DRIVER_FILES
-)
-
-for var in "${VARS[@]}"; do
-  if [ -n "${!var+x}" ]; then
-    echo "export $var=${!var@Q}"
-  fi
-done

.gitlab-ci/crosvm-runner.sh

@@ -84,7 +84,7 @@ set_vsock_context || { echo "Could not generate crosvm vsock CID" >&2; exit 1; }
 # Securely pass the current variables to the crosvm environment
 echo "Variables passed through:"
 SCRIPTS_DIR=$(readlink -en "${0%/*}")
-${SCRIPTS_DIR}/common/export-gitlab-job-env-for-dut.sh | tee ${VM_TEMP_DIR}/crosvm-env.sh
+filter_env_vars | tee ${VM_TEMP_DIR}/crosvm-env.sh
 cp ${SCRIPTS_DIR}/setup-test-env.sh ${VM_TEMP_DIR}/setup-test-env.sh
 
 # Set the crosvm-script as the arguments of the current script

.gitlab-ci/lava/lava-submit.sh

@@ -46,7 +46,7 @@ ROOTFS_URL="$(get_path_to_artifact lava-rootfs.tar.zst)"
 rm -rf results
 mkdir -p results/job-rootfs-overlay/
 
-artifacts/ci-common/export-gitlab-job-env-for-dut.sh > results/job-rootfs-overlay/set-job-env-vars.sh
+filter_env_vars > results/job-rootfs-overlay/set-job-env-vars.sh
 cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/
 cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/
 

.gitlab-ci/setup-test-env.sh

@@ -288,5 +288,58 @@ export -f get_tag_file
 export -f error
 export -f trap_err
 
+function filter_env_vars() {
+    x_off
+    if [[ -n "${S3_JWT:-}" ]]; then
+        echo >&2 "Fatal: S3_JWT is set. This should have been cleared at this point."
+        return 1
+    fi
+
+    local exclude_vars=(
+        # GitLab tokens/passwords
+        CI_JOB_TOKEN
+        CI_DEPLOY_USER
+        CI_DEPLOY_PASSWORD
+        CI_DEPENDENCY_PROXY_PASSWORD
+        CI_REGISTRY_PASSWORD
+        CI_REPOSITORY_URL
+
+        # Shell-managed variables
+        _
+        HOME
+        HOSTNAME
+        OLDPWD
+        PATH
+        PWD
+        TERM
+        XDG_RUNTIME_DIR
+    )
+
+    env -0 | sort -z | while IFS= read -r -d '' line; do
+        [[ "$line" == *=* ]] || continue
+        local varname="${line%%=*}"
+        local value="${line#*=}"
+
+        # Skip certain Mesa-specific variables
+        if echo "$varname" | grep -qxE "$CI_EXCLUDE_ENV_VAR_REGEX"; then
+            echo >&2 "${FUNCNAME[0]}: $varname is not passed to the DUT as it matches the pattern in CI_EXCLUDE_ENV_VAR_REGEX"
+            continue
+        fi
+        # Skip excluded or invalid names
+        if printf '%s\n' "${exclude_vars[@]}" | grep -qxF "$varname"; then
+            echo >&2 "${FUNCNAME[0]}: $varname is not passed to the DUT as it is a variable listed for exclusion in ${FUNCNAME[0]}"
+            continue
+        fi
+        # Skip shell function exports
+        if [[ "$varname" == BASH_FUNC_* ]] || [[ ! "$varname" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
+            continue
+        fi
+
+        printf "export %s=%s\n" "$varname" "${value@Q}"
+    done
+    x_restore
+}
+export -f filter_env_vars
+
 set -E
 trap 'trap_err $?' ERR

.gitlab-ci/test/gitlab-ci.yml

@@ -225,7 +225,7 @@ yaml-toml-shell-py-test:
     HWCI_TEST_SCRIPT: "/install/piglit/piglit-traces.sh"
   script:
     - section_start variables "Variables passed through:"
-    - install/common/export-gitlab-job-env-for-dut.sh
+    - filter_env_vars
     - section_end variables
     - install/piglit/piglit-traces.sh