From cc83b3db5ffd7a2391cb27bbd00d1e7a541bc730 Mon Sep 17 00:00:00 2001 From: Valentine Burley Date: Mon, 19 May 2025 08:38:17 +0200 Subject: [PATCH] ci: Forward all environment variables to DUTs and crosvm Instead of the current allowlist in export-gitlab-job-env-for-dut.sh, filter out unwanted environment variables and forward the rest to bare-metal and LAVA DUTs, as well as crosvm. Signed-off-by: Valentine Burley Part-of: --- .gitlab-ci.yml | 2 + .gitlab-ci/bare-metal/rootfs-setup.sh | 2 +- .../common/export-gitlab-job-env-for-dut.sh | 138 ------------------ .gitlab-ci/crosvm-runner.sh | 2 +- .gitlab-ci/lava/lava-submit.sh | 2 +- .gitlab-ci/setup-test-env.sh | 53 +++++++ .gitlab-ci/test/gitlab-ci.yml | 2 +- 7 files changed, 59 insertions(+), 142 deletions(-) delete mode 100755 .gitlab-ci/common/export-gitlab-job-env-for-dut.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 645b1000904..89919853cf5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,6 +127,8 @@ variables: JOB_PRIORITY: 50 DATA_STORAGE_PATH: data_storage KERNEL_IMAGE_BASE: "https://$S3_HOST/$S3_KERNEL_BUCKET/$KERNEL_REPO/$KERNEL_TAG" + # Mesa-specific variables that shouldn't be forwarded to DUTs and crosvm + CI_EXCLUDE_ENV_VAR_REGEX: 'SCRIPTS_DIR|RESULTS_DIR' CI_TRON_JOB_TEMPLATE_PROJECT: &ci-tron-template-project gfx-ci/ci-tron CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit ddadab0006e43f1365cd30779f565b444a6538ee diff --git a/.gitlab-ci/bare-metal/rootfs-setup.sh b/.gitlab-ci/bare-metal/rootfs-setup.sh index 1bf4e8a2768..90e45a61f36 100644 --- a/.gitlab-ci/bare-metal/rootfs-setup.sh +++ b/.gitlab-ci/bare-metal/rootfs-setup.sh @@ -23,7 +23,7 @@ set +x # Pass through relevant env vars from the gitlab job to the baremetal init script echo "Variables passed through:" -"$CI_COMMON"/export-gitlab-job-env-for-dut.sh | tee $rootfs_dst/set-job-env-vars.sh +filter_env_vars | tee $rootfs_dst/set-job-env-vars.sh set -x diff --git a/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh b/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh deleted file mode 100755 index 735588fedb0..00000000000 --- a/.gitlab-ci/common/export-gitlab-job-env-for-dut.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash - -VARS=( - ANGLE_TAG - ANGLE_TRACE_FILES_TAG - ANV_DEBUG - ARTIFACTS_BASE_URL - ASAN_OPTIONS - BASE_SYSTEM_FORK_HOST_PREFIX - BASE_SYSTEM_MAINLINE_HOST_PREFIX - CI_COMMIT_BRANCH - CI_COMMIT_REF_NAME - CI_COMMIT_TITLE - CI_JOB_ID - CI_JOB_NAME - CI_JOB_STARTED_AT - CI_JOB_URL - CI_MERGE_REQUEST_SOURCE_BRANCH_NAME - CI_MERGE_REQUEST_TITLE - CI_NODE_INDEX - CI_NODE_TOTAL - CI_PAGES_DOMAIN - CI_PIPELINE_ID - CI_PIPELINE_URL - CI_PROJECT_DIR - CI_PROJECT_NAME - CI_PROJECT_PATH - CI_PROJECT_ROOT_NAMESPACE - CI_RUNNER_DESCRIPTION - CI_SERVER_URL - CROSVM_GALLIUM_DRIVER - CROSVM_GPU_ARGS - CROSVM_TAG - CURRENT_SECTION - DEQP_BIN_DIR - DEQP_FORCE_ASAN - DEQP_FRACTION - DEQP_RUNNER_MAX_FAILS - DEQP_SUITE - DEQP_TEMP_DIR - DEVICE_NAME - DRIVER_NAME - EGL_PLATFORM - ETNA_MESA_DEBUG - FDO_CI_CONCURRENT - FDO_HTTP_CACHE_URI - FDO_UPSTREAM_REPO - FD_MESA_DEBUG - FLAKES_CHANNEL - FLUSTER_CODECS - FLUSTER_FRACTION - FLUSTER_TAG - FREEDRENO_HANGCHECK_MS - GALLIUM_DRIVER - GALLIVM_PERF - GPU_VERSION - GTEST - GTEST_FAILS - GTEST_FRACTION - GTEST_RUNNER_OPTIONS - GTEST_SKIPS - HWCI_ENABLE_X86_KVM - HWCI_FREQ_MAX - HWCI_KERNEL_MODULES - HWCI_START_WESTON - HWCI_START_XORG - HWCI_TEST_ARGS - HWCI_TEST_SCRIPT - INTEL_XE_IGNORE_EXPERIMENTAL_WARNING - IR3_SHADER_DEBUG - JOB_ARTIFACTS_BASE - JOB_RESULTS_PATH - JOB_ROOTFS_OVERLAY_PATH - KERNEL_IMAGE_BASE - KERNEL_IMAGE_NAME - LD_LIBRARY_PATH - LIBGL_ALWAYS_SOFTWARE - LP_NUM_THREADS - LVP_POISON_MEMORY - MESA_BASE_TAG - MESA_BUILD_PATH - MESA_DEBUG - MESA_GLES_VERSION_OVERRIDE - MESA_GLSL_VERSION_OVERRIDE - MESA_GL_VERSION_OVERRIDE - MESA_IMAGE - MESA_IMAGE_PATH - MESA_IMAGE_TAG - MESA_LOADER_DRIVER_OVERRIDE - MESA_SPIRV_LOG_LEVEL - MESA_TEMPLATES_COMMIT - MESA_VK_ABORT_ON_DEVICE_LOSS - MESA_VK_IGNORE_CONFORMANCE_WARNING - NIR_DEBUG - PANVK_DEBUG - PAN_I_WANT_A_BROKEN_VULKAN_DRIVER - PAN_MESA_DEBUG - PIGLIT_FRACTION - PIGLIT_NO_WINDOW - PIGLIT_OPTIONS - PIGLIT_PLATFORM - PIGLIT_REPLAY_ANGLE_ARCH - PIGLIT_REPLAY_ARTIFACTS_BASE_URL - PIGLIT_REPLAY_DEVICE_NAME - PIGLIT_REPLAY_EXTRA_ARGS - PIGLIT_REPLAY_LOOP_TIMES - PIGLIT_REPLAY_REFERENCE_IMAGES_BASE - PIGLIT_REPLAY_SUBCOMMAND - PIGLIT_RESULTS - PIGLIT_RUNNER_OPTIONS - PIGLIT_TAG - PIGLIT_TESTS - PIGLIT_TRACES_FILE - PIPELINE_ARTIFACTS_BASE - RADEON_DEBUG - S3_HOST - S3_JWT_FILE - S3_RESULTS_UPLOAD - SKQP_ASSETS_DIR - SKQP_BACKENDS - TU_DEBUG - VIRGL_HOST_API - VIRGL_RENDER_SERVER - VK_DRIVER - WAFFLE_PLATFORM - ZINK_DEBUG - ZINK_DESCRIPTORS - - # Dead code within Mesa CI, but required by virglrender CI - # (because they include our files in their CI) - VK_DRIVER_FILES -) - -for var in "${VARS[@]}"; do - if [ -n "${!var+x}" ]; then - echo "export $var=${!var@Q}" - fi -done diff --git a/.gitlab-ci/crosvm-runner.sh b/.gitlab-ci/crosvm-runner.sh index fd0379e88c0..467bf0a9d2e 100755 --- a/.gitlab-ci/crosvm-runner.sh +++ b/.gitlab-ci/crosvm-runner.sh @@ -84,7 +84,7 @@ set_vsock_context || { echo "Could not generate crosvm vsock CID" >&2; exit 1; } # Securely pass the current variables to the crosvm environment echo "Variables passed through:" SCRIPTS_DIR=$(readlink -en "${0%/*}") -${SCRIPTS_DIR}/common/export-gitlab-job-env-for-dut.sh | tee ${VM_TEMP_DIR}/crosvm-env.sh +filter_env_vars | tee ${VM_TEMP_DIR}/crosvm-env.sh cp ${SCRIPTS_DIR}/setup-test-env.sh ${VM_TEMP_DIR}/setup-test-env.sh # Set the crosvm-script as the arguments of the current script diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index 2ca490205d6..9761f6a1032 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -46,7 +46,7 @@ ROOTFS_URL="$(get_path_to_artifact lava-rootfs.tar.zst)" rm -rf results mkdir -p results/job-rootfs-overlay/ -artifacts/ci-common/export-gitlab-job-env-for-dut.sh > results/job-rootfs-overlay/set-job-env-vars.sh +filter_env_vars > results/job-rootfs-overlay/set-job-env-vars.sh cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/ cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/ diff --git a/.gitlab-ci/setup-test-env.sh b/.gitlab-ci/setup-test-env.sh index 8df29f9f938..32a4eb8f2b4 100644 --- a/.gitlab-ci/setup-test-env.sh +++ b/.gitlab-ci/setup-test-env.sh @@ -288,5 +288,58 @@ export -f get_tag_file export -f error export -f trap_err +function filter_env_vars() { + x_off + if [[ -n "${S3_JWT:-}" ]]; then + echo >&2 "Fatal: S3_JWT is set. This should have been cleared at this point." + return 1 + fi + + local exclude_vars=( + # GitLab tokens/passwords + CI_JOB_TOKEN + CI_DEPLOY_USER + CI_DEPLOY_PASSWORD + CI_DEPENDENCY_PROXY_PASSWORD + CI_REGISTRY_PASSWORD + CI_REPOSITORY_URL + + # Shell-managed variables + _ + HOME + HOSTNAME + OLDPWD + PATH + PWD + TERM + XDG_RUNTIME_DIR + ) + + env -0 | sort -z | while IFS= read -r -d '' line; do + [[ "$line" == *=* ]] || continue + local varname="${line%%=*}" + local value="${line#*=}" + + # Skip certain Mesa-specific variables + if echo "$varname" | grep -qxE "$CI_EXCLUDE_ENV_VAR_REGEX"; then + echo >&2 "${FUNCNAME[0]}: $varname is not passed to the DUT as it matches the pattern in CI_EXCLUDE_ENV_VAR_REGEX" + continue + fi + # Skip excluded or invalid names + if printf '%s\n' "${exclude_vars[@]}" | grep -qxF "$varname"; then + echo >&2 "${FUNCNAME[0]}: $varname is not passed to the DUT as it is a variable listed for exclusion in ${FUNCNAME[0]}" + continue + fi + # Skip shell function exports + if [[ "$varname" == BASH_FUNC_* ]] || [[ ! "$varname" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then + continue + fi + + printf "export %s=%s\n" "$varname" "${value@Q}" + done + x_restore +} +export -f filter_env_vars + set -E trap 'trap_err $?' ERR diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index f7e6b50782c..dda8b1c06cd 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -225,7 +225,7 @@ yaml-toml-shell-py-test: HWCI_TEST_SCRIPT: "/install/piglit/piglit-traces.sh" script: - section_start variables "Variables passed through:" - - install/common/export-gitlab-job-env-for-dut.sh + - filter_env_vars - section_end variables - install/piglit/piglit-traces.sh