diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1127b9c577d..5dc4ab5f2ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -299,7 +299,7 @@ make git archive: # compress the current folder - tar -cvzf ../$CI_PROJECT_NAME.tar.gz . - - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz + - s3_upload ../$CI_PROJECT_NAME.tar.gz "https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/" # Sanity checks of MR settings and commit logs sanity: diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh index 7043fa3166c..470ec054167 100755 --- a/.gitlab-ci/common/init-stage2.sh +++ b/.gitlab-ci/common/init-stage2.sh @@ -230,7 +230,7 @@ cleanup # upload artifacts if [ -n "$S3_RESULTS_UPLOAD" ]; then tar --zstd -cf results.tar.zst results/; - ci-fairy s3cp --token-file "${S3_JWT_FILE}" results.tar.zst https://"$S3_RESULTS_UPLOAD"/results.tar.zst; + s3_upload results.tar.zst https://"$S3_RESULTS_UPLOAD"/ fi # We still need to echo the hwci: mesa message, as some scripts rely on it, such diff --git a/.gitlab-ci/container/build-android-x86_64-llvm.sh b/.gitlab-ci/container/build-android-x86_64-llvm.sh index 79b2d87819a..5645ec4a3cd 100755 --- a/.gitlab-ci/container/build-android-x86_64-llvm.sh +++ b/.gitlab-ci/container/build-android-x86_64-llvm.sh @@ -110,7 +110,7 @@ tar --zstd -cf "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" "$LLVM_INSTALL_PREFIX" # version does not change, and delete it. # The file is not deleted for non-CI because it can be useful in local runs. if [ -n "$CI" ]; then - ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" "https://${S3_HOST}/${S3_ANDROID_BUCKET}/${CI_PROJECT_PATH}/${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" + s3_upload "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" "https://${S3_HOST}/${S3_ANDROID_BUCKET}/${CI_PROJECT_PATH}/" rm "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" fi diff --git a/.gitlab-ci/container/build-fluster.sh b/.gitlab-ci/container/build-fluster.sh index 18925da6c3a..8816e5b7574 100644 --- a/.gitlab-ci/container/build-fluster.sh +++ b/.gitlab-ci/container/build-fluster.sh @@ -25,11 +25,10 @@ if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then # Build fluster vectors archive and upload it tar --zstd -cf "vectors.tar.zst" fluster/resources/ - ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \ - "https://${S3_PATH_FLUSTER}/vectors.tar.zst" + s3_upload vectors.tar.zst "https://${S3_PATH_FLUSTER}/" touch /lava-files/done - ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done" + s3_upload /lava-files/done "https://${S3_PATH_FLUSTER}/" # Don't include the vectors in the rootfs rm -fr fluster/resources/* diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index 7f617e8b7f6..3ed2f62d14b 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -444,8 +444,7 @@ popd . .gitlab-ci/container/container_post_build.sh -ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/"${ROOTFSTAR}" \ - https://${S3_PATH}/"${ROOTFSTAR}" +s3_upload /lava-files/"${ROOTFSTAR}" "https://${S3_PATH}/" touch /lava-files/done -ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done https://${S3_PATH}/done +s3_upload /lava-files/done "https://${S3_PATH}/" diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index b3a78d15a90..f0ffa2422c8 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -52,7 +52,7 @@ cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/ cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/ tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ . -ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" +s3_upload job-rootfs-overlay.tar.gz "https://${JOB_ARTIFACTS_BASE}" # Prepare env vars for upload. section_switch variables "Environment variables passed through to device:" diff --git a/.gitlab-ci/piglit/piglit-traces.sh b/.gitlab-ci/piglit/piglit-traces.sh index 63f6145e8ab..5e8a689fbb5 100755 --- a/.gitlab-ci/piglit/piglit-traces.sh +++ b/.gitlab-ci/piglit/piglit-traces.sh @@ -13,7 +13,6 @@ set -ex export PAGER=cat # FIXME: export everywhere INSTALL=$(realpath -s "$PWD"/install) -S3_ARGS="--token-file ${S3_JWT_FILE}" export PIGLIT_REPLAY_DESCRIPTION_FILE="$INSTALL/$PIGLIT_TRACES_FILE" @@ -128,8 +127,7 @@ replay_s3_upload_images() { __DESTINATION_FILE_PATH="$__S3_TRACES_PREFIX/${line##*-}" fi - ci-fairy s3cp $S3_ARGS "$RESULTS_DIR/$__PREFIX/$line" \ - "https://${__S3_PATH}/${__DESTINATION_FILE_PATH}" + s3_upload "$RESULTS_DIR/$__PREFIX/$line" "https://${__S3_PATH}/${__DESTINATION_FILE_PATH%/*}" done } diff --git a/.gitlab-ci/prepare-artifacts-python.sh b/.gitlab-ci/prepare-artifacts-python.sh index ae318c53ace..94e79b01630 100755 --- a/.gitlab-ci/prepare-artifacts-python.sh +++ b/.gitlab-ci/prepare-artifacts-python.sh @@ -53,7 +53,7 @@ if [ -n "$S3_ARTIFACT_NAME" ]; then # Pass needed files to the test stage S3_ARTIFACT_TAR="$S3_ARTIFACT_NAME.tar.zst" tar cv artifacts/ | zstd -o "${S3_ARTIFACT_TAR}" - ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${S3_ARTIFACT_TAR}" "https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_TAR}" + s3_upload "${S3_ARTIFACT_TAR}" "https://${PIPELINE_ARTIFACTS_BASE}/" rm "${S3_ARTIFACT_TAR}" fi diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index 3a3cd866e6f..aa46e6897b0 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -84,7 +84,7 @@ if [ -n "$S3_ARTIFACT_NAME" ]; then # Pass needed files to the test stage S3_ARTIFACT_NAME="$S3_ARTIFACT_NAME.tar.zst" zstd --quiet --threads ${FDO_CI_CONCURRENT:-0} artifacts/install.tar -o ${S3_ARTIFACT_NAME} - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME} + s3_upload "${S3_ARTIFACT_NAME}" "https://${PIPELINE_ARTIFACTS_BASE}/" fi section_end prepare-artifacts diff --git a/.gitlab-ci/setup-test-env.sh b/.gitlab-ci/setup-test-env.sh index d8af3c70cff..609e0c643ad 100644 --- a/.gitlab-ci/setup-test-env.sh +++ b/.gitlab-ci/setup-test-env.sh @@ -140,5 +140,21 @@ function trap_err { export -f error export -f trap_err +s3_upload() { + x_off + local file=$1 s3_folder_url=$2 + if [ ! -f "$file" ] || [[ "$s3_folder_url" != https://* ]] + then + echo "s3_upload used incorrectly: first argument is the file, second argument is the s3 folder url" + exit 1 + fi + curl --fail --retry-all-errors --retry 4 --retry-delay 60 \ + --header "Authorization: Bearer $(cat "${S3_JWT_FILE}")" \ + -X PUT --form file=@"$file" \ + "$s3_folder_url" + x_restore +} +export -f s3_upload + set -E trap 'trap_err $?' ERR diff --git a/.pick_status.json b/.pick_status.json index 0f4786339bc..8d332b19b7f 100644 --- a/.pick_status.json +++ b/.pick_status.json @@ -674,7 +674,7 @@ "description": "ci: replace broken s3cp command with a simple curl call", "nominated": true, "nomination_type": 1, - "resolution": 0, + "resolution": 1, "main_sha": null, "because_sha": null, "notes": null