diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f4ef25e3e9..43a6c186747 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -355,7 +355,7 @@ make git archive: # compress the current folder - tar -cvzf ../$CI_PROJECT_NAME.tar.gz . - - s3_upload ../$CI_PROJECT_NAME.tar.gz "https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/" + - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz # Sanity checks of MR settings and commit logs sanity: diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh index f411722e4f2..45c3b141f06 100755 --- a/.gitlab-ci/common/init-stage2.sh +++ b/.gitlab-ci/common/init-stage2.sh @@ -230,7 +230,7 @@ cleanup # upload artifacts (lava jobs) if [ -n "$S3_RESULTS_UPLOAD" ]; then tar --zstd -cf results.tar.zst results/; - s3_upload results.tar.zst "https://${S3_RESULTS_UPLOAD}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" results.tar.zst https://"$S3_RESULTS_UPLOAD"/results.tar.zst fi # We still need to echo the hwci: mesa message, as some scripts rely on it, such diff --git a/.gitlab-ci/container/build-android-x86_64-llvm.sh b/.gitlab-ci/container/build-android-x86_64-llvm.sh index 78fbb98fcfb..aaacc8bec40 100755 --- a/.gitlab-ci/container/build-android-x86_64-llvm.sh +++ b/.gitlab-ci/container/build-android-x86_64-llvm.sh @@ -114,7 +114,7 @@ tar --zstd -cf "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" "$LLVM_INSTALL_PREFIX" # version does not change, and delete it. # The file is not deleted for non-CI because it can be useful in local runs. if [ -n "$CI" ]; then - s3_upload "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" "https://${S3_HOST}/${S3_ANDROID_BUCKET}/${CI_PROJECT_PATH}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" "https://${S3_HOST}/${S3_ANDROID_BUCKET}/${CI_PROJECT_PATH}/${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" rm "${ANDROID_LLVM_ARTIFACT_NAME}.tar.zst" fi diff --git a/.gitlab-ci/container/build-fluster.sh b/.gitlab-ci/container/build-fluster.sh index 8816e5b7574..18925da6c3a 100644 --- a/.gitlab-ci/container/build-fluster.sh +++ b/.gitlab-ci/container/build-fluster.sh @@ -25,10 +25,11 @@ if [ "${SKIP_UPDATE_FLUSTER_VECTORS}" != 1 ]; then # Build fluster vectors archive and upload it tar --zstd -cf "vectors.tar.zst" fluster/resources/ - s3_upload vectors.tar.zst "https://${S3_PATH_FLUSTER}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "vectors.tar.zst" \ + "https://${S3_PATH_FLUSTER}/vectors.tar.zst" touch /lava-files/done - s3_upload /lava-files/done "https://${S3_PATH_FLUSTER}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done "https://${S3_PATH_FLUSTER}/done" # Don't include the vectors in the rootfs rm -fr fluster/resources/* diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index d18c06a3536..bed763a3b08 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -422,7 +422,8 @@ popd . .gitlab-ci/container/container_post_build.sh -s3_upload /lava-files/"${ROOTFSTAR}" "https://${S3_PATH}/" +ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/"${ROOTFSTAR}" \ + https://${S3_PATH}/"${ROOTFSTAR}" touch /lava-files/done -s3_upload /lava-files/done "https://${S3_PATH}/" +ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done https://${S3_PATH}/done diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index cdd89c6d0ae..14837347b7d 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -51,7 +51,7 @@ cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/ cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/ tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ . -s3_upload job-rootfs-overlay.tar.gz "https://${JOB_ARTIFACTS_BASE}/" +ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" # Prepare env vars for upload. section_switch variables "Environment variables passed through to device:" diff --git a/.gitlab-ci/piglit/piglit-traces.sh b/.gitlab-ci/piglit/piglit-traces.sh index 7b02c8a27e5..dba58e2468b 100755 --- a/.gitlab-ci/piglit/piglit-traces.sh +++ b/.gitlab-ci/piglit/piglit-traces.sh @@ -130,7 +130,8 @@ replay_s3_upload_images() { __DESTINATION_FILE_PATH="$__S3_TRACES_PREFIX/${line##*-}" fi - s3_upload "$RESULTS_DIR/$__PREFIX/$line" "https://${__S3_PATH}/${__DESTINATION_FILE_PATH%/*}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "$RESULTS_DIR/$__PREFIX/$line" \ + "https://${__S3_PATH}/${__DESTINATION_FILE_PATH}" done } diff --git a/.gitlab-ci/prepare-artifacts-python.sh b/.gitlab-ci/prepare-artifacts-python.sh index 8c6e34dacb5..30074ce28a7 100755 --- a/.gitlab-ci/prepare-artifacts-python.sh +++ b/.gitlab-ci/prepare-artifacts-python.sh @@ -53,7 +53,7 @@ if [ -n "$S3_ARTIFACT_NAME" ]; then # Pass needed files to the test stage S3_ARTIFACT_TAR="$S3_ARTIFACT_NAME.tar.zst" tar c artifacts/ | zstd -o "${S3_ARTIFACT_TAR}" - s3_upload "${S3_ARTIFACT_TAR}" "https://${PIPELINE_ARTIFACTS_BASE}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${S3_ARTIFACT_TAR}" "https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_TAR}" rm "${S3_ARTIFACT_TAR}" fi diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index aa46e6897b0..3a3cd866e6f 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -84,7 +84,7 @@ if [ -n "$S3_ARTIFACT_NAME" ]; then # Pass needed files to the test stage S3_ARTIFACT_NAME="$S3_ARTIFACT_NAME.tar.zst" zstd --quiet --threads ${FDO_CI_CONCURRENT:-0} artifacts/install.tar -o ${S3_ARTIFACT_NAME} - s3_upload "${S3_ARTIFACT_NAME}" "https://${PIPELINE_ARTIFACTS_BASE}/" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME} fi section_end prepare-artifacts diff --git a/.gitlab-ci/setup-test-env.sh b/.gitlab-ci/setup-test-env.sh index f141730d159..8df29f9f938 100644 --- a/.gitlab-ci/setup-test-env.sh +++ b/.gitlab-ci/setup-test-env.sh @@ -285,25 +285,6 @@ export -f get_tag_file # Structured tagging ------ -s3_upload() { - x_off - local file=$1 s3_folder_url=$2 - if [ ! -f "$file" ] || [[ "$s3_folder_url" != https://*/ ]] - then - echo "Error: s3_upload incorrect usage." - echo "Usage: s3_upload " - echo " - must exist." - echo " - must start with 'https://' and end with '/'." - exit 1 - fi - curl --fail --retry-all-errors --retry 4 --retry-delay 60 \ - --header "Authorization: Bearer $(cat "${S3_JWT_FILE}")" \ - -X PUT --form file=@"$file" \ - "$s3_folder_url" - x_restore -} -export -f s3_upload - export -f error export -f trap_err