CI: convert to use the new S3 server instead of the legacy minio

We don't need to login anymore, but we can't use plain minio commands
now. `ci-fairy` got a helper as `s3cp` to keep an almost identical
API.

Solved Conflicts:
	.gitlab-ci.yml
	.gitlab-ci/common/init-stage2.sh
	.gitlab-ci/container/lava_build.sh
	.gitlab-ci/lava/lava-submit.sh
	.gitlab-ci/piglit/piglit-traces.sh
	.gitlab-ci/prepare-artifacts.sh
	.gitlab-ci/test/gitlab-ci.yml
	src/broadcom/ci/traces-broadcom.yml
	src/freedreno/ci/gitlab-ci.yml
	src/gallium/drivers/crocus/ci/traces-crocus.yml
	src/gallium/drivers/virgl/ci/traces-virgl-iris.yml

Signed-off-by: Benjamin Tissoires <benjamin.tissoires@gmail.com>
(cherry picked from commit 67cee534a8)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19928>
This commit is contained in:
Benjamin Tissoires 2022-10-13 21:25:38 +02:00 committed by Guilherme Gallo
parent 700b671e02
commit c5a2af1167
24 changed files with 70 additions and 51 deletions

View file

@ -5,7 +5,8 @@ variables:
set -o xtrace
/usr/bin/wget -q -O- ${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/download-git-cache.sh | bash -
set +o xtrace
MINIO_HOST: minio-packet.freedesktop.org
CI_JOB_JWT_FILE: /minio_jwt
MINIO_HOST: s3.freedesktop.org
# per-pipeline artifact storage on MinIO
PIPELINE_ARTIFACTS_BASE: ${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
# per-job artifact storage on MinIO
@ -16,9 +17,24 @@ variables:
# running on a particular CI farm (ie. for outages, etc):
FD_FARM: "online"
default:
before_script:
- echo -e "\e[0Ksection_start:$(date +%s):unset_env_vars_section[collapsed=true]\r\e[0KUnsetting vulnerable environment variables"
- echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}"
- unset CI_JOB_JWT
- echo -e "\e[0Ksection_end:$(date +%s):unset_env_vars_section\r\e[0K"
after_script:
- >
set +x
test -e "${CI_JOB_JWT_FILE}" &&
export CI_JOB_JWT="$(<${CI_JOB_JWT_FILE})" &&
rm "${CI_JOB_JWT_FILE}"
include:
- project: 'freedesktop/ci-templates'
ref: 79c325922670137e8f0a4dc5f6f097e0eb57c1af
ref: ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2
file:
- '/templates/ci-fairy.yml'
- project: 'freedesktop/ci-templates'
@ -290,7 +306,7 @@ debian/x86_build-base:
- .fdo.container-build@debian
- .container
variables:
MESA_IMAGE_TAG: &debian-x86_build-base "2021-07-26-python"
MESA_IMAGE_TAG: &debian-x86_build-base "2022-11-16-ci-fairy"
.use-debian/x86_build-base:
extends:
@ -410,13 +426,13 @@ debian/x86_test-gl:
variables:
FDO_DISTRIBUTION_EXEC: 'env KERNEL_URL=${KERNEL_URL} FDO_CI_CONCURRENT=${FDO_CI_CONCURRENT} bash .gitlab-ci/container/${CI_JOB_NAME}.sh'
KERNEL_URL: &kernel-rootfs-url "https://gitlab.freedesktop.org/gfx-ci/linux/-/archive/v5.15-for-mesa-ci-540a4af22d71/linux-v5.15-for-mesa-ci-540a4af22d71.tar.bz2"
MESA_IMAGE_TAG: &debian-x86_test-gl "2021-11-02-lazor"
MESA_IMAGE_TAG: &debian-x86_test-gl "2022-11-13-ci-fairy"
# Debian 11 based x86 test image for VK
debian/x86_test-vk:
extends: .use-debian/x86_test-base
variables:
MESA_IMAGE_TAG: &debian-x86_test-vk "2021-2021-10-15-va"
MESA_IMAGE_TAG: &debian-x86_test-vk "2022-11-13-ci-fairy"
# Debian 11 based ARM build image
debian/arm_build:
@ -426,7 +442,7 @@ debian/arm_build:
tags:
- aarch64
variables:
MESA_IMAGE_TAG: &debian-arm_build "2021-07-26-python"
MESA_IMAGE_TAG: &debian-arm_build "2022-11-16-ci-fairy"
.use-debian/arm_build:
extends:
@ -554,6 +570,8 @@ debian/arm_test:
MESA_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/$MESA_IMAGE_PATH:${MESA_IMAGE_TAG}"
windows_build_vs2019:
inherit:
default: false
extends:
- .container
- .windows-docker-vs2019
@ -580,6 +598,8 @@ windows_build_vs2019:
- when: never
.use-windows_build_vs2019:
inherit:
default: false
extends: .windows-docker-vs2019
image: "$MESA_IMAGE"
needs:
@ -614,9 +634,7 @@ make git archive:
# compress the current folder
- tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
# login with the JWT token
- ci-fairy minio login $CI_JOB_JWT
- ci-fairy minio cp ../$CI_PROJECT_NAME.tar.gz minio://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
- ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
# Sanity checks of MR settings and commit logs
@ -663,11 +681,13 @@ sanity:
CCACHE_DIR: /cache/mesa/ccache
# Use ccache transparently, and print stats before/after
before_script:
- !reference [default, before_script]
- export PATH="/usr/lib/ccache:$PATH"
- export CCACHE_BASEDIR="$PWD"
- ccache --show-stats
after_script:
- ccache --show-stats
- !reference [default, after_script]
.build-windows:
extends: .build-common
@ -1187,6 +1207,7 @@ debian-mingw32-x86_64:
variables:
GIT_STRATEGY: none # testing doesn't build anything from source
before_script:
- !reference [default, before_script]
# Note: Build dir (and thus install) may be dirty due to GIT_STRATEGY
- rm -rf install
- tar -xf artifacts/install.tar
@ -1336,6 +1357,7 @@ debian-mingw32-x86_64:
interruptible: true
stage: test
before_script:
- !reference [default, before_script]
# Use this instead of gitlab's artifacts download because it hits packet.net
# instead of fd.o. Set FDO_HTTP_CACHE_URI to an http cache for your test lab to
# improve it even more (see https://docs.mesa3d.org/ci/bare-metal.html for

View file

@ -16,7 +16,8 @@ set +x
chmod +x $rootfs_dst/set-job-env-vars.sh
echo "Variables passed through:"
cat $rootfs_dst/set-job-env-vars.sh
echo "export CI_JOB_JWT=${CI_JOB_JWT@Q}" >> $rootfs_dst/set-job-env-vars.sh
mkdir -p "$(dirname "$rootfs_dst/${CI_JOB_JWT_FILE}")"
cp "${CI_JOB_JWT_FILE}" "$rootfs_dst/${CI_JOB_JWT_FILE}"
set -x
# Add the Mesa drivers we built, and make a consistent symlink to them.

View file

@ -8,6 +8,7 @@ for var in \
CI_COMMIT_TITLE \
CI_JOB_ID \
CI_JOB_URL \
CI_JOB_JWT_FILE \
CI_MERGE_REQUEST_SOURCE_BRANCH_NAME \
CI_MERGE_REQUEST_TITLE \
CI_NODE_INDEX \

View file

@ -71,8 +71,7 @@ fi
MINIO=$(cat /proc/cmdline | tr ' ' '\n' | grep minio_results | cut -d '=' -f 2 || true)
if [ -n "$MINIO" ]; then
tar -czf results.tar.gz results/;
ci-fairy minio login "$CI_JOB_JWT";
ci-fairy minio cp results.tar.gz minio://"$MINIO"/results.tar.gz;
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.gz https://"$MINIO_RESULTS_UPLOAD"/results.tar.gz;
fi
echo "hwci: mesa: $RESULT"

View file

@ -71,7 +71,7 @@ apt-get -y install --no-install-recommends \
# Needed for ci-fairy, this revision is able to upload files to
# MinIO and doesn't depend on git
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@0f1abc24c043e63894085a6bd12f14263e8b29eb
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2
apt-get purge -y \
$INSTALL_CI_FAIRY_PACKAGES

View file

@ -57,7 +57,7 @@ apt-get -y install \
apt-get install -y --no-remove -t buster \
android-sdk-ext4-utils
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@6f5af7e5574509726c79109e3c147cee95e81366
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2
arch=armhf
. .gitlab-ci/container/cross_build.sh

View file

@ -73,7 +73,7 @@ apt-get install -y --no-remove \
zlib1g-dev
# Needed for ci-fairy, this revision is able to upload files to MinIO
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@6f5af7e5574509726c79109e3c147cee95e81366
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2
############### Uninstall ephemeral packages

View file

@ -59,7 +59,7 @@ apt-get install -y --no-install-recommends \
# Needed for ci-fairy, this revision is able to upload files to MinIO
# and doesn't depend on git
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@0f1abc24c043e63894085a6bd12f14263e8b29eb
pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2
############### Build dEQP runner
. .gitlab-ci/container/build-deqp-runner.sh

View file

@ -17,6 +17,16 @@ check_minio()
check_minio "${FDO_UPSTREAM_REPO}"
check_minio "${CI_PROJECT_PATH}"
# This script is arguably the longest one when if it comes to this line
# Let's make some sanity checks, before failing miserably after waiting a long time
ci-fairy s3cp --help >/dev/null 2>&1 ||
(
ERRCODE=$?
set +x
echo "ci-fairy must have s3cp support. Please update ci-templates package."
exit $ERRCODE
)
. .gitlab-ci/container/container_pre_build.sh
# Install rust, which we'll be using for deqp-runner. It will be cleaned up at the end.
@ -205,7 +215,6 @@ popd
. .gitlab-ci/container/container_post_build.sh
############### Upload the files!
ci-fairy minio login $CI_JOB_JWT
FILES_TO_UPLOAD="lava-rootfs.tgz \
$KERNEL_IMAGE_NAME"
@ -214,9 +223,9 @@ if [[ -n $DEVICE_TREES ]]; then
fi
for f in $FILES_TO_UPLOAD; do
ci-fairy minio cp /lava-files/$f \
minio://${MINIO_PATH}/$f
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \
https://${MINIO_PATH}/$f
done
touch /lava-files/done
ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${MINIO_PATH}/done

View file

@ -22,8 +22,7 @@ cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/
artifacts/ci-common/generate-env.sh > results/job-rootfs-overlay/set-job-env-vars.sh
tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
ci-fairy minio login "${CI_JOB_JWT}"
ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}"
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
touch results/lava.log
tail -f results/lava.log &
@ -39,7 +38,6 @@ artifacts/lava/lava_job_submitter.py \
--ci-project-dir ${CI_PROJECT_DIR} \
--device-type ${DEVICE_TYPE} \
--dtb ${DTB} \
--jwt "${CI_JOB_JWT}" \
--kernel-image-name ${KERNEL_IMAGE_NAME} \
--kernel-image-type "${KERNEL_IMAGE_TYPE}" \
--boot-method ${BOOT_METHOD} \

View file

@ -146,9 +146,6 @@ def generate_lava_yaml(args):
'mkdir -p {}'.format(args.ci_project_dir),
'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.mesa_build_url, args.ci_project_dir),
'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url),
'set +x',
'export CI_JOB_JWT="{}"'.format(args.jwt),
'set -x',
'exec /init-stage2.sh',
]
test['definitions'][0]['repository']['run']['steps'] = init_lines
@ -286,7 +283,6 @@ def main(args):
if args.dump_yaml:
censored_args = args
censored_args.jwt = "jwt-hidden"
print(generate_lava_yaml(censored_args))
if args.validate_only:
@ -341,7 +337,6 @@ if __name__ == '__main__':
parser.add_argument("--kernel-image-type", nargs='?', default="")
parser.add_argument("--boot-method")
parser.add_argument("--lava-tags", nargs='?', default="")
parser.add_argument("--jwt")
parser.add_argument("--validate-only", action='store_true')
parser.add_argument("--dump-yaml", action='store_true')
parser.add_argument("--visibility-group")

View file

@ -3,7 +3,7 @@
set -ex
INSTALL=$(realpath -s "$PWD"/install)
MINIO_ARGS="--credentials=/tmp/.minio_credentials"
MINIO_ARGS="--credentials=/tmp/.minio_credentials --token-file=${CI_JOB_JWT_FILE}"
RESULTS=$(realpath -s "$PWD"/results)
mkdir -p "$RESULTS"
@ -162,8 +162,8 @@ replay_minio_upload_images() {
__DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
fi
ci-fairy minio cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
"minio://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
ci-fairy s3cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
"https://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
done
}
@ -200,11 +200,6 @@ fi
FAILURE_MESSAGE=$(printf "%s" "Unexpected change in results:")
if [ "x$PIGLIT_PROFILES" = "xreplay" ] \
&& [ ${PIGLIT_REPLAY_UPLOAD_TO_MINIO:-0} -eq 1 ]; then
ci-fairy minio login $MINIO_ARGS $CI_JOB_JWT
fi
eval $RUN_CMD
if [ $? -ne 0 ]; then

View file

@ -52,6 +52,5 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then
# Pass needed files to the test stage
MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz"
gzip -c artifacts/install.tar > ${MINIO_ARTIFACT_NAME}
ci-fairy minio login $CI_JOB_JWT
ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
fi

View file

@ -153,12 +153,12 @@ Enable the site and restart nginx:
sudo service nginx restart
# First download will hit the internet
wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
# Second download should be cached.
wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
Now, set ``download-url`` in your ``traces-*.yml`` entry to something like
``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public``
``http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public``
and you should have cached downloads for traces. Add it to
``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you
can use it for cached artifact downloads instead of going all the way to

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
- path: KhronosGroup-Vulkan-Tools/amd/polaris10/vkcube.gfxr

View file

@ -216,7 +216,7 @@ a630-traces-restricted:
- .freedreno-rules-restricted
variables:
PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml"
PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt=${CI_JOB_JWT}
PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE}
allow_failure: true
a630-traces-performance:

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-private/"
download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-private/"
traces:
# Crashes

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
# NOTE for faster runtime, try to sort tests so the longer running traces
# appear first (and therefore are started first).

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
# glmark2 looks like maybe it fails due to wanting an xserver, but we don't have
# Xorg in the x86 docker images we're using.

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
# - path: 0ad/0ad.trace

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
- path: glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true.trace

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
- path: glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true.trace

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
- path: glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true.trace

View file

@ -1,5 +1,5 @@
traces-db:
download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
- path: behdad-glyphy/glyphy.trace