From dd5d737e6ca4ddd2b48c2d47b7419bd3884031ea Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Thu, 12 Sep 2024 14:36:17 +0100 Subject: [PATCH] ci/lava: Use new pyutils container Instead of unpacking the x86_64_build container and its billion build dependencies every time, switch to using only what's in the minimal pyutils container, and the Python scripts we get as an artifact from the python-test job. Pulling the artifacts from S3 rather than using GitLab is also much more efficient. This should substantially reduce the runtime required to get to testing. Signed-off-by: Daniel Stone Part-of: --- .gitlab-ci/container/debian/x86_64_pyutils.sh | 2 +- .gitlab-ci/lava/lava-gitlab-ci.yml | 45 ++++++++++++------- .gitlab-ci/lava/lava-submit.sh | 4 +- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci/container/debian/x86_64_pyutils.sh b/.gitlab-ci/container/debian/x86_64_pyutils.sh index 156daabd411..94be8fd3265 100644 --- a/.gitlab-ci/container/debian/x86_64_pyutils.sh +++ b/.gitlab-ci/container/debian/x86_64_pyutils.sh @@ -24,6 +24,7 @@ EPHEMERAL=( g++ gcc libc6-dev + perl python3-dev ) @@ -33,7 +34,6 @@ DEPS=( file findutils git - python3-mako python3-pil python3-pip python3-ply diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml index 2a91a66e62d..4aefbef4796 100755 --- a/.gitlab-ci/lava/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava/lava-gitlab-ci.yml @@ -8,8 +8,8 @@ variables: variables: GIT_STRATEGY: none # testing doesn't build anything from source FDO_CI_CONCURRENT: 6 # should be replaced by per-machine definitions - # proxy used to cache data locally - FDO_HTTP_CACHE_URI: "http://caching-proxy/cache/?uri=" + # the dispatchers use this to cache data locally + LAVA_HTTP_CACHE_URI: "http://caching-proxy/cache/?uri=" # base system generated by the container build job, shared between many pipelines BASE_SYSTEM_HOST_PREFIX: "${S3_HOST}/${S3_KERNEL_BUCKET}" BASE_SYSTEM_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${DEBIAN_ARCH}" @@ -17,11 +17,15 @@ variables: # per-job build artifacts JOB_ROOTFS_OVERLAY_PATH: "${JOB_ARTIFACTS_BASE}/job-rootfs-overlay.tar.gz" JOB_RESULTS_PATH: "${JOB_ARTIFACTS_BASE}/results.tar.zst" - S3_ARTIFACT_NAME: "mesa-${ARCH}-default-debugoptimized" + LAVA_S3_ARTIFACT_NAME: "mesa-${ARCH}-default-debugoptimized" + S3_ARTIFACT_NAME: "mesa-python-test" S3_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}" PIGLIT_NO_WINDOW: 1 VISIBILITY_GROUP: "Collabora+fdo" + before_script: + - !reference [.download_s3, before_script] script: + - . artifacts/setup-test-env.sh - ./artifacts/lava/lava-submit.sh artifacts: name: "${CI_PROJECT_NAME}_${CI_JOB_NAME}" @@ -37,8 +41,13 @@ variables: after_script: - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 -s "https://${JOB_RESULTS_PATH}" | tar --zstd -x needs: - - alpine/x86_64_lava_ssh_client - !reference [.required-for-hardware-jobs, needs] + - job: alpine/x86_64_lava_ssh_client + artifacts: false + - job: debian/x86_64_pyutils + artifacts: false + - job: python-test + artifacts: false .lava-test:arm32: variables: @@ -49,14 +58,15 @@ variables: BOOT_METHOD: u-boot extends: - .use-debian/arm64_build # for same $MESA_ARTIFACTS_TAG as in kernel+rootfs_arm32 - - .use-debian/x86_64_build + - .use-debian/x86_64_pyutils - .lava-test - .use-kernel+rootfs-arm needs: - !reference [.lava-test, needs] - - kernel+rootfs_arm32 - - debian/x86_64_build - - debian-arm32 + - job: kernel+rootfs_arm32 + artifacts: false + - job: debian-arm32 + artifacts: false .lava-test-deqp:arm32: extends: @@ -73,16 +83,15 @@ variables: BOOT_METHOD: u-boot extends: - .use-debian/arm64_build # for same $MESA_ARTIFACTS_TAG as in kernel+rootfs_arm64 - - .use-debian/x86_64_build + - .use-debian/x86_64_pyutils - .lava-test - .use-kernel+rootfs-arm - dependencies: - - debian-arm64 needs: - !reference [.lava-test, needs] - - kernel+rootfs_arm64 - - debian/x86_64_build - - debian-arm64 + - job: kernel+rootfs_arm64 + artifacts: false + - job: debian-arm64 + artifacts: false .lava-test-deqp:arm64: variables: @@ -99,13 +108,15 @@ variables: BOOT_METHOD: u-boot extends: - .use-debian/x86_64_build-base # for same $MESA_ARTIFACTS_BASE_TAG as in kernel+rootfs_x86_64 - - .use-debian/x86_64_build + - .use-debian/x86_64_pyutils - .lava-test - .use-kernel+rootfs-x86_64 needs: - !reference [.lava-test, needs] - - kernel+rootfs_x86_64 - - debian-testing + - job: kernel+rootfs_x86_64 + artifacts: false + - job: debian-testing + artifacts: false .lava-test-deqp:x86_64: variables: diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index b600d06185e..675fe5b2d47 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -32,7 +32,7 @@ section_end variables tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ . ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" -ARTIFACT_URL="${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME:?}.tar.zst" +ARTIFACT_URL="${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${LAVA_S3_ARTIFACT_NAME:?}.tar.zst" touch results/lava.log tail -f results/lava.log & @@ -44,7 +44,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \ --kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \ --kernel-external "${FORCE_KERNEL_TAG}" \ --build-url "${ARTIFACT_URL}" \ - --job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \ + --job-rootfs-overlay-url "${LAVA_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \ --job-timeout-min ${JOB_TIMEOUT:-30} \ --first-stage-init artifacts/ci-common/init-stage1.sh \ --ci-project-dir "${CI_PROJECT_DIR}" \