From 34012d5af32ff9b90d951b4c13c80e9d00e2c242 Mon Sep 17 00:00:00 2001 From: Valentine Burley Date: Thu, 8 May 2025 08:57:59 +0200 Subject: [PATCH] ci: Remove EXTERNAL_KERNEL_TAG variable The EXTERNAL_KERNEL_TAG variable is no longer needed. For LAVA and bare-metal, we can override the KERNEL_TAG variable to fetch both the kernel image and modules from a different tag than the default mainline gfx-ci/linux kernel. For LAVA, this also avoids the issue where jobs using EXTERNAL_KERNEL_TAG would still have mainline kernel modules downloaded by the LAVA overlay. Signed-off-by: Valentine Burley Part-of: --- .gitlab-ci.yml | 5 +---- .gitlab-ci/bare-metal/cros-servo.sh | 9 +------- .gitlab-ci/bare-metal/fastboot.sh | 16 -------------- .gitlab-ci/bare-metal/poe-powered.sh | 22 ++----------------- .gitlab-ci/crosvm-runner.sh | 2 +- .gitlab-ci/lava/lava-submit.sh | 3 +-- .gitlab-ci/lava/lava_job_submitter.py | 1 - .gitlab-ci/lava/utils/lava_job_definition.py | 7 ------ .gitlab-ci/lava/utils/uart_job_definition.py | 3 --- src/amd/ci/gitlab-ci-inc.yml | 6 ++--- src/broadcom/ci/gitlab-ci-inc.yml | 6 ----- .../drivers/llvmpipe/ci/gitlab-ci-inc.yml | 2 -- .../drivers/softpipe/ci/gitlab-ci-inc.yml | 2 -- src/gallium/drivers/svga/ci/gitlab-ci.yml | 2 +- .../drivers/virgl/ci/gitlab-ci-inc.yml | 2 -- .../frontends/lavapipe/ci/gitlab-ci-inc.yml | 2 -- src/panfrost/ci/gitlab-ci-inc.yml | 2 +- 17 files changed, 11 insertions(+), 81 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00a99fb10f0..a7d001b2a75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,6 +125,7 @@ variables: CI_TRON_JOB_PRIORITY_TAG: ci-tron:priority:low JOB_PRIORITY: 50 DATA_STORAGE_PATH: data_storage + KERNEL_IMAGE_BASE: "https://$S3_HOST/$S3_KERNEL_BUCKET/$KERNEL_REPO/$KERNEL_TAG" CI_TRON_JOB_TEMPLATE_PROJECT: &ci-tron-template-project gfx-ci/ci-tron CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit e4ab9c5ecc941c7893d991cf723cde877b5dca8d @@ -137,10 +138,6 @@ default: S3_JWT: aud: https://s3.freedesktop.org before_script: - - | - if [ -z "${KERNEL_IMAGE_BASE:-}" ]; then - export KERNEL_IMAGE_BASE="https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${EXTERNAL_KERNEL_TAG:-$KERNEL_TAG}" - fi - > export SCRIPTS_DIR=$(mktemp -d) && curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" && diff --git a/.gitlab-ci/bare-metal/cros-servo.sh b/.gitlab-ci/bare-metal/cros-servo.sh index 6106bf0d0e7..f95886afddb 100755 --- a/.gitlab-ci/bare-metal/cros-servo.sh +++ b/.gitlab-ci/bare-metal/cros-servo.sh @@ -89,14 +89,7 @@ mkdir -p /nfs/results rm -rf /tftp/* if echo "$BM_KERNEL" | grep -q http; then curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - $BM_KERNEL -o /tftp/vmlinuz -elif [ -n "${EXTERNAL_KERNEL_TAG}" ]; then - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o /tftp/vmlinuz - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst - tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "/nfs/" - rm modules.tar.zst & + "$BM_KERNEL" -o /tftp/vmlinuz else cp /baremetal-files/"$BM_KERNEL" /tftp/vmlinuz fi diff --git a/.gitlab-ci/bare-metal/fastboot.sh b/.gitlab-ci/bare-metal/fastboot.sh index f1bbf1b9d3b..7142f72fd24 100755 --- a/.gitlab-ci/bare-metal/fastboot.sh +++ b/.gitlab-ci/bare-metal/fastboot.sh @@ -102,22 +102,6 @@ if echo "$BM_KERNEL $BM_DTB" | grep -q http; then "$BM_DTB" -o dtb cat kernel dtb > Image.gz-dtb - -elif [ -n "${EXTERNAL_KERNEL_TAG}" ]; then - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o kernel - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst - - if [ -n "$BM_DTB" ]; then - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_DTB}.dtb" -o dtb - fi - - cat kernel dtb > Image.gz-dtb || echo "No DTB available, using pure kernel." - rm kernel - tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "$BM_ROOTFS/" - rm modules.tar.zst & else cat /baremetal-files/"$BM_KERNEL" /baremetal-files/"$BM_DTB".dtb > Image.gz-dtb cp /baremetal-files/"$BM_DTB".dtb dtb diff --git a/.gitlab-ci/bare-metal/poe-powered.sh b/.gitlab-ci/bare-metal/poe-powered.sh index 26f716c74eb..4325dffc6ef 100755 --- a/.gitlab-ci/bare-metal/poe-powered.sh +++ b/.gitlab-ci/bare-metal/poe-powered.sh @@ -103,29 +103,11 @@ if [ -f "${BM_BOOTFS}" ]; then BM_BOOTFS=/tmp/bootfs fi -# If BM_KERNEL and BM_DTS is present -if [ -n "${EXTERNAL_KERNEL_TAG}" ]; then - if [ -z "${BM_KERNEL}" ] || [ -z "${BM_DTB}" ]; then - echo "This machine cannot be tested with external kernel since BM_KERNEL or BM_DTB missing!" - exit 1 - fi - - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o "${BM_KERNEL}" - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_DTB}.dtb" -o "${BM_DTB}.dtb" - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ - "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst -fi - date +'%F %T' # Install kernel modules (it could be either in /lib/modules or # /usr/lib/modules, but we want to install in the latter) -if [ -n "${EXTERNAL_KERNEL_TAG}" ]; then - tar --keep-directory-symlink --zstd -xf modules.tar.zst -C /nfs/ - rm modules.tar.zst & -elif [ -n "${BM_BOOTFS}" ]; then +if [ -n "${BM_BOOTFS}" ]; then [ -d $BM_BOOTFS/usr/lib/modules ] && rsync -a $BM_BOOTFS/usr/lib/modules/ /nfs/usr/lib/modules/ [ -d $BM_BOOTFS/lib/modules ] && rsync -a $BM_BOOTFS/lib/modules/ /nfs/lib/modules/ else @@ -136,7 +118,7 @@ fi date +'%F %T' # Install kernel image + bootloader files -if [ -n "${EXTERNAL_KERNEL_TAG}" ] || [ -z "$BM_BOOTFS" ]; then +if [ -z "$BM_BOOTFS" ]; then mv "${BM_KERNEL}" "${BM_DTB}.dtb" /tftp/ else # BM_BOOTFS rsync -aL --delete $BM_BOOTFS/boot/ /tftp/ diff --git a/.gitlab-ci/crosvm-runner.sh b/.gitlab-ci/crosvm-runner.sh index 75638fcf6f8..fd0379e88c0 100755 --- a/.gitlab-ci/crosvm-runner.sh +++ b/.gitlab-ci/crosvm-runner.sh @@ -122,7 +122,7 @@ fi if [ ! -f "/kernel/${KERNEL_IMAGE_NAME:-bzImage}" ]; then mkdir -p /kernel - # shellcheck disable=SC2153 # KERNEL_IMAGE_BASE is set in the default before_script + # shellcheck disable=SC2153 curl -L --retry 4 -f --retry-connrefused --retry-delay 30 \ -o "/kernel/${KERNEL_IMAGE_NAME:-bzImage}" "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH:-amd64}/${KERNEL_IMAGE_NAME:-bzImage}" fi diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index 14837347b7d..a2185884dd7 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -73,7 +73,6 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \ --pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \ --rootfs-url "${ROOTFS_URL}" \ --kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \ - --kernel-external "${EXTERNAL_KERNEL_TAG}" \ --first-stage-init artifacts/ci-common/init-stage1.sh \ --dtb-filename "${DTB}" \ --jwt-file "${S3_JWT_FILE}" \ @@ -100,7 +99,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \ --path="/" \ --format=tar \ - append-overlay \ - --name=extra-modules \ + --name=kernel-modules \ --url="${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" \ --compression=zstd \ --path="/" \ diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index c62ee5afd57..afe35e2a1fc 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -407,7 +407,6 @@ class LAVAJobSubmitter(PathResolver): kernel_image_name: str = None kernel_image_type: str = "" kernel_url_prefix: str = None - kernel_external: str = None lava_tags: str | tuple[str, ...] = () # Comma-separated LAVA tags for the job mesa_job_name: str = "mesa_ci_job" pipeline_info: str = "" diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py index 721c36d0a40..14cca70fcc1 100644 --- a/.gitlab-ci/lava/utils/lava_job_definition.py +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -202,13 +202,6 @@ class LAVAJobDefinition: f"{self.job_submitter.dtb_filename}.dtb" } - def attach_external_modules(self, deploy_field): - if self.job_submitter.kernel_external: - deploy_field["modules"] = { - "url": f"{self.job_submitter.kernel_url_prefix}/modules.tar.zst", - "compression": "zstd" - } - def jwt_steps(self): """ This function is responsible for setting up the SSH server in the DUT and to diff --git a/.gitlab-ci/lava/utils/uart_job_definition.py b/.gitlab-ci/lava/utils/uart_job_definition.py index be06f61f453..7432d810588 100644 --- a/.gitlab-ci/lava/utils/uart_job_definition.py +++ b/.gitlab-ci/lava/utils/uart_job_definition.py @@ -61,7 +61,6 @@ def fastboot_deploy_actions( # URLs to our kernel rootfs to boot from, both generated by the base # container build job_definition.attach_kernel_and_dtb(fastboot_deploy_prepare["images"]) - job_definition.attach_external_modules(fastboot_deploy_nfs) return (fastboot_deploy_nfs, fastboot_deploy_prepare, fastboot_deploy) @@ -78,7 +77,6 @@ def tftp_deploy_actions(job_definition: "LAVAJobDefinition", nfsrootfs) -> tuple "nfsrootfs": nfsrootfs, } job_definition.attach_kernel_and_dtb(tftp_deploy) - job_definition.attach_external_modules(tftp_deploy) return (tftp_deploy,) @@ -96,7 +94,6 @@ def qemu_deploy_actions(job_definition: "LAVAJobDefinition", nfsrootfs) -> tuple "nfsrootfs": nfsrootfs, }, } - job_definition.attach_external_modules(qemu_deploy) return (qemu_deploy,) diff --git a/src/amd/ci/gitlab-ci-inc.yml b/src/amd/ci/gitlab-ci-inc.yml index 040f9bc3e1c..e2655ddd23e 100644 --- a/src/amd/ci/gitlab-ci-inc.yml +++ b/src/amd/ci/gitlab-ci-inc.yml @@ -202,7 +202,7 @@ RUNNER_TAG: mesa-ci-x86-64-lava-asus-CM1400CXA-dalboz # Force fixed 6.6 kernel, amdgpu doesn't revcover from GPU resets on 6.13 # https://gitlab.freedesktop.org/drm/amd/-/issues/3861 - EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea" + KERNEL_TAG: "v6.6.21-mesa-f8ea" # Status: https://lava.collabora.dev/scheduler/device_type/lenovo-TPad-C13-Yoga-zork .lava-lenovo-TPad-C13-Yoga-zork:x86_64: @@ -218,7 +218,7 @@ RUNNER_TAG: mesa-ci-x86-64-lava-lenovo-TPad-C13-Yoga-zork # Force fixed 6.6 kernel, amdgpu doesn't revcover from GPU resets on 6.13 # https://gitlab.freedesktop.org/drm/amd/-/issues/3861 - EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea" + KERNEL_TAG: "v6.6.21-mesa-f8ea" # Status: https://lava.collabora.dev/scheduler/device_type/hp-x360-14a-cb0001xx-zork .lava-hp-x360-14a-cb0001xx-zork:x86_64: @@ -234,7 +234,7 @@ RUNNER_TAG: mesa-ci-x86-64-lava-hp-x360-14a-cb0001xx-zork # Force fixed 6.6 kernel, amdgpu doesn't revcover from GPU resets on 6.13 # https://gitlab.freedesktop.org/drm/amd/-/issues/3861 - EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea" + KERNEL_TAG: "v6.6.21-mesa-f8ea" ############### LAVA diff --git a/src/broadcom/ci/gitlab-ci-inc.yml b/src/broadcom/ci/gitlab-ci-inc.yml index 15a1a3f2e2f..d0a5bef4d73 100644 --- a/src/broadcom/ci/gitlab-ci-inc.yml +++ b/src/broadcom/ci/gitlab-ci-inc.yml @@ -29,8 +29,6 @@ .vc4-rules: stage: broadcom rules: - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.test, rules] - !reference [.igalia-farm-rules, rules] - !reference [.gl-rules, rules] @@ -56,8 +54,6 @@ .v3d-rules: stage: broadcom rules: - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.test, rules] - !reference [.igalia-farm-rules, rules] - !reference [.gl-rules, rules] @@ -97,8 +93,6 @@ .v3dv-rules: stage: broadcom rules: - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.test, rules] - !reference [.igalia-farm-rules, rules] - !reference [.vulkan-rules, rules] diff --git a/src/gallium/drivers/llvmpipe/ci/gitlab-ci-inc.yml b/src/gallium/drivers/llvmpipe/ci/gitlab-ci-inc.yml index 679f353eb24..d4b14e288fd 100644 --- a/src/gallium/drivers/llvmpipe/ci/gitlab-ci-inc.yml +++ b/src/gallium/drivers/llvmpipe/ci/gitlab-ci-inc.yml @@ -3,8 +3,6 @@ timeout: 12m rules: - !reference [.test, rules] - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.gl-rules, rules] - changes: &llvmpipe_file_list - src/gallium/drivers/llvmpipe/* diff --git a/src/gallium/drivers/softpipe/ci/gitlab-ci-inc.yml b/src/gallium/drivers/softpipe/ci/gitlab-ci-inc.yml index 8d397cd5301..96ecf8f7324 100644 --- a/src/gallium/drivers/softpipe/ci/gitlab-ci-inc.yml +++ b/src/gallium/drivers/softpipe/ci/gitlab-ci-inc.yml @@ -2,8 +2,6 @@ stage: software-renderer rules: - !reference [.test, rules] - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.gl-rules, rules] - changes: &softpipe_file_list - src/gallium/drivers/softpipe/**/* diff --git a/src/gallium/drivers/svga/ci/gitlab-ci.yml b/src/gallium/drivers/svga/ci/gitlab-ci.yml index a7aa798841d..859e26480ad 100644 --- a/src/gallium/drivers/svga/ci/gitlab-ci.yml +++ b/src/gallium/drivers/svga/ci/gitlab-ci.yml @@ -39,5 +39,5 @@ vmware-vmx-piglit:x86_64: DEVICE_TYPE: x86 BOOT_METHOD: ipxe KERNEL_REPO: "blu/linux" - EXTERNAL_KERNEL_TAG: "1106ba88b" + KERNEL_TAG: "1106ba88b" FDO_CI_CONCURRENT: 1 diff --git a/src/gallium/drivers/virgl/ci/gitlab-ci-inc.yml b/src/gallium/drivers/virgl/ci/gitlab-ci-inc.yml index f04395f56e0..cfb41754c20 100644 --- a/src/gallium/drivers/virgl/ci/gitlab-ci-inc.yml +++ b/src/gallium/drivers/virgl/ci/gitlab-ci-inc.yml @@ -2,8 +2,6 @@ stage: layered-backends rules: - !reference [.test, rules] - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.gl-rules, rules] - !reference [.llvmpipe-rules, rules] - changes: &virgl_file_list diff --git a/src/gallium/frontends/lavapipe/ci/gitlab-ci-inc.yml b/src/gallium/frontends/lavapipe/ci/gitlab-ci-inc.yml index 267d558b44e..63fbcf728ba 100644 --- a/src/gallium/frontends/lavapipe/ci/gitlab-ci-inc.yml +++ b/src/gallium/frontends/lavapipe/ci/gitlab-ci-inc.yml @@ -3,8 +3,6 @@ timeout: 15m rules: - !reference [.test, rules] - - if: $EXTERNAL_KERNEL_TAG != null - when: never - !reference [.vulkan-rules, rules] - !reference [.gallium-core-rules, rules] - changes: &lavapipe_file_list diff --git a/src/panfrost/ci/gitlab-ci-inc.yml b/src/panfrost/ci/gitlab-ci-inc.yml index 841f79573e4..bcff3ed4bb0 100644 --- a/src/panfrost/ci/gitlab-ci-inc.yml +++ b/src/panfrost/ci/gitlab-ci-inc.yml @@ -204,7 +204,7 @@ KERNEL_IMAGE_TYPE: "" RUNNER_TAG: mesa-ci-x86-64-lava-mt8192-asurada-spherion-r0 # Force fixed 6.6 kernel, we're not reliably getting a GPU on 6.13 - EXTERNAL_KERNEL_TAG: "v6.6.21-mesa-f8ea" + KERNEL_TAG: "v6.6.21-mesa-f8ea" # Status: https://lava.collabora.dev/scheduler/device_type/mt8195-cherry-tomato-r2 .lava-mt8195-cherry-tomato-r2:arm64: