ci: Remove firmware from test-base

Firmware packages continue to grow in size, so stop installing them in the test-base image. The necessary firmware is now collected and uploaded per vendor in an external repository. LAVA devices can opt into optional firmware by specifying the name of the archive via LAVA_FIRMWARE. For bare-metal, Qualcomm firmware required for DUTs in the Google lab is included in the baremetal image. Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/13051 Signed-off-by: Valentine Burley <valentine.burley@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34861>
2025-12-20 07:20:10 +01:00 · 2025-05-07 16:22:32 +02:00 · 2025-05-07 16:22:32 +02:00 · dc483ea924
commit dc483ea924
parent a394b7b64c
14 changed files with 53 additions and 126 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -387,6 +387,7 @@ sanity:
        DEBIAN_TEST_GL_TAG
        DEBIAN_TEST_VK_TAG
        FEDORA_X86_64_BUILD_TAG
+        FIRMWARE_TAG
        KERNEL_TAG
        PKG_REPO_REV
        WINDOWS_X64_BUILD_TAG
--- a/.gitlab-ci/container/baremetal_build.sh
+++ b/.gitlab-ci/container/baremetal_build.sh
@ -42,4 +42,26 @@ if [[ $arch == "arm64" ]]; then
    done

    popd
+
+    # Download and extract the firmware for the Qualcomm DUTs
+    curl --location --fail --retry-connrefused --retry 3 --retry-delay 10 \
+        -O http://ftp.de.debian.org/debian/pool/non-free-firmware/f/firmware-nonfree/firmware-qcom-soc_20230210-5_all.deb
+
+    mkdir firmware/
+    dpkg-deb -x firmware-qcom-soc_20230210-5_all.deb firmware/
+
+    # Copy only the firmware files for the devices in CI
+    install -Dm644 firmware/lib/firmware/qcom/a300_pfp.fw /rootfs-arm64/lib/firmware/qcom/a300_pfp.fw
+    install -Dm644 firmware/lib/firmware/qcom/a300_pm4.fw /rootfs-arm64/lib/firmware/qcom/a300_pm4.fw
+    install -Dm644 firmware/lib/firmware/qcom/a530_pfp.fw /rootfs-arm64/lib/firmware/qcom/a530_pfp.fw
+    install -Dm644 firmware/lib/firmware/qcom/a530_pm4.fw /rootfs-arm64/lib/firmware/qcom/a530_pm4.fw
+    install -Dm644 firmware/lib/firmware/qcom/a530_zap.mdt /rootfs-arm64/lib/firmware/qcom/a530_zap.mdt
+    install -Dm644 firmware/lib/firmware/qcom/a530v3_gpmu.fw2 /rootfs-arm64/lib/firmware/qcom/a530v3_gpmu.fw2
+    install -Dm644 firmware/lib/firmware/qcom/apq8096/a530_zap.mbn /rootfs-arm64/lib/firmware/qcom/apq8096/a530_zap.mbn
+    install -Dm644 firmware/lib/firmware/qcom/a630_gmu.bin /rootfs-arm64/lib/firmware/qcom/a630_gmu.bin
+    install -Dm644 firmware/lib/firmware/qcom/a630_sqe.fw /rootfs-arm64/lib/firmware/qcom/a630_sqe.fw
+    install -Dm644 firmware/lib/firmware/qcom/sdm845/a630_zap.mbn /rootfs-arm64/lib/firmware/qcom/sdm845/a630_zap.mbn
+
+    rm firmware-qcom-soc_20230210-5_all.deb
+    rm -rf firmware/
 fi
--- a/.gitlab-ci/container/debian/test-base.sh
+++ b/.gitlab-ci/container/debian/test-base.sh
@ -21,8 +21,6 @@ sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list.d/*

 echo "deb [trusted=yes] https://gitlab.freedesktop.org/gfx-ci/ci-deb-repo/-/raw/${PKG_REPO_REV}/ ${FDO_DISTRIBUTION_VERSION%-*} main" | tee /etc/apt/sources.list.d/gfx-ci_.list

-echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://deb.debian.org/debian ${FDO_DISTRIBUTION_VERSION%-*} non-free-firmware" | tee /etc/apt/sources.list.d/non-free-firmware.list
-
 : "${LLVM_VERSION:?llvm version not set!}"

 . .gitlab-ci/container/debian/maybe-add-llvm-repo.sh
@ -150,7 +148,6 @@ DEPS=(
 )

 HW_DEPS=(
-    firmware-realtek
    netcat-openbsd
    mount
    python3-distutils
@ -159,25 +156,13 @@ HW_DEPS=(
    zstd
 )

-[ "$DEBIAN_ARCH" = "arm64" ] && ARCH_DEPS=(
-    firmware-linux-nonfree
-    firmware-qcom-media
-)
-[ "$DEBIAN_ARCH" = "armhf" ] && ARCH_DEPS=(
-    firmware-misc-nonfree
-)
-[ "$DEBIAN_ARCH" = "amd64" ] && ARCH_DEPS=(
-    firmware-amd-graphics
-    firmware-misc-nonfree
-)
-
 apt-get update
 apt-get dist-upgrade -y

 apt-get install --purge -y \
      sysvinit-core libelogind0

-apt-get install -y --no-remove "${DEPS[@]}" "${HW_DEPS[@]}" "${ARCH_DEPS[@]}"
+apt-get install -y --no-remove "${DEPS[@]}" "${HW_DEPS[@]}"

 apt-get install -y --no-install-recommends "${EPHEMERAL[@]}"

@ -195,10 +180,6 @@ section_end debian_setup

 . .gitlab-ci/container/build-kdl.sh

-############### Download firmware
-
-. .gitlab-ci/container/get-firmware-from-source.sh / "$FIRMWARE_FILES"
-
 ############### Build mold

 . .gitlab-ci/container/build-mold.sh
--- a/.gitlab-ci/container/get-firmware-from-source.sh
+++ b/.gitlab-ci/container/get-firmware-from-source.sh
@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2046 # we want to arg-split FIRMWARE_FILES
-# shellcheck disable=SC2086 # as above
-# shellcheck disable=SC2116 # as above
-
-set -e
-
-ROOTFS=$1
-FIRMWARE_FILES=$2
-
-if [ -n "${FIRMWARE_FILES:-}" ]; then
-  FIRMWARE=$(jq -s '.' $(echo "$FIRMWARE_FILES"))
-else
-  FIRMWARE=""
-fi
-
-if ! echo "$FIRMWARE" | jq empty; then
-  echo "FIRMWARE contains invalid JSON."
-fi
-
-for item in $(echo "$FIRMWARE" | jq -c '.[]'); do
-  src=$(echo "$item" | jq -r '.src')
-  git_hash=$(echo "$item" | jq -r '.git_hash')
-  dst=$(echo "$item" | jq -r '.dst')
-
-  if [ "$src" = "null" ] || [ "$dst" = "null"  ]; then
-    echo "Missing src or dst for $item."
-    continue
-  fi
-
-  # Remove any trailing slashes from src and dst
-  src=${src%/}
-  dst=${dst%/}
-
-  # Remove any leading slash
-  dst=${dst#/}
-
-  if [ "$(echo "$item" | jq '.files | length')" -eq 0 ]; then
-    echo "No files specified for $item."
-    continue
-  fi
-
-  for file in $(echo "$item" | jq -r '.files[]'); do
-    FIRMWARE_SRC_PATH="${src}/${file}"
-    if [ "$git_hash" != "null" ]; then
-      FIRMWARE_SRC_PATH="${FIRMWARE_SRC_PATH}?h=${git_hash}"
-    fi
-    FIRMWARE_DST_DIR="${ROOTFS}/${dst}"
-
-    curl -L --retry 4 -f --retry-all-errors --retry-delay 60 --create-dirs --output-dir "${FIRMWARE_DST_DIR}" -o "${file}" "${FIRMWARE_SRC_PATH}"
-  done
-
-done
--- a/.gitlab-ci/container/gitlab-ci.yml
+++ b/.gitlab-ci/container/gitlab-ci.yml
@ -365,7 +365,6 @@ fedora/x86_64_build:
 debian/x86_64_test-base:
  extends:
    - .debian-container
-    - .firmware_x86_64
    - .container-builds-base
  variables:
    MESA_IMAGE_TAG: &debian-x86_64_test-base "${DEBIAN_BASE_TAG}--${PKG_REPO_REV}"
@ -391,7 +390,6 @@ debian/arm32_test-base:
    - $FDO_RUNNER_JOB_PRIORITY_TAG_AARCH64
  extends:
    - .debian-container
-    - .firmware_arm32
    - .container-builds-base
  variables:
    MESA_IMAGE_TAG: &debian-arm32_test-base "${DEBIAN_BASE_TAG}--${PKG_REPO_REV}"
@ -419,7 +417,6 @@ debian/arm64_test-base:
    - $FDO_RUNNER_JOB_PRIORITY_TAG_AARCH64
  extends:
    - .debian-container
-    - .firmware_arm64
    - .container-builds-base
  variables:
    MESA_IMAGE_TAG: &debian-arm64_test-base "${DEBIAN_BASE_TAG}--${PKG_REPO_REV}"
@ -643,24 +640,6 @@ debian/arm64_test-vk:
  needs:
    - debian/arm64_test-vk

-# Get firmware directly rather than using package versions.
-# Change DEBIAN_BASE_TAG to add firmware changes.
-# FIRMWARE_FILES is a list of json files arranged by vendor in .gitlab-ci/firmware/*
-.firmware_x86_64:
-  variables:
-    FIRMWARE_FILES: |
-      .gitlab-ci/firmware/i915/adl-fw.json
-      .gitlab-ci/firmware/i915/mtl-fw.json
-
-.firmware_arm64:
-  variables:
-    FIRMWARE_FILES: |
-      .gitlab-ci/firmware/arm/mali/arch10.8/mali-fw.json
-
-.firmware_arm32:
-  variables:
-    FIRMWARE_FILES: |
-
 # x86_64 image with ARM64 & ARM32 kernel & rootfs for baremetal testing
 .debian/baremetal_arm_test:
  extends:
--- a/.gitlab-ci/firmware/arm/mali/arch10.8/mali-fw.json
+++ b/.gitlab-ci/firmware/arm/mali/arch10.8/mali-fw.json
@ -1,8 +0,0 @@
-{
-  "src": "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/arm/mali/arch10.8/",
-  "git_hash": "ad8d5f76c429e5485764a9ecb7a2ce3fbc1386ae",
-  "files": [
-      "mali_csffw.bin"
-  ],
-  "dst": "/lib/firmware/arm/mali/arch10.8/"
-}
--- a/.gitlab-ci/firmware/i915/adl-fw.json
+++ b/.gitlab-ci/firmware/i915/adl-fw.json
@ -1,10 +0,0 @@
-{
-  "src": "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/i915/",
-  "git_hash": "36d0c64694dcee6cf49654f430ab70c22dd647d8",
-  "files": [
-      "adlp_dmc.bin",
-      "adlp_guc_70.bin",
-      "tgl_guc_70.bin"
-  ],
-  "dst": "/lib/firmware/i915/"
-}
--- a/.gitlab-ci/firmware/i915/mtl-fw.json
+++ b/.gitlab-ci/firmware/i915/mtl-fw.json
@ -1,12 +0,0 @@
-{
-  "src": "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/plain/i915/",
-  "git_hash": "ad8d5f76c429e5485764a9ecb7a2ce3fbc1386ae",
-  "files": [
-      "mtl_dmc.bin",
-      "mtl_dmc_ver2_10.bin",
-      "mtl_gsc_1.bin",
-      "mtl_guc_70.bin",
-      "mtl_huc_gsc.bin"
-  ],
-  "dst": "/lib/firmware/i915/"
-}
--- a/.gitlab-ci/image-tags.yml
+++ b/.gitlab-ci/image-tags.yml
@ -19,7 +19,7 @@ include:
  - .gitlab-ci/conditional-build-image-tags.yml

 variables:
-  DEBIAN_BASE_TAG: "20250505-crosvm"
+  DEBIAN_BASE_TAG: "20250515-firmware"

  DEBIAN_BUILD_TAG: "20250515-aarch64"

@ -37,6 +37,8 @@ variables:
  KERNEL_TAG: "v6.14-mesa-dea4"
  KERNEL_REPO: "gfx-ci/linux"
  PKG_REPO_REV: "95bf62c"
+  FIRMWARE_TAG: "36f9bbfa"
+  FIRMWARE_REPO: "gfx-ci/firmware"

  WINDOWS_X64_MSVC_TAG: "20250129-llvm19"

--- a/.gitlab-ci/lava/lava-submit.sh
+++ b/.gitlab-ci/lava/lava-submit.sh
@ -64,6 +64,21 @@ tail -f results/lava.log &
 # Ensure that we are printing the commands that are being executed,
 # making it easier to debug the job in case it fails.
 set -x
+
+# List of optional overlays
+LAVA_EXTRA_OVERLAYS=()
+if [ -n "${LAVA_FIRMWARE:-}" ]; then
+    for fw in $LAVA_FIRMWARE; do
+        LAVA_EXTRA_OVERLAYS+=(
+            - append-overlay
+              --name=linux-firmware
+              --url="https://${BASE_SYSTEM_HOST_PREFIX}/${FIRMWARE_REPO}/${fw}-${FIRMWARE_TAG}.tar"
+              --path="/"
+              --format=tar
+        )
+    done
+fi
+
 PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 	--farm "${FARM}" \
 	--device-type "${DEVICE_TYPE}" \
@ -104,5 +119,6 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 		--compression=zstd \
 		--path="/" \
 		--format=tar \
+	"${LAVA_EXTRA_OVERLAYS[@]}" \
 	- submit \
 	>> results/lava.log
--- a/src/amd/ci/gitlab-ci-inc.yml
+++ b/src/amd/ci/gitlab-ci-inc.yml
@ -245,6 +245,7 @@
 .stoney-test:x86_64:
  variables:
    FLAKES_CHANNEL: "#amd-ci"
+    LAVA_FIRMWARE: amdgpu-lava

 .radeonsi-stoney-test:x86_64:
  extends:
@ -271,6 +272,7 @@
  variables:
    GPU_VERSION: radeonsi-raven
    DRIVER_NAME: radeonsi
+    LAVA_FIRMWARE: amdgpu-lava

 .radv-raven-test:x86_64:
  extends:
@ -279,6 +281,7 @@
    DRIVER_NAME: radv
    GPU_VERSION: radv-raven
    VK_DRIVER: radeon
+    LAVA_FIRMWARE: amdgpu-lava

 .amd-raven-test:x86_64:
  extends:
@ -287,6 +290,7 @@
    DRIVER_NAME: radeonsi
    GPU_VERSION: amd-raven
    VK_DRIVER: radeon
+    LAVA_FIRMWARE: amdgpu-lava

 ############### CI-tron
 .ci-tron-test-amdgpu:
--- a/src/freedreno/ci/gitlab-ci-inc.yml
+++ b/src/freedreno/ci/gitlab-ci-inc.yml
@ -237,6 +237,7 @@
    KERNEL_IMAGE_TYPE: ""
    RUNNER_TAG: mesa-ci-x86-64-lava-sc7180-trogdor-kingoftown
    VK_DRIVER: freedreno
+    LAVA_FIRMWARE: qcom-lava

 # Status: https://lava.collabora.dev/scheduler/device_type/sc7180-trogdor-lazor-limozeen
 .lava-sc7180-trogdor-lazor-limozeen:arm64:
@ -253,6 +254,7 @@
    KERNEL_IMAGE_TYPE: ""
    RUNNER_TAG: mesa-ci-x86-64-lava-sc7180-trogdor-lazor-limozeen
    VK_DRIVER: freedreno
+    LAVA_FIRMWARE: qcom-lava

 # Status: https://lava.collabora.dev/scheduler/device_type/sm8350-hdk
 .lava-sm8350-hdk:arm64:
@ -270,6 +272,7 @@
    KERNEL_IMAGE_TYPE: ""
    RUNNER_TAG: mesa-ci-x86-64-lava-sm8350-hdk
    VK_DRIVER: freedreno
+    LAVA_FIRMWARE: qcom-lava


 .google-freedreno-test-gl:
--- a/src/intel/ci/gitlab-ci-inc.yml
+++ b/src/intel/ci/gitlab-ci-inc.yml
@ -343,6 +343,7 @@
    KERNEL_IMAGE_TYPE: ""
    FLAKES_CHANNEL: "#intel-ci"
    HWCI_FREQ_MAX: "true"
+    LAVA_FIRMWARE: i915

 .anv-test:
  extends:
--- a/src/panfrost/ci/gitlab-ci-inc.yml
+++ b/src/panfrost/ci/gitlab-ci-inc.yml
@ -256,6 +256,7 @@
    KERNEL_IMAGE_TYPE: "image"
    RUNNER_TAG: mesa-ci-x86-64-lava-rk3588-rock-5b
    VK_DRIVER: panfrost
+    LAVA_FIRMWARE: arm-lava

 # Status: https://lava.collabora.dev/scheduler/device_type/sun50i-h6-pine-h64
 .lava-sun50i-h6-pine-h64:arm64: