ci: add firmware files to rootfs

Currently only package versions of firmware files are available in the
rootfs.

This commit allows firmware files to be pulled directly from a specific
git hash of a remote repository.

Signed-off-by: Deborah Brouwer <deborah.brouwer@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30770>

.gitlab-ci/container/get-firmware-from-source.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOTFS=$1
+FIRMWARE_FILES=$2
+FIRMWARE=$(jq -s '.' $FIRMWARE_FILES)
+
+if [ -z "$FIRMWARE" ] || [ "$(echo "$FIRMWARE" | jq '. | length')" -eq 0 ]; then
+  echo "FIRMWARE is not set or is empty."
+  exit
+fi
+
+if ! echo "$FIRMWARE" | jq empty; then
+  echo "FIRMWARE contains invalid JSON."
+  exit
+fi
+
+for item in $(echo "$FIRMWARE" | jq -c '.[]'); do
+  src=$(echo "$item" | jq -r '.src')
+  git_hash=$(echo "$item" | jq -r '.git_hash')
+  dst=$(echo "$item" | jq -r '.dst')
+
+  if [ "$src" = "null" ] || [ "$dst" = "null"  ]; then
+    echo "Missing src or dst for $item."
+    continue
+  fi
+
+  # Remove any trailing slashes from src and dst
+  src=${src%/}
+  dst=${dst%/}
+
+  # Remove any leading slash
+  dst=${dst#/}
+
+  if [ "$(echo "$item" | jq '.files | length')" -eq 0 ]; then
+    echo "No files specified for $item."
+    continue
+  fi
+
+  for file in $(echo "$item" | jq -r '.files[]'); do
+    FIRMWARE_SRC_PATH="${src}/${file}"
+    if [ "$git_hash" != "null" ]; then
+      FIRMWARE_SRC_PATH="${FIRMWARE_SRC_PATH}?h=${git_hash}"
+    fi
+    FIRMWARE_DST_DIR="${ROOTFS}/${dst}"
+
+    curl -L --retry 4 -f --retry-all-errors --retry-delay 60 --create-dirs --output-dir "${FIRMWARE_DST_DIR}" -o "${file}" "${FIRMWARE_SRC_PATH}"
+  done
+
+done

.gitlab-ci/container/gitlab-ci.yml

@@ -391,11 +391,18 @@ fedora/x86_64_build:
   needs:
     - fedora/x86_64_build
 
+# Get firmware directly rather than using package versions.
+# Change KERNEL_ROOTFS_TAG to add firmware changes.
+# FIRMWARE_FILES is a list of json files arranged by vendor in src/**/ci/
+.firmware:
+  variables:
+    FIRMWARE_FILES: |
 
 .kernel+rootfs:
   extends:
     - .container+build-rules
     - .debian-container-version
+    - .firmware
   stage: container
   timeout: 90m
   variables:

.gitlab-ci/container/lava_build.sh

@@ -11,6 +11,7 @@ set -o xtrace
 
 export DEBIAN_FRONTEND=noninteractive
 export LLVM_VERSION="${LLVM_VERSION:=15}"
+export FIRMWARE_FILES="${FIRMWARE_FILES}"
 
 check_minio()
 {
@@ -120,6 +121,7 @@ CONTAINER_EPHEMERAL=(
     mmdebstrap
     git
     glslang-tools
+    jq
     libdrm-dev
     libegl1-mesa-dev
     libxext-dev
@@ -168,7 +170,7 @@ apt-get install -y --no-remove \
                    "${CONTAINER_ARCH_PACKAGES[@]}" \
                    ${EXTRA_LOCAL_PACKAGES}
 
-ROOTFS=/lava-files/rootfs-${DEBIAN_ARCH}
+export ROOTFS=/lava-files/rootfs-${DEBIAN_ARCH}
 mkdir -p "$ROOTFS"
 
 # rootfs packages
@@ -236,6 +238,7 @@ mmdebstrap \
     --variant=apt \
     --arch="${DEBIAN_ARCH}" \
     --components main,contrib,non-free-firmware \
+    --customize-hook='.gitlab-ci/container/get-firmware-from-source.sh "$ROOTFS" "$FIRMWARE_FILES"' \
     --include "${PKG_BASE[*]} ${PKG_CI[*]} ${PKG_DEP[*]} ${PKG_MESA_DEP[*]} ${PKG_ARCH[*]}" \
     bookworm \
     "$ROOTFS/" \