ci: build piglit inside baremetal and LAVA's rootfs

v2:
  - Updated the ci-fairy commit to use.

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6388>

.gitlab-ci.yml

@@ -408,7 +408,7 @@ arm64_test:
   extends:
     - .use-arm_test-base
   variables:
-    MESA_IMAGE_TAG: &arm64_test "2020-12-18-lava"
+    MESA_IMAGE_TAG: &arm64_test "2020-12-19-build-piglit"
 
 .use-arm64_test:
   variables:
@@ -422,7 +422,7 @@ armhf_test:
   extends:
     - .use-arm_test-base
   variables:
-    MESA_IMAGE_TAG: &armhf_test "2020-12-18-lava"
+    MESA_IMAGE_TAG: &armhf_test "2020-12-19-build-piglit"
 
 .use-armhf_test:
   variables:

.gitlab-ci/bare-metal/rootfs-setup.sh

@@ -28,6 +28,7 @@ for var in \
     CI_PIPELINE_ID \
     CI_PROJECT_PATH \
     CI_RUNNER_DESCRIPTION \
+    CI_SERVER_URL \
     DEQP_CASELIST_FILTER \
     DEQP_CONFIG \
     DEQP_EXPECTED_RENDERER \
@@ -48,7 +49,18 @@ for var in \
     MESA_GL_VERSION_OVERRIDE \
     MESA_GLSL_VERSION_OVERRIDE \
     MESA_GLES_VERSION_OVERRIDE \
+    MINIO_HOST \
     NIR_VALIDATE \
+    PIGLIT_HTML_SUMMARY \
+    PIGLIT_JUNIT_RESULTS \
+    PIGLIT_PROFILES \
+    PIGLIT_REPLAY_ARTIFACTS_BASE_URL \
+    PIGLIT_REPLAY_DESCRIPTION_FILE \
+    PIGLIT_REPLAY_DEVICE_NAME \
+    PIGLIT_REPLAY_EXTRA_ARGS \
+    PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL \
+    PIGLIT_REPLAY_UPLOAD_TO_MINIO \
+    PIGLIT_RESULTS \
     TRACIE_NO_UNIT_TESTS \
     TRACIE_UPLOAD_TO_MINIO \
     TU_DEBUG \

.gitlab-ci/build-piglit.sh

@@ -10,7 +10,7 @@ git clone https://gitlab.freedesktop.org/mesa/piglit.git --single-branch --no-ch
 pushd /piglit
 git checkout c702d2bbf28b01a18ce613f386a4ffef03f6c0c9
 patch -p1 <$OLDPWD/.gitlab-ci/piglit/disable-vs_in.diff
-cmake -S . -B . -G Ninja -DCMAKE_BUILD_TYPE=Release $PIGLIT_OPTS
+cmake -S . -B . -G Ninja -DCMAKE_BUILD_TYPE=Release $PIGLIT_OPTS $EXTRA_CMAKE_ARGS
 ninja $PIGLIT_BUILD_TARGETS
 find -name .git -o -name '*ninja*' -o -iname '*cmake*' -o -name '*.[chao]' | xargs rm -rf
 rm -rf target_api

.gitlab-ci/container/arm64_test.sh

@@ -2,4 +2,7 @@
 
 arch=arm64
 
+INCLUDE_PIGLIT=1
+PIGLIT_BUILD_TARGETS="piglit_replayer"
+
 . .gitlab-ci/container/baremetal_build.sh

.gitlab-ci/container/baremetal_build.sh

@@ -29,7 +29,9 @@ BAREMETAL_EPHEMERAL=" \
         libstdc++6:$arch \
         libtinfo-dev:$arch \
         libvulkan-dev:$arch \
+        libwaffle-dev:$arch \
         libxcb-keysyms1-dev:$arch \
+        libxkbcommon-dev:$arch \
         python3-dev \
         qt5-default \
         qt5-qmake \

.gitlab-ci/container/lava_build.sh

@@ -73,9 +73,14 @@ apt-get install -y automake \
                    libpython3-dev \
                    libssl-dev \
                    libvulkan-dev \
+                   libwaffle-dev \
                    libxcb-keysyms1-dev \
+                   libxkbcommon-dev \
+                   patch \
                    python3-dev \
                    python3-distutils \
+                   python3-mako \
+                   python3-numpy \
                    python3-serial \
                    qt5-default \
                    qt5-qmake \
@@ -84,7 +89,8 @@ apt-get install -y automake \
 
 
 if [[ "$DEBIAN_ARCH" = "armhf" ]]; then
-    apt-get install -y libegl1-mesa-dev:armhf \
+    apt-get install -y libboost-dev:armhf \
+                       libegl1-mesa-dev:armhf \
                        libelf-dev:armhf \
                        libgbm-dev:armhf \
                        libgles2-mesa-dev:armhf \
@@ -92,8 +98,9 @@ if [[ "$DEBIAN_ARCH" = "armhf" ]]; then
                        libpng-dev:armhf \
                        libpython3-dev:armhf \
                        libvulkan-dev:armhf \
+                       libwaffle-dev:armhf \
                        libxcb-keysyms1-dev:armhf \
-                       libboost-dev:armhf \
+                       libxkbcommon-dev:armhf \
                        qtbase5-dev:armhf
 fi
 
@@ -115,6 +122,13 @@ DEQP_TARGET=surfaceless . .gitlab-ci/build-deqp.sh
 mv /deqp /lava-files/rootfs-${DEBIAN_ARCH}/.
 
 
+############### Build piglit
+if [ -n "$INCLUDE_PIGLIT" ]; then
+    . .gitlab-ci/build-piglit.sh
+    mv /piglit /lava-files/rootfs-${DEBIAN_ARCH}/.
+fi
+
+
 ############### Build apitrace
 . .gitlab-ci/build-apitrace.sh
 mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/apitrace
@@ -137,9 +151,6 @@ rm -rf /renderdoc
 ############### Build libdrm
 EXTRA_MESON_ARGS+=" -D prefix=/libdrm"
 . .gitlab-ci/build-libdrm.sh
-mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/usr/lib/$GCC_ARCH
-find /libdrm/ -name lib\*\.so\* | xargs cp -t /lava-files/rootfs-${DEBIAN_ARCH}/usr/lib/$GCC_ARCH/.
-rm -rf /libdrm
 
 
 ############### Cross-build kernel
@@ -215,9 +226,21 @@ set -e
 
 cp .gitlab-ci/create-rootfs.sh /lava-files/rootfs-${DEBIAN_ARCH}/.
 cp .gitlab-ci/container/llvm-snapshot.gpg.key /lava-files/rootfs-${DEBIAN_ARCH}/.
-chroot /lava-files/rootfs-${DEBIAN_ARCH} sh /create-rootfs.sh
+chroot /lava-files/rootfs-${DEBIAN_ARCH} \
+    sh -c "INCLUDE_PIGLIT=$INCLUDE_PIGLIT sh /create-rootfs.sh"
 rm /lava-files/rootfs-${DEBIAN_ARCH}/create-rootfs.sh
 rm /lava-files/rootfs-${DEBIAN_ARCH}/llvm-snapshot.gpg.key
+
+
+############### Install the built libdrm
+# Dependencies pulled during the creation of the rootfs may overwrite
+# the built libdrm. Hence, we add it after the rootfs has been already
+# created.
+mkdir -p /lava-files/rootfs-${DEBIAN_ARCH}/usr/lib/$GCC_ARCH
+find /libdrm/ -name lib\*\.so\* | xargs cp -t /lava-files/rootfs-${DEBIAN_ARCH}/usr/lib/$GCC_ARCH/.
+rm -rf /libdrm
+
+
 du -ah /lava-files/rootfs-${DEBIAN_ARCH} | sort -h | tail -100
 pushd /lava-files/rootfs-${DEBIAN_ARCH}
   tar cvzf /lava-files/lava-rootfs.tgz .

.gitlab-ci/create-rootfs.sh

@@ -23,8 +23,32 @@ elif [ $DEBIAN_ARCH = amd64 ]; then
                   "
 fi
 
+if [ -n "$INCLUDE_VK_CTS" ]; then
+    VK_CTS_PACKAGES="libvulkan1"
+fi
+
+if [ -n "$INCLUDE_PIGLIT" ]; then
+    PIGLIT_PACKAGES="libwaffle-1-0
+                     libxkbcommon0
+                     python3-lxml
+                     python3-mako
+                     python3-numpy
+                     python3-simplejson
+                    "
+    INSTALL_CI_FAIRY_PACKAGES="git
+                               python3-dev
+                               python3-pip
+                               python3-setuptools
+                               python3-wheel
+                              "
+fi
+
 apt-get -y install --no-install-recommends \
     $ARCH_PACKAGES \
+    $CI_FAIRY_PACKAGES \
+    $INSTALL_CI_FAIRY_PACKAGES \
+    $PIGLIT_PACKAGES \
+    $VK_CTS_PACKAGES \
     ca-certificates \
     curl \
     initramfs-tools \
@@ -45,8 +69,13 @@ apt-get -y install --no-install-recommends \
     wget \
     xz-utils
 
-if [ -n "$INCLUDE_VK_CTS" ]; then
-    apt-get install -y libvulkan1
+if [ -n "$INCLUDE_PIGLIT" ]; then
+    # Needed for ci-fairy, this revision is able to upload files to
+    # MinIO and doesn't depend on git
+    pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@0f1abc24c043e63894085a6bd12f14263e8b29eb
+
+    apt-get purge -y \
+        $INSTALL_CI_FAIRY_PACKAGES
 fi
 
 passwd root -d

.gitlab-ci/lava-gitlab-ci.yml

@@ -1,5 +1,5 @@
 variables:
-  DISTRIBUTION_TAG: "2020-12-18-lava"
+  DISTRIBUTION_TAG: "2020-12-19-build-piglit"
 
 .kernel+rootfs:
   stage: container-2
@@ -10,6 +10,8 @@ variables:
     KERNEL_URL: "https://gitlab.freedesktop.org/tomeu/linux/-/archive/v5.10-rc2-for-mesa-ci/linux-v5.10-rc2-for-mesa-ci.tar.gz"
     UPLOAD_FOR_LAVA: 1
     INSTALL_KERNEL_MODULES: 1
+    INCLUDE_PIGLIT: 1
+    PIGLIT_BUILD_TARGETS: "piglit_replayer"
   script:
     - .gitlab-ci/container/lava_build.sh