gitlab CI: use b2c instead of qemu

This is copied from libinput's CI but as one large change rather than
cherry-picking the process on how to get here. meson-build.sh is synched
with libinput's version - it is a more generic version anyway.

With this change we no longer require separate images for the qemu runs,
our default image is qemu-capable and can be run in qemu via
boot2container (b2c).

Signed-off-by: Peter Hutterer <peter.hutterer@who-t.net>
This commit is contained in:
Peter Hutterer 2023-01-04 15:17:16 +10:00
parent 50ac79078e
commit b2e12fe926
4 changed files with 145 additions and 152 deletions

View file

@ -72,49 +72,49 @@ variables:
.fedora:34:
extends: .fdo.distribution-image@fedora
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: '34'
.fedora:35:
extends: .fdo.distribution-image@fedora
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: '35'
.ubuntu:22.04:
extends: .fdo.distribution-image@ubuntu
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: '22.04'
.ubuntu:22.10:
extends: .fdo.distribution-image@ubuntu
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: '22.10'
.debian:stable:
extends: .fdo.distribution-image@debian
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: 'stable'
.debian:sid:
extends: .fdo.distribution-image@debian
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: 'sid'
.arch:rolling:
extends: .fdo.distribution-image@arch
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: 'rolling'
.alpine:latest:
extends: .fdo.distribution-image@alpine
variables:
FDO_DISTRIBUTION_TAG: '2023-01-04.5'
FDO_DISTRIBUTION_TAG: '2023-01-05.0'
FDO_DISTRIBUTION_VERSION: 'latest'
@ -182,7 +182,7 @@ check-merge-request:
.fedora.packages:
variables:
FDO_DISTRIBUTION_PACKAGES: 'git gcc gcc-c++ meson automake autoconf libtool make pkgconfig python3 check-devel valgrind binutils doxygen xz clang-analyzer'
FDO_DISTRIBUTION_PACKAGES: 'git gcc gcc-c++ meson automake autoconf libtool make pkgconfig python3 check-devel valgrind binutils doxygen xz clang-analyzer systemd-udev qemu-img qemu-system-x86-core qemu-system-aarch64-core jq python3-click python3-rich'
.ubuntu.packages:
variables:
@ -201,36 +201,6 @@ check-merge-request:
FDO_DISTRIBUTION_PACKAGES: 'git gcc g++ meson automake autoconf libtool make pkgconfig python3 check-dev valgrind binutils doxygen xz linux-headers'
# Pulls in the qemu container from upstream or rebuilds it if missing
.fedora:35@qemu-prep:
extends:
- .fedora:35
- .fedora.packages
- .fdo.qemu-build@fedora
stage: prep
tags:
- kvm
variables:
GIT_STRATEGY: none
FDO_DISTRIBUTION_TAG: qemu-2023-01-04.5
allow_failure: true
# Always rebuilds the container
.fedora:35@qemu-forced-rebuild:
extends:
- .fedora:35@qemu-prep
variables:
FDO_FORCE_REBUILD: 1
only:
- schedules
# This is the actual job
fedora:35@qemu-prep:
extends: .fedora:35@qemu-prep
fedora:35@qemu-forced-rebuild:
extends: .fedora:35@qemu-forced-rebuild
# Pulls in the container from upstream or rebuilds it if missing
fedora:34@container-prep:
@ -748,59 +718,46 @@ soname:
exit 1 ;
fi
.qemu@fedora:35:
# build on the host, then run a systemd service to execute the test suite
# inside the qemu VM handled by b2c
.build-in-b2c@template:
extends:
- .fedora:35
stage: VM
image: $CI_REGISTRY_IMAGE/$FDO_DISTRIBUTION_NAME/$FDO_DISTRIBUTION_VERSION:qemu-$FDO_DISTRIBUTION_TAG
- .default_artifacts
tags:
- kvm
variables:
MESON_BUILDDIR: build_dir
B2C_KERNEL: https://gitlab.freedesktop.org/api/v4/projects/libevdev%2Fhid-tools/packages/generic/kernel-x86_64/v5.19/bzImage
B2C_VERSION: v0.9.8
B2C_IMAGE: $FDO_DISTRIBUTION_IMAGE
B2C_COMMAND: .gitlab-ci/start-in-systemd.sh
script:
# start our vm, no args required
- /app/vmctl start
# first build in the host container
- .gitlab-ci/meson-build.sh --skip-test
- *check_tainted
# pull b2c
- curl -L -o /app/boot2container https://gitlab.freedesktop.org/mupuf/boot2container/-/raw/v0.9.8/vm2c.py
- chmod +x /app/boot2container
- "scp -r $PWD vm:"
- echo "CI_JOB_ID=\"$CI_JOB_ID\"" > sshenv
- echo "CI_JOB_NAME=\"$CI_JOB_NAME\"" >> sshenv
- echo "MESON_BUILDDIR=\"$MESON_BUILDDIR\"" >> sshenv
- echo "MESON_TEST_ARGS=\"$MESON_TEST_ARGS\"" >> sshenv
- echo "NINJA_ARGS=\"$NINJA_ARGS\"" >> sshenv
- "scp sshenv vm:~/$CI_PROJECT_NAME/.meson_environment"
- /app/vmctl exec "cd $CI_PROJECT_NAME ; .gitlab-ci/meson-build.sh" && touch .success || true
# no matter the results of the tests, we want to fetch the logs
- scp -r vm:$CI_PROJECT_NAME/"$MESON_BUILDDIR" .
- *check_tainted
- /app/vmctl stop
- if [[ ! -e .success ]] ;
then
exit 1 ;
fi
artifacts:
name: "qemu-meson-logs-$CI_JOB_NAME"
when: always
expire_in: 1 week
paths:
- $MESON_BUILDDIR/meson-logs
- console.out
reports:
junit: $MESON_BUILDDIR/junit-*.xml
needs: ['fedora:35@qemu-prep']
# runs the test suite only
- /app/boot2container
qemu:meson:
extends: .qemu@fedora:35
stage: VM
extends:
- .fdo.distribution-image@fedora
- .fedora:35
- .build-in-b2c@template
needs:
- "fedora:35@container-prep"
qemu:meson:valgrind:
extends: .qemu@fedora:35
extends:
- qemu:meson
variables:
MESON_TEST_ARGS: '--setup=valgrind'
meson-from-tarball:
extends:
- .fedora:35

View file

@ -148,39 +148,6 @@ check-merge-request:
{% endfor %}
{% for distro in distributions if distro.want_qemu %}
{% set version = "{}".format(distro.versions|last()) %}
# Pulls in the qemu container from upstream or rebuilds it if missing
.{{ distro.name }}:{{ version }}@qemu-prep:
extends:
- .{{ distro.name }}:{{ version }}
- .{{ distro.name}}.packages
- .fdo.qemu-build@{{distro.name}}
stage: prep
tags:
- kvm
variables:
GIT_STRATEGY: none
FDO_DISTRIBUTION_TAG: qemu-{{ distro.tag }}
allow_failure: true
# Always rebuilds the container
.{{ distro.name }}:{{ version }}@qemu-forced-rebuild:
extends:
- .{{ distro.name }}:{{ version }}@qemu-prep
variables:
FDO_FORCE_REBUILD: 1
only:
- schedules
# This is the actual job
{{distro.name}}:{{version}}@qemu-prep:
extends: .{{distro.name}}:{{version}}@qemu-prep
{{distro.name}}:{{version}}@qemu-forced-rebuild:
extends: .{{distro.name}}:{{version}}@qemu-forced-rebuild
{% endfor %}
{% for distro in distributions %}
{% for version in distro.versions %}
@ -433,60 +400,47 @@ soname:
exit 1 ;
fi
{% for distro in distributions if distro.want_qemu %}
{% set version = "{}".format(distro.versions|last()) %}
.qemu@{{distro.name}}:{{version}}:
# build on the host, then run a systemd service to execute the test suite
# inside the qemu VM handled by b2c
.build-in-b2c@template:
extends:
- .{{distro.name}}:{{version}}
stage: VM
image: $CI_REGISTRY_IMAGE/$FDO_DISTRIBUTION_NAME/$FDO_DISTRIBUTION_VERSION:qemu-$FDO_DISTRIBUTION_TAG
- .default_artifacts
tags:
- kvm
variables:
MESON_BUILDDIR: build_dir
B2C_KERNEL: {{ b2c.kernel }}
B2C_VERSION: {{ b2c.version }}
B2C_IMAGE: $FDO_DISTRIBUTION_IMAGE
B2C_COMMAND: .gitlab-ci/start-in-systemd.sh
script:
# start our vm, no args required
- /app/vmctl start
# first build in the host container
- .gitlab-ci/meson-build.sh --skip-test
- *check_tainted
# pull b2c
- curl -L -o /app/boot2container https://gitlab.freedesktop.org/mupuf/boot2container/-/raw/{{b2c.version}}/vm2c.py
- chmod +x /app/boot2container
- "scp -r $PWD vm:"
- echo "CI_JOB_ID=\"$CI_JOB_ID\"" > sshenv
- echo "CI_JOB_NAME=\"$CI_JOB_NAME\"" >> sshenv
- echo "MESON_BUILDDIR=\"$MESON_BUILDDIR\"" >> sshenv
- echo "MESON_TEST_ARGS=\"$MESON_TEST_ARGS\"" >> sshenv
- echo "NINJA_ARGS=\"$NINJA_ARGS\"" >> sshenv
- "scp sshenv vm:~/$CI_PROJECT_NAME/.meson_environment"
- /app/vmctl exec "cd $CI_PROJECT_NAME ; .gitlab-ci/meson-build.sh" && touch .success || true
# no matter the results of the tests, we want to fetch the logs
- scp -r vm:$CI_PROJECT_NAME/"$MESON_BUILDDIR" .
- *check_tainted
- /app/vmctl stop
- if [[ ! -e .success ]] ;
then
exit 1 ;
fi
artifacts:
name: "qemu-meson-logs-$CI_JOB_NAME"
when: always
expire_in: 1 week
paths:
- $MESON_BUILDDIR/meson-logs
- console.out
reports:
junit: $MESON_BUILDDIR/junit-*.xml
needs: ['{{distro.name}}:{{version}}@qemu-prep']
# runs the test suite only
- /app/boot2container
{% for distro in distributions if distro.use_for_qemu_tests %}
{% set version = "{}".format(distro.versions|last()) %}
qemu:meson:
extends: .qemu@{{distro.name}}:{{version}}
stage: VM
extends:
- .fdo.distribution-image@{{distro.name}}
- .{{distro.name}}:{{version}}
- .build-in-b2c@template
needs:
- "{{distro.name}}:{{version}}@container-prep"
qemu:meson:valgrind:
extends: .qemu@{{distro.name}}:{{version}}
extends:
- qemu:meson
variables:
MESON_TEST_ARGS: '--setup=valgrind'
{% endfor %}
{% for distro in distributions if distro.use_for_tarball_tests %}

View file

@ -3,13 +3,14 @@
#
# We're happy to rebuild all containers when one changes.
.default_tag: &default_tag '2023-01-04.5'
.default_tag: &default_tag '2023-01-05.0'
distributions:
- name: fedora
tag: *default_tag
want_qemu: true
use_for_tarball_tests: true
# only one distro for qemu tests
use_for_qemu_tests: true
versions:
- '34'
- '35'
@ -30,6 +31,14 @@ distributions:
- doxygen
- xz
- clang-analyzer
# below packages are for the qemu runs, so optional
- systemd-udev
- qemu-img
- qemu-system-x86-core
- qemu-system-aarch64-core
- jq
- python3-click
- python3-rich
- name: ubuntu
tag: *default_tag
versions:
@ -111,3 +120,7 @@ distributions:
- doxygen
- xz
- linux-headers
b2c:
version: v0.9.8
kernel: https://gitlab.freedesktop.org/api/v4/projects/libevdev%2Fhid-tools/packages/generic/kernel-x86_64/v5.19/bzImage

69
.gitlab-ci/start-in-systemd.sh Executable file
View file

@ -0,0 +1,69 @@
#!/usr/bin/env bash
set -x
systemd_target=basic.target
post_command="/usr/bin/systemctl exit \$EXIT_STATUS"
while [[ $# -gt 0 ]]; do
case $1 in
--debug-mode)
shift
systemd_target=multi-user.target
post_command="echo you can now log in as root (no password) and then turn off by running \'/usr/bin/systemctl exit \$EXIT_STATUS\'"
;;
*)
echo "Unknow commandline argument $1"
exit 1
;;
esac
done
WORKDIR=${FDO_DISTRIBUTION_WORKINGDIR:-$PWD}
B2C_WORKDIR=${FDO_B2C_WORKDIR:-/app}
# remove root password for debugging
sed -i 's/root:!locked::/root:::/' /etc/shadow
# create a libevdev test suite service
cat <<EOF > /etc/systemd/system/libevdev-testsuite.service
[Unit]
Description=libevdev test suite
After=$systemd_target
[Service]
Type=simple
StandardOutput=journal+console
EnvironmentFile=$B2C_WORKDIR/.b2c_env
WorkingDirectory=$WORKDIR
ExecStart=$WORKDIR/.gitlab-ci/meson-build.sh --skip-setup --skip-build --run-test
# exit the container on termination
ExecStopPost=$post_command
[Install]
WantedBy=default.target
EOF
cat /etc/systemd/system/libevdev-testsuite.service
# enable the service
systemctl enable libevdev-testsuite.service
# disable some services we don't need in the CI
systemctl mask network-online.target
systemctl mask network-pre.target
systemctl mask timers.target
systemctl mask dnf-makecache.timer
systemctl mask systemd-logind.service
systemctl mask rpmdb-migrate.service
systemctl mask systemd-network-generator.service
systemctl mask cryptsetup-pre.target
systemctl mask cryptsetup.target
#change default target
systemctl set-default $systemd_target
# start the system
exec /usr/sbin/init