From 1e3d82d85e9a59ccabb329c8f2401207edd419be Mon Sep 17 00:00:00 2001 From: Ralf Habacker Date: Fri, 6 Nov 2020 09:45:40 +0100 Subject: [PATCH 1/5] Move installing packages into tools/ci-install.sh In tools/ci-build.sh the cross compile setup has to be moved further up to match the correct subdirectory. --- tools/ci-build.sh | 33 +++++++++------------------------ tools/ci-install.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/tools/ci-build.sh b/tools/ci-build.sh index 94caa370..aea26710 100755 --- a/tools/ci-build.sh +++ b/tools/ci-build.sh @@ -165,46 +165,31 @@ case "$ci_buildsys" in ;; esac -srcdir="$(pwd)" -mkdir ci-build-${ci_variant}-${ci_host} -cd ci-build-${ci_variant}-${ci_host} - -make="make -j${ci_parallel} V=1 VERBOSE=1" - +# +# cross compile setup +# case "$ci_host" in (*-w64-mingw32) - mirror=http://repo.msys2.org/mingw/${ci_host%%-*} if [ "${ci_host%%-*}" = i686 ]; then mingw="$(pwd)/mingw32" else mingw="$(pwd)/mingw64" fi - install -d "${mingw}" export PKG_CONFIG_LIBDIR="${mingw}/lib/pkgconfig" export PKG_CONFIG_PATH= export PKG_CONFIG="pkg-config --define-variable=prefix=${mingw}" unset CC unset CXX - for pkg in \ - bzip2-1.0.8-1 \ - expat-2.2.9-1 \ - gcc-libs-9.3.0-2 \ - gettext-0.19.8.1-8 \ - glib2-2.64.2-1 \ - iconv-1.16-1 \ - libffi-3.3-1 \ - libiconv-1.16-1 \ - libwinpthread-git-8.0.0.5814.9dbf4cc1-1 \ - pcre-8.44-1 \ - zlib-1.2.11-7 \ - ; do - wget ${mirror}/mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz - tar -xvf mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz - done export TMPDIR=/tmp ;; esac +srcdir="$(pwd)" +mkdir ci-build-${ci_variant}-${ci_host} +cd ci-build-${ci_variant}-${ci_host} + +make="make -j${ci_parallel} V=1 VERBOSE=1" + case "$ci_buildsys" in (autotools) case "$ci_variant" in diff --git a/tools/ci-install.sh b/tools/ci-install.sh index 10a0e1d3..0badcbf3 100755 --- a/tools/ci-install.sh +++ b/tools/ci-install.sh @@ -122,6 +122,33 @@ case "$ci_distro" in ${NULL} ;; esac + case "$ci_host" in + (*-w64-mingw32) + mirror=http://repo.msys2.org/mingw/${ci_host%%-*} + if [ "${ci_host%%-*}" = i686 ]; then + mingw="$(pwd)/mingw32" + else + mingw="$(pwd)/mingw64" + fi + install -d "${mingw}" + for pkg in \ + bzip2-1.0.8-1 \ + expat-2.2.9-1 \ + gcc-libs-9.3.0-2 \ + gettext-0.19.8.1-8 \ + glib2-2.64.2-1 \ + iconv-1.16-1 \ + libffi-3.3-1 \ + libiconv-1.16-1 \ + libwinpthread-git-8.0.0.5814.9dbf4cc1-1 \ + pcre-8.44-1 \ + zlib-1.2.11-7 \ + ; do + wget ${mirror}/mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz + tar -xvf mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz + done + ;; + esac if [ "$ci_host/$ci_variant/$ci_suite" = "native/production/buster" ]; then $sudo apt-get -qq -y --no-install-recommends install \ From 478484f13c9f291d1c8bc1b3e3672f382d4d03d9 Mon Sep 17 00:00:00 2001 From: Ralf Habacker Date: Fri, 6 Nov 2020 12:08:12 +0100 Subject: [PATCH 2/5] tools/ci-install.sh: merge multiple calls to apt-get This required a reorganization of the steps that had been carried out. The new order is: 1. install packages with apt-get 2. create user for build if required 3. fetch and unpack tar balls 4. create messagebus user --- tools/ci-install.sh | 176 ++++++++++++++++++++++++-------------------- 1 file changed, 98 insertions(+), 78 deletions(-) diff --git a/tools/ci-install.sh b/tools/ci-install.sh index 0badcbf3..a1f8a86d 100755 --- a/tools/ci-install.sh +++ b/tools/ci-install.sh @@ -105,23 +105,105 @@ case "$ci_distro" in esac $sudo apt-get -qq -y update + packages=() case "$ci_host" in (i686-w64-mingw32) - $sudo apt-get -qq -y --no-install-recommends install \ - binutils-mingw-w64-i686 \ - g++-mingw-w64-i686 \ - $wine32 wine \ - ${NULL} + packages=( + "${packages[@]}" + binutils-mingw-w64-i686 + g++-mingw-w64-i686 + $wine32 wine + ) ;; (x86_64-w64-mingw32) - $sudo apt-get -qq -y --no-install-recommends install \ - binutils-mingw-w64-x86-64\ - g++-mingw-w64-x86-64 \ - $wine64 wine \ - ${NULL} + packages=( + "${packages[@]}" + binutils-mingw-w64-x86-64 + g++-mingw-w64-x86-64 + $wine64 wine + ) ;; esac + + if [ "$ci_host/$ci_variant/$ci_suite" = "native/production/buster" ]; then + packages=( + "${packages[@]}" + qttools5-dev-tools + qt5-default + ) + fi + + packages=( + "${packages[@]}" + adduser + autoconf-archive + automake + autotools-dev + ccache + cmake + debhelper + dh-autoreconf + dh-exec + docbook-xml + docbook-xsl + doxygen + dpkg-dev + g++ + gcc + gnome-desktop-testing + libapparmor-dev + libaudit-dev + libcap-ng-dev + libexpat-dev + libglib2.0-dev + libselinux1-dev + libsystemd-dev + libx11-dev + sudo + valgrind + wget + xauth + xmlto + xsltproc + xvfb + ) + + case "$ci_suite" in + (stretch) + # Debian 9 'stretch' didn't have the ducktype package + ;; + + (*) + # assume Ubuntu 18.04 'bionic', Debian 10 'buster' or newer + packages=( + "${packages[@]}" + ducktype yelp-tools + ) + ;; + esac + + $sudo apt-get -qq -y --no-install-recommends install "${packages[@]}" + + if [ "$ci_in_docker" = yes ]; then + # Add the user that we will use to do the build inside the + # Docker container, and let them use sudo + adduser --disabled-password --gecos "" user + echo "user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/nopasswd + chmod 0440 /etc/sudoers.d/nopasswd + fi + + # manual package setup + case "$ci_suite" in + (jessie|xenial) + # autoconf-archive in Debian 8 and Ubuntu 16.04 is too old, + # use the one from Debian 9 instead + wget http://deb.debian.org/debian/pool/main/a/autoconf-archive/autoconf-archive_20160916-1_all.deb + $sudo dpkg -i autoconf-archive_*_all.deb + rm autoconf-archive_*_all.deb + ;; + esac + case "$ci_host" in (*-w64-mingw32) mirror=http://repo.msys2.org/mingw/${ci_host%%-*} @@ -143,84 +225,22 @@ case "$ci_distro" in libwinpthread-git-8.0.0.5814.9dbf4cc1-1 \ pcre-8.44-1 \ zlib-1.2.11-7 \ - ; do + ; do wget ${mirror}/mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz tar -xvf mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz done + + # limit access rights + if [ "$ci_in_docker" = yes ]; then + chown -R user "${mingw}" + fi ;; esac - if [ "$ci_host/$ci_variant/$ci_suite" = "native/production/buster" ]; then - $sudo apt-get -qq -y --no-install-recommends install \ - qttools5-dev-tools qt5-default \ - ${NULL} - fi - - $sudo apt-get -qq -y --no-install-recommends install \ - adduser \ - autoconf-archive \ - automake \ - autotools-dev \ - ccache \ - cmake \ - debhelper \ - dh-autoreconf \ - dh-exec \ - docbook-xml \ - docbook-xsl \ - doxygen \ - dpkg-dev \ - g++ \ - gcc \ - gnome-desktop-testing \ - libapparmor-dev \ - libaudit-dev \ - libcap-ng-dev \ - libexpat-dev \ - libglib2.0-dev \ - libselinux1-dev \ - libsystemd-dev \ - libx11-dev \ - sudo \ - valgrind \ - wget \ - xauth \ - xmlto \ - xsltproc \ - xvfb \ - ${NULL} - # Make sure we have a messagebus user, even if the dbus package # isn't installed $sudo adduser --system --quiet --home /nonexistent --no-create-home \ --disabled-password --group messagebus - - if [ "$ci_in_docker" = yes ]; then - # Add the user that we will use to do the build inside the - # Docker container, and let them use sudo - adduser --disabled-password --gecos "" user - echo "user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/nopasswd - chmod 0440 /etc/sudoers.d/nopasswd - fi - - case "$ci_suite" in - (jessie|xenial) - # autoconf-archive in Debian 8 and Ubuntu 16.04 is too old, - # use the one from Debian 9 instead - wget http://deb.debian.org/debian/pool/main/a/autoconf-archive/autoconf-archive_20160916-1_all.deb - $sudo dpkg -i autoconf-archive_*_all.deb - rm autoconf-archive_*_all.deb - ;; - - (stretch) - # Debian 9 'stretch' didn't have the ducktype package - ;; - - (*) - # assume Ubuntu 18.04 'bionic', Debian 10 'buster' or newer - $sudo apt-get -qq -y --no-install-recommends install ducktype yelp-tools - ;; - esac ;; (*) From e82d5fe48623337fd5a69317e27b7a8ce195398a Mon Sep 17 00:00:00 2001 From: Ralf Habacker Date: Mon, 9 Nov 2020 11:06:14 +0100 Subject: [PATCH 3/5] tools/ci-build.sh: Clean up directories from possible previous builds Otherwise, ci-build.sh cannot be executed on a local system if there are previous runs. --- tools/ci-build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/ci-build.sh b/tools/ci-build.sh index aea26710..7fed2e46 100755 --- a/tools/ci-build.sh +++ b/tools/ci-build.sh @@ -154,10 +154,12 @@ NOCONFIGURE=1 ./autogen.sh case "$ci_buildsys" in (cmake-dist) + # clean up directories from possible previous builds + rm -rf ci-build-dist # Do an Autotools `make dist`, then build *that* with CMake, # to assert that our official release tarballs will be enough # to build with CMake. - mkdir ci-build-dist + mkdir -p ci-build-dist ( cd ci-build-dist; ../configure ) make -C ci-build-dist dist tar --xz -xvf ci-build-dist/dbus-1.*.tar.xz @@ -185,7 +187,9 @@ case "$ci_host" in esac srcdir="$(pwd)" -mkdir ci-build-${ci_variant}-${ci_host} +# clean up directories from possible previous builds +rm -rf ci-build-${ci_variant}-${ci_host} +mkdir -p ci-build-${ci_variant}-${ci_host} cd ci-build-${ci_variant}-${ci_host} make="make -j${ci_parallel} V=1 VERBOSE=1" From 9c5734fe9019dd40a5c7491206aa820f430035d6 Mon Sep 17 00:00:00 2001 From: Ralf Habacker Date: Mon, 9 Nov 2020 10:58:22 +0100 Subject: [PATCH 4/5] On CI use common install prefix named 'dep_prefix' for installing local packages With this commit a new variable 'ci_local_packages' has been introduced to have a choice for using development packages from a local installation or from the distribution. --- .gitlab-ci.yml | 1 + tools/ci-build.sh | 39 ++++++++++++++++++-------- tools/ci-install.sh | 68 +++++++++++++++++++++++---------------------- 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63598e89..d0b1a01f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,7 @@ cache: variables: ci_in_docker: "yes" + ci_local_packages: "yes" ci_parallel: "2" ci_sudo: "yes" ci_distro: "debian" diff --git a/tools/ci-build.sh b/tools/ci-build.sh index 7fed2e46..f77f11c2 100755 --- a/tools/ci-build.sh +++ b/tools/ci-build.sh @@ -95,6 +95,11 @@ init_wine() { # See ci-install.sh : "${ci_host:=native}" +# ci_local_packages: +# prefer local packages instead of distribution +# See ci-install.sh +: "${ci_local_packages:=yes}" + # ci_parallel: # A number of parallel jobs, passed to make -j : "${ci_parallel:=1}" @@ -172,14 +177,24 @@ esac # case "$ci_host" in (*-w64-mingw32) - if [ "${ci_host%%-*}" = i686 ]; then - mingw="$(pwd)/mingw32" + if [ "$ci_local_packages" = yes ]; then + dep_prefix=$(pwd)/${ci_host}-prefix else - mingw="$(pwd)/mingw64" + # assume the compiler was configured with a sysroot (e.g. openSUSE) + sysroot=$("${ci_host}-gcc" --print-sysroot) + # check if the prefix is a subdir of sysroot (e.g. openSUSE) + if [ -d "${sysroot}/${ci_host}" ]; then + dep_prefix="${sysroot}/${ci_host}" + else + # fallback: assume the dependency libraries were built with --prefix=/${ci_host} + dep_prefix="/${ci_host}" + export PKG_CONFIG_SYSROOT_DIR="${sysroot}" + fi fi - export PKG_CONFIG_LIBDIR="${mingw}/lib/pkgconfig" + + export PKG_CONFIG_LIBDIR="${dep_prefix}/lib/pkgconfig" export PKG_CONFIG_PATH= - export PKG_CONFIG="pkg-config --define-variable=prefix=${mingw}" + export PKG_CONFIG="pkg-config --define-variable=prefix=${dep_prefix}" unset CC unset CXX export TMPDIR=/tmp @@ -359,16 +374,18 @@ case "$ci_buildsys" in if [ "$ci_runtime" = "shared" ]; then libgcc_path=$(dirname "$("${ci_host}-gcc" -print-libgcc-file-name)") fi - init_wine "${mingw}/bin" "$(pwd)/bin" ${libgcc_path:+"$libgcc_path"} + init_wine "${dep_prefix}/bin" "$(pwd)/bin" ${libgcc_path:+"$libgcc_path"} cmdwrapper="xvfb-run -a" fi set _ "$@" set "$@" -D CMAKE_TOOLCHAIN_FILE="${srcdir}/cmake/${ci_host}.cmake" - set "$@" -D CMAKE_PREFIX_PATH="${mingw}" - set "$@" -D CMAKE_INCLUDE_PATH="${mingw}/include" - set "$@" -D CMAKE_LIBRARY_PATH="${mingw}/lib" - set "$@" -D EXPAT_LIBRARY="${mingw}/lib/libexpat.dll.a" - set "$@" -D GLIB2_LIBRARIES="${mingw}/lib/libglib-2.0.dll.a ${mingw}/lib/libgobject-2.0.dll.a ${mingw}/lib/libgio-2.0.dll.a" + set "$@" -D CMAKE_PREFIX_PATH="${dep_prefix}" + if [ "$ci_local_packages" = yes ]; then + set "$@" -D CMAKE_INCLUDE_PATH="${dep_prefix}/include" + set "$@" -D CMAKE_LIBRARY_PATH="${dep_prefix}/lib" + set "$@" -D EXPAT_LIBRARY="${dep_prefix}/lib/libexpat.dll.a" + set "$@" -D GLIB2_LIBRARIES="${dep_prefix}/lib/libglib-2.0.dll.a ${dep_prefix}/lib/libgobject-2.0.dll.a ${dep_prefix}/lib/libgio-2.0.dll.a" + fi if [ "$ci_test" = yes ]; then set "$@" -D DBUS_USE_WINE=1 fi diff --git a/tools/ci-install.sh b/tools/ci-install.sh index a1f8a86d..882b0da0 100755 --- a/tools/ci-install.sh +++ b/tools/ci-install.sh @@ -48,6 +48,10 @@ NULL= # (ci_docker is empty in this case). : "${ci_in_docker:=no}" +# ci_local_packages: +# prefer local packages instead of distribution +: "${ci_local_packages:=yes}" + # ci_suite: # OS suite (release, branch) in which we are testing. # Typical values for ci_distro=debian: sid, jessie @@ -204,39 +208,6 @@ case "$ci_distro" in ;; esac - case "$ci_host" in - (*-w64-mingw32) - mirror=http://repo.msys2.org/mingw/${ci_host%%-*} - if [ "${ci_host%%-*}" = i686 ]; then - mingw="$(pwd)/mingw32" - else - mingw="$(pwd)/mingw64" - fi - install -d "${mingw}" - for pkg in \ - bzip2-1.0.8-1 \ - expat-2.2.9-1 \ - gcc-libs-9.3.0-2 \ - gettext-0.19.8.1-8 \ - glib2-2.64.2-1 \ - iconv-1.16-1 \ - libffi-3.3-1 \ - libiconv-1.16-1 \ - libwinpthread-git-8.0.0.5814.9dbf4cc1-1 \ - pcre-8.44-1 \ - zlib-1.2.11-7 \ - ; do - wget ${mirror}/mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz - tar -xvf mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz - done - - # limit access rights - if [ "$ci_in_docker" = yes ]; then - chown -R user "${mingw}" - fi - ;; - esac - # Make sure we have a messagebus user, even if the dbus package # isn't installed $sudo adduser --system --quiet --home /nonexistent --no-create-home \ @@ -249,4 +220,35 @@ case "$ci_distro" in ;; esac +if [ "$ci_local_packages" = yes ]; then + case "$ci_host" in + (*-w64-mingw32) + mirror=http://repo.msys2.org/mingw/${ci_host%%-*} + dep_prefix=$(pwd)/${ci_host}-prefix + install -d "${dep_prefix}" + for pkg in \ + bzip2-1.0.8-1 \ + expat-2.2.9-1 \ + gcc-libs-9.3.0-2 \ + gettext-0.19.8.1-8 \ + glib2-2.64.2-1 \ + iconv-1.16-1 \ + libffi-3.3-1 \ + libiconv-1.16-1 \ + libwinpthread-git-8.0.0.5814.9dbf4cc1-1 \ + pcre-8.44-1 \ + zlib-1.2.11-7 \ + ; do + wget ${mirror}/mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz + tar -C ${dep_prefix} --strip-components=1 -xvf mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz + done + + # limit access rights + if [ "$ci_in_docker" = yes ]; then + chown -R user "${dep_prefix}" + fi + ;; + esac +fi + # vim:set sw=4 sts=4 et: From 6412ccdf67b8440916a6a0f8b9b1741aa988192f Mon Sep 17 00:00:00 2001 From: Ralf Habacker Date: Tue, 10 Nov 2020 23:35:16 +0100 Subject: [PATCH 5/5] tools/ci-install.sh: Use package=() style for local package list to avoid '\' notation --- tools/ci-install.sh | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tools/ci-install.sh b/tools/ci-install.sh index 882b0da0..b026d4d3 100755 --- a/tools/ci-install.sh +++ b/tools/ci-install.sh @@ -226,19 +226,20 @@ if [ "$ci_local_packages" = yes ]; then mirror=http://repo.msys2.org/mingw/${ci_host%%-*} dep_prefix=$(pwd)/${ci_host}-prefix install -d "${dep_prefix}" - for pkg in \ - bzip2-1.0.8-1 \ - expat-2.2.9-1 \ - gcc-libs-9.3.0-2 \ - gettext-0.19.8.1-8 \ - glib2-2.64.2-1 \ - iconv-1.16-1 \ - libffi-3.3-1 \ - libiconv-1.16-1 \ - libwinpthread-git-8.0.0.5814.9dbf4cc1-1 \ - pcre-8.44-1 \ - zlib-1.2.11-7 \ - ; do + packages=( + bzip2-1.0.8-1 + expat-2.2.9-1 + gcc-libs-9.3.0-2 + gettext-0.19.8.1-8 + glib2-2.64.2-1 + iconv-1.16-1 + libffi-3.3-1 + libiconv-1.16-1 + libwinpthread-git-8.0.0.5814.9dbf4cc1-1 + pcre-8.44-1 + zlib-1.2.11-7 + ) + for pkg in "${packages[@]}" ; do wget ${mirror}/mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz tar -C ${dep_prefix} --strip-components=1 -xvf mingw-w64-${ci_host%%-*}-${pkg}-any.pkg.tar.xz done