ci: Use pre-built Docker images for CI builds

Rather than updating the packages on a generic Debian Unstable image
every time a CI build happens, pre-build the image and pre-download all
the dependencies.

This should speed the CI runs up; they currently take about 4 minutes.

Signed-off-by: Philip Withnall <withnall@endlessm.com>
This commit is contained in:
Philip Withnall 2020-07-21 11:51:29 +01:00
parent ab75b35d91
commit e6f82a4a86
9 changed files with 432 additions and 21 deletions

View File

@ -1,14 +1,3 @@
image: debian:unstable
before_script:
- apt update
- apt install -y meson pkg-config gtk-doc-tools libxml2-utils
libglib2.0-dev libgirepository1.0-dev libpam0g-dev
gettext policykit-1 libpolkit-gobject-1-dev git
lcov libgtk-3-dev libaccountsservice-dev libflatpak-dev
libglib-testing-0-dev libappstream-glib-dev
- export LANG=C.UTF-8
stages: stages:
- build - build
- deploy - deploy
@ -17,22 +6,53 @@ cache:
paths: paths:
- _ccache/ - _ccache/
debian: variables:
stage: build DEBIAN_IMAGE: "registry.freedesktop.org/pwithnall/malcontent/debian-unstable:v1"
MESON_TEST_TIMEOUT_MULTIPLIER: 2
G_MESSAGES_DEBUG: all
MESON_COMMON_OPTIONS: "--buildtype debug --wrap-mode=nodownload"
.only-default:
only:
- branches
except: except:
- tags - tags
.build:
extends: .only-default
before_script:
- cp -r $HOME/subprojects/* subprojects/
debian:
extends: .build
image: $DEBIAN_IMAGE
stage: build
variables:
CFLAGS: "-coverage -ftest-coverage -fprofile-arcs"
script: script:
- meson --buildtype debug --werror -Db_coverage=true -Ddocumentation=true -Dprivileged_group=sudo _build . - meson ${MESON_COMMON_OPTIONS}
- meson test -C _build --werror
# FIXME: lcov doesn't support gcc9 yet: -Db_coverage=true
# https://github.com/linux-test-project/lcov/issues/58 -Dinstalled_tests=true
- ninja -C _build coverage || true -Dprivileged_group=sudo
_build
- ninja -C _build
- mkdir -p _coverage
- lcov --config-file .gitlab-ci/lcovrc --directory _build --capture --initial --output-file "_coverage/${CI_JOB_NAME}-baseline.lcov"
- .gitlab-ci/run-tests.sh
- lcov --config-file .gitlab-ci/lcovrc --directory _build --capture --output-file "_coverage/${CI_JOB_NAME}.lcov"
- bash -x ./.gitlab-ci/coverage-docker.sh
coverage: '/^\s+lines\.+:\s+([\d.]+\%)\s+/' coverage: '/^\s+lines\.+:\s+([\d.]+\%)\s+/'
artifacts: artifacts:
when: always reports:
junit: "_build/${CI_JOB_NAME}-report.xml"
name: "malcontent-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}" name: "malcontent-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
when: always
paths: paths:
- "_build/config.h"
- "_build/meson-logs" - "_build/meson-logs"
- "_build/${CI_JOB_NAME}-report.xml"
- "_coverage"
# FIXME: Run gtkdoc-check when we can. See: # FIXME: Run gtkdoc-check when we can. See:
# https://github.com/mesonbuild/meson/issues/3580 # https://github.com/mesonbuild/meson/issues/3580
@ -42,8 +62,7 @@ pages:
only: only:
- master - master
script: script:
- mkdir -p public/ - mv _coverage/ public/
- mv _build/meson-logs/coveragereport/ public/coverage/
artifacts: artifacts:
paths: paths:
- public - public

23
.gitlab-ci/README.md Normal file
View File

@ -0,0 +1,23 @@
# CI support stuff
## Docker image
GitLab CI jobs run in a Docker image, defined here. To update that image
(perhaps to install some more packages):
1. Edit `.gitlab-ci/Dockerfile` with the changes you want
1. Run `.gitlab-ci/run-docker.sh build --base=debian-unstable --base-version=1` to
build the new image (bump the version from the latest listed for that `base`
on https://gitlab.freedesktop.org/pwithnall/malcontent/container_registry)
1. Run `.gitlab-ci/run-docker.sh push --base=debian-unstable --base-version=1` to
upload the new image to the GNOME GitLab Docker registry
* If this is the first time you're doing this, you'll need to log into the
registry
* If you use 2-factor authentication on your GNOME GitLab account, you'll
need to [create a personal access token][pat] and use that rather than
your normal password — the token should have `read_registry` and
`write_registry` permissions
1. Edit `.gitlab-ci.yml` (in the root of this repository) to use your new
image
[pat]: https://gitlab.freedesktop.org/profile/personal_access_tokens

View File

@ -0,0 +1,9 @@
#!/bin/bash
set -e
git clone https://gitlab.freedesktop.org/pwithnall/malcontent.git
meson subprojects download --sourcedir malcontent
rm malcontent/subprojects/*.wrap
mv malcontent/subprojects/ .
rm -rf malcontent

29
.gitlab-ci/coverage-docker.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
set -e
for path in _coverage/*.lcov; do
# Remove coverage from generated code in the build directory
lcov --config-file .gitlab-ci/lcovrc -r "${path}" '*/_build/*' -o "$(pwd)/${path}"
# Remove any coverage from system files
lcov --config-file .gitlab-ci/lcovrc -e "${path}" "$(pwd)/*" -o "$(pwd)/${path}"
done
genhtml \
--ignore-errors=source \
--config-file .gitlab-ci/lcovrc \
_coverage/*.lcov \
-o _coverage/coverage
cd _coverage
rm -f ./*.lcov
cat >index.html <<EOL
<html>
<body>
<ul>
<li><a href="coverage/index.html">Coverage</a></li>
</ul>
</body>
</html>
EOL

View File

@ -0,0 +1,42 @@
FROM debian:unstable
RUN apt-get update -qq && apt-get install --no-install-recommends -qq -y \
gettext \
git \
gtk-doc-tools \
lcov \
libaccountsservice-dev \
libappstream-glib-dev \
libflatpak-dev \
libgirepository1.0-dev \
libglib2.0-dev \
libglib-testing-0-dev \
libgtk-3-dev \
libpam0g-dev \
libpolkit-gobject-1-dev \
libxml2-utils \
locales \
meson \
pkg-config \
policykit-1 \
python3-pip \
&& rm -rf /usr/share/doc/* /usr/share/man/*
# Locale for our build
RUN locale-gen C.UTF-8 && /usr/sbin/update-locale LANG=C.UTF-8
ENV LANG=C.UTF-8 LANGUAGE=C.UTF-8 LC_ALL=C.UTF-8
RUN pip3 install meson==0.54.3
ARG HOST_USER_ID=5555
ENV HOST_USER_ID ${HOST_USER_ID}
RUN useradd -u $HOST_USER_ID -ms /bin/bash user
USER user
WORKDIR /home/user
COPY cache-subprojects.sh .
RUN ./cache-subprojects.sh
ENV LANG=C.UTF-8 LANGUAGE=C.UTF-8 LC_ALL=C.UTF-8

13
.gitlab-ci/lcovrc Normal file
View File

@ -0,0 +1,13 @@
# lcov and genhtml configuration
# See http://ltp.sourceforge.net/coverage/lcov/lcovrc.5.php
# Always enable branch coverage
lcov_branch_coverage = 1
# Exclude precondition assertions, as we can never reasonably get full branch
# coverage of them, as they should never normally fail.
# See https://github.com/linux-test-project/lcov/issues/44
lcov_excl_br_line = LCOV_EXCL_BR_LINE|g_return_if_fail|g_return_val_if_fail|g_assert|g_assert_
# Similarly for unreachable assertions.
lcov_excl_line = LCOV_EXCL_LINE|g_return_if_reached|g_return_val_if_reached|g_assert_not_reached

115
.gitlab-ci/meson-junit-report.py Executable file
View File

@ -0,0 +1,115 @@
#!/usr/bin/env python3
# Turns a Meson testlog.json file into a JUnit XML report
#
# Copyright 2019 GNOME Foundation
#
# SPDX-License-Identifier: LGPL-2.1-or-later
#
# Original author: Emmanuele Bassi
import argparse
import datetime
import json
import os
import sys
import xml.etree.ElementTree as ET
aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report')
aparser.add_argument('--project-name', metavar='NAME',
help='The project name',
default='unknown')
aparser.add_argument('--job-id', metavar='ID',
help='The job ID for the report',
default='Unknown')
aparser.add_argument('--branch', metavar='NAME',
help='Branch of the project being tested',
default='master')
aparser.add_argument('--output', metavar='FILE',
help='The output file, stdout by default',
type=argparse.FileType('w', encoding='UTF-8'),
default=sys.stdout)
aparser.add_argument('infile', metavar='FILE',
help='The input testlog.json, stdin by default',
type=argparse.FileType('r', encoding='UTF-8'),
default=sys.stdin)
args = aparser.parse_args()
outfile = args.output
testsuites = ET.Element('testsuites')
testsuites.set('id', '{}/{}'.format(args.job_id, args.branch))
testsuites.set('package', args.project_name)
testsuites.set('timestamp', datetime.datetime.utcnow().isoformat())
suites = {}
for line in args.infile:
data = json.loads(line)
(full_suite, unit_name) = data['name'].split(' / ')
try:
(project_name, suite_name) = full_suite.split(':')
except ValueError:
project_name = full_suite
suite_name = full_suite
duration = data['duration']
return_code = data['returncode']
log = data['stdout']
log_stderr = data.get('stderr', '')
unit = {
'suite': suite_name,
'name': unit_name,
'duration': duration,
'returncode': return_code,
'stdout': log,
'stderr': log_stderr,
}
units = suites.setdefault(suite_name, [])
units.append(unit)
for name, units in suites.items():
print('Processing suite {} (units: {})'.format(name, len(units)))
def if_failed(unit):
if unit['returncode'] != 0:
return True
return False
def if_succeded(unit):
if unit['returncode'] == 0:
return True
return False
successes = list(filter(if_succeded, units))
failures = list(filter(if_failed, units))
print(' - {}: {} pass, {} fail'.format(name, len(successes), len(failures)))
testsuite = ET.SubElement(testsuites, 'testsuite')
testsuite.set('name', '{}/{}'.format(args.project_name, name))
testsuite.set('tests', str(len(units)))
testsuite.set('errors', str(len(failures)))
testsuite.set('failures', str(len(failures)))
for unit in successes:
testcase = ET.SubElement(testsuite, 'testcase')
testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
testcase.set('name', unit['name'])
testcase.set('time', str(unit['duration']))
for unit in failures:
testcase = ET.SubElement(testsuite, 'testcase')
testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
testcase.set('name', unit['name'])
testcase.set('time', str(unit['duration']))
failure = ET.SubElement(testcase, 'failure')
failure.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
failure.set('name', unit['name'])
failure.set('type', 'error')
failure.text = unit['stdout'] + '\n' + unit['stderr']
output = ET.tostring(testsuites, encoding='unicode')
outfile.write(output)

132
.gitlab-ci/run-docker.sh Executable file
View File

@ -0,0 +1,132 @@
#!/bin/bash
read_arg() {
# $1 = arg name
# $2 = arg value
# $3 = arg parameter
local rematch='^[^=]*=(.*)$'
if [[ $2 =~ $rematch ]]; then
read -r "$1" <<< "${BASH_REMATCH[1]}"
else
read -r "$1" <<< "$3"
# There is no way to shift our callers args, so
# return 1 to indicate they should do it instead.
return 1
fi
}
SUDO_CMD="sudo"
if docker -v |& grep -q podman; then
# Using podman
SUDO_CMD=""
# Docker is actually implemented by podman, and its OCI output
# is incompatible with some of the dockerd instances on GitLab
# CI runners.
export BUILDAH_FORMAT=docker
fi
set -e
base=""
base_version=""
build=0
run=0
push=0
list=0
print_help=0
no_login=0
while (($# > 0)); do
case "${1%%=*}" in
build) build=1;;
run) run=1;;
push) push=1;;
list) list=1;;
help) print_help=1;;
--base|-b) read_arg base "$@" || shift;;
--base-version) read_arg base_version "$@" || shift;;
--no-login) no_login=1;;
*) echo -e "\e[1;31mERROR\e[0m: Unknown option '$1'"; exit 1;;
esac
shift
done
if [ $print_help == 1 ]; then
echo "$0 - Build and run Docker images"
echo ""
echo "Usage: $0 <command> [options] [basename]"
echo ""
echo "Available commands"
echo ""
echo " build --base=<BASENAME> - Build Docker image <BASENAME>.Dockerfile"
echo " run --base=<BASENAME> - Run Docker image <BASENAME>"
echo " push --base=<BASENAME> - Push Docker image <BASENAME> to the registry"
echo " list - List available images"
echo " help - This help message"
echo ""
exit 0
fi
cd "$(dirname "$0")"
if [ $list == 1 ]; then
echo "Available Docker images:"
for f in *.Dockerfile; do
filename=$( basename -- "$f" )
basename="${filename%.*}"
echo -e " \e[1;39m$basename\e[0m"
done
exit 0
fi
# All commands after this require --base to be set
if [ -z "${base}" ]; then
echo "Usage: $0 <command>"
exit 1
fi
if [ ! -f "$base.Dockerfile" ]; then
echo -e "\e[1;31mERROR\e[0m: Dockerfile for '$base' not found"
exit 1
fi
if [ -z "${base_version}" ]; then
base_version="latest"
else
base_version="v$base_version"
fi
TAG="registry.freedesktop.org/pwithnall/malcontent/${base}:${base_version}"
if [ $build == 1 ]; then
echo -e "\e[1;32mBUILDING\e[0m: ${base} as ${TAG}"
$SUDO_CMD docker build \
--build-arg HOST_USER_ID="$UID" \
--tag "${TAG}" \
--file "${base}.Dockerfile" .
exit $?
fi
if [ $push == 1 ]; then
echo -e "\e[1;32mPUSHING\e[0m: ${base} as ${TAG}"
if [ $no_login == 0 ]; then
$SUDO_CMD docker login registry.freedesktop.org
fi
$SUDO_CMD docker push $TAG
exit $?
fi
if [ $run == 1 ]; then
echo -e "\e[1;32mRUNNING\e[0m: ${base} as ${TAG}"
$SUDO_CMD docker run \
--rm \
--volume "$(pwd)/..:/home/user/app" \
--workdir "/home/user/app" \
--tty \
--interactive "${TAG}" \
bash
exit $?
fi

29
.gitlab-ci/run-tests.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
set +e
case "$1" in
--log-file)
log_file="$2"
shift
shift
;;
*)
log_file="_build/meson-logs/testlog.json"
esac
meson test \
-C _build \
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
--no-suite flaky \
"$@"
exit_code=$?
python3 .gitlab-ci/meson-junit-report.py \
--project-name=malcontent \
--job-id "${CI_JOB_NAME}" \
--output "_build/${CI_JOB_NAME}-report.xml" \
"${log_file}"
exit $exit_code