ci: Use pre-built Docker images for CI builds
Rather than updating the packages on a generic Debian Unstable image every time a CI build happens, pre-build the image and pre-download all the dependencies. This should speed the CI runs up; they currently take about 4 minutes. Signed-off-by: Philip Withnall <withnall@endlessm.com>
This commit is contained in:
parent
ab75b35d91
commit
e6f82a4a86
|
@ -1,14 +1,3 @@
|
||||||
image: debian:unstable
|
|
||||||
|
|
||||||
before_script:
|
|
||||||
- apt update
|
|
||||||
- apt install -y meson pkg-config gtk-doc-tools libxml2-utils
|
|
||||||
libglib2.0-dev libgirepository1.0-dev libpam0g-dev
|
|
||||||
gettext policykit-1 libpolkit-gobject-1-dev git
|
|
||||||
lcov libgtk-3-dev libaccountsservice-dev libflatpak-dev
|
|
||||||
libglib-testing-0-dev libappstream-glib-dev
|
|
||||||
- export LANG=C.UTF-8
|
|
||||||
|
|
||||||
stages:
|
stages:
|
||||||
- build
|
- build
|
||||||
- deploy
|
- deploy
|
||||||
|
@ -17,22 +6,53 @@ cache:
|
||||||
paths:
|
paths:
|
||||||
- _ccache/
|
- _ccache/
|
||||||
|
|
||||||
debian:
|
variables:
|
||||||
stage: build
|
DEBIAN_IMAGE: "registry.freedesktop.org/pwithnall/malcontent/debian-unstable:v1"
|
||||||
|
MESON_TEST_TIMEOUT_MULTIPLIER: 2
|
||||||
|
G_MESSAGES_DEBUG: all
|
||||||
|
MESON_COMMON_OPTIONS: "--buildtype debug --wrap-mode=nodownload"
|
||||||
|
|
||||||
|
.only-default:
|
||||||
|
only:
|
||||||
|
- branches
|
||||||
except:
|
except:
|
||||||
- tags
|
- tags
|
||||||
|
|
||||||
|
.build:
|
||||||
|
extends: .only-default
|
||||||
|
before_script:
|
||||||
|
- cp -r $HOME/subprojects/* subprojects/
|
||||||
|
|
||||||
|
debian:
|
||||||
|
extends: .build
|
||||||
|
image: $DEBIAN_IMAGE
|
||||||
|
stage: build
|
||||||
|
variables:
|
||||||
|
CFLAGS: "-coverage -ftest-coverage -fprofile-arcs"
|
||||||
script:
|
script:
|
||||||
- meson --buildtype debug --werror -Db_coverage=true -Ddocumentation=true -Dprivileged_group=sudo _build .
|
- meson ${MESON_COMMON_OPTIONS}
|
||||||
- meson test -C _build
|
--werror
|
||||||
# FIXME: lcov doesn't support gcc9 yet:
|
-Db_coverage=true
|
||||||
# https://github.com/linux-test-project/lcov/issues/58
|
-Dinstalled_tests=true
|
||||||
- ninja -C _build coverage || true
|
-Dprivileged_group=sudo
|
||||||
|
_build
|
||||||
|
- ninja -C _build
|
||||||
|
- mkdir -p _coverage
|
||||||
|
- lcov --config-file .gitlab-ci/lcovrc --directory _build --capture --initial --output-file "_coverage/${CI_JOB_NAME}-baseline.lcov"
|
||||||
|
- .gitlab-ci/run-tests.sh
|
||||||
|
- lcov --config-file .gitlab-ci/lcovrc --directory _build --capture --output-file "_coverage/${CI_JOB_NAME}.lcov"
|
||||||
|
- bash -x ./.gitlab-ci/coverage-docker.sh
|
||||||
coverage: '/^\s+lines\.+:\s+([\d.]+\%)\s+/'
|
coverage: '/^\s+lines\.+:\s+([\d.]+\%)\s+/'
|
||||||
artifacts:
|
artifacts:
|
||||||
when: always
|
reports:
|
||||||
|
junit: "_build/${CI_JOB_NAME}-report.xml"
|
||||||
name: "malcontent-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
|
name: "malcontent-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
|
||||||
|
when: always
|
||||||
paths:
|
paths:
|
||||||
|
- "_build/config.h"
|
||||||
- "_build/meson-logs"
|
- "_build/meson-logs"
|
||||||
|
- "_build/${CI_JOB_NAME}-report.xml"
|
||||||
|
- "_coverage"
|
||||||
|
|
||||||
# FIXME: Run gtkdoc-check when we can. See:
|
# FIXME: Run gtkdoc-check when we can. See:
|
||||||
# https://github.com/mesonbuild/meson/issues/3580
|
# https://github.com/mesonbuild/meson/issues/3580
|
||||||
|
@ -42,8 +62,7 @@ pages:
|
||||||
only:
|
only:
|
||||||
- master
|
- master
|
||||||
script:
|
script:
|
||||||
- mkdir -p public/
|
- mv _coverage/ public/
|
||||||
- mv _build/meson-logs/coveragereport/ public/coverage/
|
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- public
|
- public
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
# CI support stuff
|
||||||
|
|
||||||
|
## Docker image
|
||||||
|
|
||||||
|
GitLab CI jobs run in a Docker image, defined here. To update that image
|
||||||
|
(perhaps to install some more packages):
|
||||||
|
|
||||||
|
1. Edit `.gitlab-ci/Dockerfile` with the changes you want
|
||||||
|
1. Run `.gitlab-ci/run-docker.sh build --base=debian-unstable --base-version=1` to
|
||||||
|
build the new image (bump the version from the latest listed for that `base`
|
||||||
|
on https://gitlab.freedesktop.org/pwithnall/malcontent/container_registry)
|
||||||
|
1. Run `.gitlab-ci/run-docker.sh push --base=debian-unstable --base-version=1` to
|
||||||
|
upload the new image to the GNOME GitLab Docker registry
|
||||||
|
* If this is the first time you're doing this, you'll need to log into the
|
||||||
|
registry
|
||||||
|
* If you use 2-factor authentication on your GNOME GitLab account, you'll
|
||||||
|
need to [create a personal access token][pat] and use that rather than
|
||||||
|
your normal password — the token should have `read_registry` and
|
||||||
|
`write_registry` permissions
|
||||||
|
1. Edit `.gitlab-ci.yml` (in the root of this repository) to use your new
|
||||||
|
image
|
||||||
|
|
||||||
|
[pat]: https://gitlab.freedesktop.org/profile/personal_access_tokens
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
git clone https://gitlab.freedesktop.org/pwithnall/malcontent.git
|
||||||
|
meson subprojects download --sourcedir malcontent
|
||||||
|
rm malcontent/subprojects/*.wrap
|
||||||
|
mv malcontent/subprojects/ .
|
||||||
|
rm -rf malcontent
|
|
@ -0,0 +1,29 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
for path in _coverage/*.lcov; do
|
||||||
|
# Remove coverage from generated code in the build directory
|
||||||
|
lcov --config-file .gitlab-ci/lcovrc -r "${path}" '*/_build/*' -o "$(pwd)/${path}"
|
||||||
|
# Remove any coverage from system files
|
||||||
|
lcov --config-file .gitlab-ci/lcovrc -e "${path}" "$(pwd)/*" -o "$(pwd)/${path}"
|
||||||
|
done
|
||||||
|
|
||||||
|
genhtml \
|
||||||
|
--ignore-errors=source \
|
||||||
|
--config-file .gitlab-ci/lcovrc \
|
||||||
|
_coverage/*.lcov \
|
||||||
|
-o _coverage/coverage
|
||||||
|
|
||||||
|
cd _coverage
|
||||||
|
rm -f ./*.lcov
|
||||||
|
|
||||||
|
cat >index.html <<EOL
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<ul>
|
||||||
|
<li><a href="coverage/index.html">Coverage</a></li>
|
||||||
|
</ul>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
EOL
|
|
@ -0,0 +1,42 @@
|
||||||
|
FROM debian:unstable
|
||||||
|
|
||||||
|
RUN apt-get update -qq && apt-get install --no-install-recommends -qq -y \
|
||||||
|
gettext \
|
||||||
|
git \
|
||||||
|
gtk-doc-tools \
|
||||||
|
lcov \
|
||||||
|
libaccountsservice-dev \
|
||||||
|
libappstream-glib-dev \
|
||||||
|
libflatpak-dev \
|
||||||
|
libgirepository1.0-dev \
|
||||||
|
libglib2.0-dev \
|
||||||
|
libglib-testing-0-dev \
|
||||||
|
libgtk-3-dev \
|
||||||
|
libpam0g-dev \
|
||||||
|
libpolkit-gobject-1-dev \
|
||||||
|
libxml2-utils \
|
||||||
|
locales \
|
||||||
|
meson \
|
||||||
|
pkg-config \
|
||||||
|
policykit-1 \
|
||||||
|
python3-pip \
|
||||||
|
&& rm -rf /usr/share/doc/* /usr/share/man/*
|
||||||
|
|
||||||
|
# Locale for our build
|
||||||
|
RUN locale-gen C.UTF-8 && /usr/sbin/update-locale LANG=C.UTF-8
|
||||||
|
|
||||||
|
ENV LANG=C.UTF-8 LANGUAGE=C.UTF-8 LC_ALL=C.UTF-8
|
||||||
|
|
||||||
|
RUN pip3 install meson==0.54.3
|
||||||
|
|
||||||
|
ARG HOST_USER_ID=5555
|
||||||
|
ENV HOST_USER_ID ${HOST_USER_ID}
|
||||||
|
RUN useradd -u $HOST_USER_ID -ms /bin/bash user
|
||||||
|
|
||||||
|
USER user
|
||||||
|
WORKDIR /home/user
|
||||||
|
|
||||||
|
COPY cache-subprojects.sh .
|
||||||
|
RUN ./cache-subprojects.sh
|
||||||
|
|
||||||
|
ENV LANG=C.UTF-8 LANGUAGE=C.UTF-8 LC_ALL=C.UTF-8
|
|
@ -0,0 +1,13 @@
|
||||||
|
# lcov and genhtml configuration
|
||||||
|
# See http://ltp.sourceforge.net/coverage/lcov/lcovrc.5.php
|
||||||
|
|
||||||
|
# Always enable branch coverage
|
||||||
|
lcov_branch_coverage = 1
|
||||||
|
|
||||||
|
# Exclude precondition assertions, as we can never reasonably get full branch
|
||||||
|
# coverage of them, as they should never normally fail.
|
||||||
|
# See https://github.com/linux-test-project/lcov/issues/44
|
||||||
|
lcov_excl_br_line = LCOV_EXCL_BR_LINE|g_return_if_fail|g_return_val_if_fail|g_assert|g_assert_
|
||||||
|
|
||||||
|
# Similarly for unreachable assertions.
|
||||||
|
lcov_excl_line = LCOV_EXCL_LINE|g_return_if_reached|g_return_val_if_reached|g_assert_not_reached
|
|
@ -0,0 +1,115 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# Turns a Meson testlog.json file into a JUnit XML report
|
||||||
|
#
|
||||||
|
# Copyright 2019 GNOME Foundation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: LGPL-2.1-or-later
|
||||||
|
#
|
||||||
|
# Original author: Emmanuele Bassi
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
|
aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report')
|
||||||
|
aparser.add_argument('--project-name', metavar='NAME',
|
||||||
|
help='The project name',
|
||||||
|
default='unknown')
|
||||||
|
aparser.add_argument('--job-id', metavar='ID',
|
||||||
|
help='The job ID for the report',
|
||||||
|
default='Unknown')
|
||||||
|
aparser.add_argument('--branch', metavar='NAME',
|
||||||
|
help='Branch of the project being tested',
|
||||||
|
default='master')
|
||||||
|
aparser.add_argument('--output', metavar='FILE',
|
||||||
|
help='The output file, stdout by default',
|
||||||
|
type=argparse.FileType('w', encoding='UTF-8'),
|
||||||
|
default=sys.stdout)
|
||||||
|
aparser.add_argument('infile', metavar='FILE',
|
||||||
|
help='The input testlog.json, stdin by default',
|
||||||
|
type=argparse.FileType('r', encoding='UTF-8'),
|
||||||
|
default=sys.stdin)
|
||||||
|
|
||||||
|
args = aparser.parse_args()
|
||||||
|
|
||||||
|
outfile = args.output
|
||||||
|
|
||||||
|
testsuites = ET.Element('testsuites')
|
||||||
|
testsuites.set('id', '{}/{}'.format(args.job_id, args.branch))
|
||||||
|
testsuites.set('package', args.project_name)
|
||||||
|
testsuites.set('timestamp', datetime.datetime.utcnow().isoformat())
|
||||||
|
|
||||||
|
suites = {}
|
||||||
|
for line in args.infile:
|
||||||
|
data = json.loads(line)
|
||||||
|
(full_suite, unit_name) = data['name'].split(' / ')
|
||||||
|
try:
|
||||||
|
(project_name, suite_name) = full_suite.split(':')
|
||||||
|
except ValueError:
|
||||||
|
project_name = full_suite
|
||||||
|
suite_name = full_suite
|
||||||
|
|
||||||
|
duration = data['duration']
|
||||||
|
return_code = data['returncode']
|
||||||
|
log = data['stdout']
|
||||||
|
log_stderr = data.get('stderr', '')
|
||||||
|
|
||||||
|
unit = {
|
||||||
|
'suite': suite_name,
|
||||||
|
'name': unit_name,
|
||||||
|
'duration': duration,
|
||||||
|
'returncode': return_code,
|
||||||
|
'stdout': log,
|
||||||
|
'stderr': log_stderr,
|
||||||
|
}
|
||||||
|
|
||||||
|
units = suites.setdefault(suite_name, [])
|
||||||
|
units.append(unit)
|
||||||
|
|
||||||
|
for name, units in suites.items():
|
||||||
|
print('Processing suite {} (units: {})'.format(name, len(units)))
|
||||||
|
|
||||||
|
def if_failed(unit):
|
||||||
|
if unit['returncode'] != 0:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def if_succeded(unit):
|
||||||
|
if unit['returncode'] == 0:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
successes = list(filter(if_succeded, units))
|
||||||
|
failures = list(filter(if_failed, units))
|
||||||
|
print(' - {}: {} pass, {} fail'.format(name, len(successes), len(failures)))
|
||||||
|
|
||||||
|
testsuite = ET.SubElement(testsuites, 'testsuite')
|
||||||
|
testsuite.set('name', '{}/{}'.format(args.project_name, name))
|
||||||
|
testsuite.set('tests', str(len(units)))
|
||||||
|
testsuite.set('errors', str(len(failures)))
|
||||||
|
testsuite.set('failures', str(len(failures)))
|
||||||
|
|
||||||
|
for unit in successes:
|
||||||
|
testcase = ET.SubElement(testsuite, 'testcase')
|
||||||
|
testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
|
||||||
|
testcase.set('name', unit['name'])
|
||||||
|
testcase.set('time', str(unit['duration']))
|
||||||
|
|
||||||
|
for unit in failures:
|
||||||
|
testcase = ET.SubElement(testsuite, 'testcase')
|
||||||
|
testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
|
||||||
|
testcase.set('name', unit['name'])
|
||||||
|
testcase.set('time', str(unit['duration']))
|
||||||
|
|
||||||
|
failure = ET.SubElement(testcase, 'failure')
|
||||||
|
failure.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
|
||||||
|
failure.set('name', unit['name'])
|
||||||
|
failure.set('type', 'error')
|
||||||
|
failure.text = unit['stdout'] + '\n' + unit['stderr']
|
||||||
|
|
||||||
|
output = ET.tostring(testsuites, encoding='unicode')
|
||||||
|
outfile.write(output)
|
|
@ -0,0 +1,132 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
read_arg() {
|
||||||
|
# $1 = arg name
|
||||||
|
# $2 = arg value
|
||||||
|
# $3 = arg parameter
|
||||||
|
local rematch='^[^=]*=(.*)$'
|
||||||
|
if [[ $2 =~ $rematch ]]; then
|
||||||
|
read -r "$1" <<< "${BASH_REMATCH[1]}"
|
||||||
|
else
|
||||||
|
read -r "$1" <<< "$3"
|
||||||
|
# There is no way to shift our callers args, so
|
||||||
|
# return 1 to indicate they should do it instead.
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
SUDO_CMD="sudo"
|
||||||
|
if docker -v |& grep -q podman; then
|
||||||
|
# Using podman
|
||||||
|
SUDO_CMD=""
|
||||||
|
# Docker is actually implemented by podman, and its OCI output
|
||||||
|
# is incompatible with some of the dockerd instances on GitLab
|
||||||
|
# CI runners.
|
||||||
|
export BUILDAH_FORMAT=docker
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
base=""
|
||||||
|
base_version=""
|
||||||
|
build=0
|
||||||
|
run=0
|
||||||
|
push=0
|
||||||
|
list=0
|
||||||
|
print_help=0
|
||||||
|
no_login=0
|
||||||
|
|
||||||
|
while (($# > 0)); do
|
||||||
|
case "${1%%=*}" in
|
||||||
|
build) build=1;;
|
||||||
|
run) run=1;;
|
||||||
|
push) push=1;;
|
||||||
|
list) list=1;;
|
||||||
|
help) print_help=1;;
|
||||||
|
--base|-b) read_arg base "$@" || shift;;
|
||||||
|
--base-version) read_arg base_version "$@" || shift;;
|
||||||
|
--no-login) no_login=1;;
|
||||||
|
*) echo -e "\e[1;31mERROR\e[0m: Unknown option '$1'"; exit 1;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ $print_help == 1 ]; then
|
||||||
|
echo "$0 - Build and run Docker images"
|
||||||
|
echo ""
|
||||||
|
echo "Usage: $0 <command> [options] [basename]"
|
||||||
|
echo ""
|
||||||
|
echo "Available commands"
|
||||||
|
echo ""
|
||||||
|
echo " build --base=<BASENAME> - Build Docker image <BASENAME>.Dockerfile"
|
||||||
|
echo " run --base=<BASENAME> - Run Docker image <BASENAME>"
|
||||||
|
echo " push --base=<BASENAME> - Push Docker image <BASENAME> to the registry"
|
||||||
|
echo " list - List available images"
|
||||||
|
echo " help - This help message"
|
||||||
|
echo ""
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
if [ $list == 1 ]; then
|
||||||
|
echo "Available Docker images:"
|
||||||
|
for f in *.Dockerfile; do
|
||||||
|
filename=$( basename -- "$f" )
|
||||||
|
basename="${filename%.*}"
|
||||||
|
|
||||||
|
echo -e " \e[1;39m$basename\e[0m"
|
||||||
|
done
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# All commands after this require --base to be set
|
||||||
|
if [ -z "${base}" ]; then
|
||||||
|
echo "Usage: $0 <command>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "$base.Dockerfile" ]; then
|
||||||
|
echo -e "\e[1;31mERROR\e[0m: Dockerfile for '$base' not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${base_version}" ]; then
|
||||||
|
base_version="latest"
|
||||||
|
else
|
||||||
|
base_version="v$base_version"
|
||||||
|
fi
|
||||||
|
|
||||||
|
TAG="registry.freedesktop.org/pwithnall/malcontent/${base}:${base_version}"
|
||||||
|
|
||||||
|
if [ $build == 1 ]; then
|
||||||
|
echo -e "\e[1;32mBUILDING\e[0m: ${base} as ${TAG}"
|
||||||
|
$SUDO_CMD docker build \
|
||||||
|
--build-arg HOST_USER_ID="$UID" \
|
||||||
|
--tag "${TAG}" \
|
||||||
|
--file "${base}.Dockerfile" .
|
||||||
|
exit $?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $push == 1 ]; then
|
||||||
|
echo -e "\e[1;32mPUSHING\e[0m: ${base} as ${TAG}"
|
||||||
|
|
||||||
|
if [ $no_login == 0 ]; then
|
||||||
|
$SUDO_CMD docker login registry.freedesktop.org
|
||||||
|
fi
|
||||||
|
|
||||||
|
$SUDO_CMD docker push $TAG
|
||||||
|
exit $?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $run == 1 ]; then
|
||||||
|
echo -e "\e[1;32mRUNNING\e[0m: ${base} as ${TAG}"
|
||||||
|
$SUDO_CMD docker run \
|
||||||
|
--rm \
|
||||||
|
--volume "$(pwd)/..:/home/user/app" \
|
||||||
|
--workdir "/home/user/app" \
|
||||||
|
--tty \
|
||||||
|
--interactive "${TAG}" \
|
||||||
|
bash
|
||||||
|
exit $?
|
||||||
|
fi
|
|
@ -0,0 +1,29 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set +e
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
--log-file)
|
||||||
|
log_file="$2"
|
||||||
|
shift
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log_file="_build/meson-logs/testlog.json"
|
||||||
|
esac
|
||||||
|
|
||||||
|
meson test \
|
||||||
|
-C _build \
|
||||||
|
--timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
|
||||||
|
--no-suite flaky \
|
||||||
|
"$@"
|
||||||
|
|
||||||
|
exit_code=$?
|
||||||
|
|
||||||
|
python3 .gitlab-ci/meson-junit-report.py \
|
||||||
|
--project-name=malcontent \
|
||||||
|
--job-id "${CI_JOB_NAME}" \
|
||||||
|
--output "_build/${CI_JOB_NAME}-report.xml" \
|
||||||
|
"${log_file}"
|
||||||
|
|
||||||
|
exit $exit_code
|
Loading…
Reference in New Issue