Commit d0954b88 authored by Paul Barker's avatar Paul Barker

Merge branch 'pbarker/ci' into 'master'

CI overhaul, import build directory and update/fix master

See merge request !4
parents ec3e945b 4082e94a
Pipeline #51079443 (#95) passed with stage
in 9 minutes and 43 seconds
......@@ -2,29 +2,20 @@ variables:
- export ORYX_VERSION=${CI_BUILD_REF_SLUG}_$(date -I)
- echo 'INHERIT += "rm_work"' > build/conf/auto.conf
- mkdir -p ~/ci-cache/{sstate-cache,downloads}
- ln -sfn ~/ci-cache/sstate-cache build/
- ln -sfn ~/ci-cache/downloads build/
- source build/conf/setenv
- rm -rf build/tmp
- rm -rf build/cache
- ./patches/
- ./scripts/ -M qemux86 -V ${ORYX_VERSION}
- ./ci/ ${CI_BUILD_REF_SLUG}_$(date -I) qemux86
- ./scripts/ -M raspberrypi3 -V ${ORYX_VERSION}
- ./ci/ ${CI_BUILD_REF_SLUG}_$(date -I) raspberrypi3
- ./scripts/ -M raspberrypi3-64 -V ${ORYX_VERSION}
- ./ci/ ${CI_BUILD_REF_SLUG}_$(date -I) raspberrypi3-64
- ./scripts/ -M beaglebone-yocto -V ${ORYX_VERSION}
- ./ci/ ${CI_BUILD_REF_SLUG}_$(date -I) beaglebone-yocto
[submodule "meta-oryx"]
path = meta-oryx
url =
[submodule "build"]
path = build
url =
[submodule "bitbake"]
path = bitbake
url = git://
bitbake @ 6065540c
Subproject commit d89358c7b8aa69f12b8c384c4fdb493782633494
Subproject commit 6065540cb923cde2299b1c16a4e2bb9376186799
Subproject commit 6ec20d789e202fb411a674c80ef7d5054ef39cd9
# LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf
# changes incompatibly
${ORYX_BASE}/openembedded-core/meta \
${ORYX_BASE}/meta-openembedded/meta-oe \
${ORYX_BASE}/meta-openembedded/meta-networking \
${ORYX_BASE}/meta-openembedded/meta-python \
${ORYX_BASE}/meta-openembedded/meta-filesystems \
${ORYX_BASE}/meta-virtualization \
${ORYX_BASE}/meta-raspberrypi \
${ORYX_BASE}/meta-yocto/meta-yocto-bsp \
${ORYX_BASE}/meta-oryx \
# This file is your local configuration file and is where all local user settings
# are placed. The comments in this file give some guide to the options a new user
# to the system might want to change but pretty much any configuration option can
# be set in this file. More adventurous users can look at local.conf.extended
# which contains other examples of configuration which can be placed in this file
# but new users likely won't need any of them initially.
# Lines starting with the '#' character are commented out and in some cases the
# default values are provided as comments to show people example syntax. Enabling
# the option is a question of removing the # character and making any change to the
# variable as required.
# Machine Selection
# You need to select a specific machine to target the build with. There are a selection
# of emulated machines available which can boot and run in the QEMU emulator:
#MACHINE ?= "qemuarm"
#MACHINE ?= "qemuarm64"
#MACHINE ?= "qemumips"
#MACHINE ?= "qemumips64"
#MACHINE ?= "qemuppc"
#MACHINE ?= "qemux86"
#MACHINE ?= "qemux86-64"
# This sets the default machine to be qemux86 if no other machine is selected:
MACHINE ??= "qemux86"
# Where to place downloads
# During a first build the system will download many different source code tarballs
# from various upstream projects. This can take a while, particularly if your network
# connection is slow. These are all stored in DL_DIR. When wiping and rebuilding you
# can preserve this directory to speed up this part of subsequent builds. This directory
# is safe to share between multiple builds on the same machine too.
# The default is a downloads directory under TOPDIR which is the build directory.
#DL_DIR ?= "${TOPDIR}/downloads"
# Where to place shared-state files
# BitBake has the capability to accelerate builds based on previously built output.
# This is done using "shared state" files which can be thought of as cache objects
# and this option determines where those files are placed.
# You can wipe out TMPDIR leaving this directory intact and the build would regenerate
# from these files if no changes were made to the configuration. If changes were made
# to the configuration, only shared state files where the state was still valid would
# be used (done using checksums).
# The default is a sstate-cache directory under TOPDIR.
#SSTATE_DIR ?= "${TOPDIR}/sstate-cache"
# Where to place the build output
# This option specifies where the bulk of the building work should be done and
# where BitBake should place its temporary files and output. Keep in mind that
# this includes the extraction and compilation of many applications and the toolchain
# which can use Gigabytes of hard disk space.
# The default is a tmp directory under TOPDIR.
TMPDIR = "${TOPDIR}/tmp"
# Package Management configuration
# This variable lists which packaging formats to enable. Multiple package backends
# can be enabled at once and the first item listed in the variable will be used
# to generate the root filesystems.
# Options are:
# - 'package_deb' for debian style deb files
# - 'package_ipk' for ipk files are used by opkg (a debian style embedded package manager)
# - 'package_rpm' for rpm style packages
# E.g.: PACKAGE_CLASSES ?= "package_rpm package_deb package_ipk"
# We default to ipk:
PACKAGE_CLASSES ?= "package_ipk"
# SDK/ADT target architecture
# This variable specifies the architecture to build SDK/ADT items for and means
# you can build the SDK packages for architectures other than the machine you are
# running the build on (i.e. building i686 packages on an x86_64 host).
# Supported values are i686 and x86_64
SDKMACHINE ?= "x86_64"
# Extra image configuration defaults
# The EXTRA_IMAGE_FEATURES variable allows extra packages to be added to the generated
# images. Some of these options are added to certain image types automatically. The
# variable can contain the following options:
# "dbg-pkgs" - add -dbg packages for all installed packages
# (adds symbol information for debugging/profiling)
# "dev-pkgs" - add -dev packages for all installed packages
# (useful if you want to develop against libs in the image)
# "ptest-pkgs" - add -ptest packages for all ptest-enabled packages
# (useful if you want to run the package test suites)
# "tools-sdk" - add development tools (gcc, make, pkgconfig etc.)
# "tools-debug" - add debugging tools (gdb, strace)
# "eclipse-debug" - add Eclipse remote debugging support
# "tools-profile" - add profiling tools (oprofile, lttng, valgrind)
# "tools-testapps" - add useful testing tools (ts_print, aplay, arecord etc.)
# "debug-tweaks" - make an image suitable for development
# e.g. ssh root access has a blank password
# There are other application targets that can be used here too, see
# meta/classes/image.bbclass and meta/classes/core-image.bbclass for more details.
# We default to enabling the debugging tweaks.
#EXTRA_IMAGE_FEATURES = "debug-tweaks"
# Additional image features
# The following is a list of additional classes to use when building images which
# enable extra features. Some available options which can be included in this variable
# are:
# - 'buildstats' collect build statistics
# - 'image-mklibs' to reduce shared library files size for an image
# - 'image-prelink' in order to prelink the filesystem image
# - 'image-swab' to perform host system intrusion detection
# NOTE: if listing mklibs & prelink both, then make sure mklibs is before prelink
# NOTE: mklibs also needs to be explicitly enabled for a given image, see local.conf.extended
# image-prelink disabled for now due to issues with IFUNC symbol relocation
USER_CLASSES ?= "buildstats image-mklibs"
# Runtime testing of images
# The build system can test booting virtual machine images under qemu (an emulator)
# after any root filesystems are created and run tests against those images. To
# enable this uncomment this line. See classes/testimage(-auto).bbclass for
# further details.
# Interactive shell configuration
# Under certain circumstances the system may need input from you and to do this it
# can launch an interactive shell. It needs to do this since the build is
# multithreaded and needs to be able to handle the case where more than one parallel
# process may require the user's attention. The default is iterate over the available
# terminal types to find one that works.
# Examples of the occasions this may happen are when resolving patches which cannot
# be applied, to use the devshell or the kernel menuconfig
# Supported values are auto, gnome, xfce, rxvt, screen, konsole (KDE 3.x only), none
# Note: currently, Konsole support only works for KDE 3.x due to the way
# newer Konsole versions behave
#OE_TERMINAL = "auto"
# By default disable interactive patch resolution (tasks will just fail instead):
# Disk Space Monitoring during the build
# Monitor the disk space during the build. If there is less that 1GB of space or less
# than 100K inodes in any key build location (TMPDIR, DL_DIR, SSTATE_DIR), gracefully
# shutdown the build. If there is less that 100MB or 1K inodes, perform a hard abort
# of the build. The reason for this is that running completely out of space can corrupt
# files and damages the build in ways which may not be easily recoverable.
# It's necesary to monitor /tmp, if there is no space left the build will fail
# with very exotic errors.
STOPTASKS,/tmp,100M,100K \
ABORT,${DL_DIR},100M,1K \
# Shared-state files from other locations
# As mentioned above, shared state files are prebuilt cache data objects which can
# used to accelerate build time. This variable can be used to configure the system
# to search other mirror locations for these objects before it builds the data itself.
# This can be a filesystem directory, or a remote url such as http or ftp. These
# would contain the sstate-cache results from previous builds (possibly from other
# machines). This variable works like fetcher MIRRORS/PREMIRRORS and points to the
# cache locations to check for the shared objects.
# NOTE: if the mirror uses the same structure as SSTATE_DIR, you need to add PATH
# at the end as shown in the examples below. This will be substituted with the
# correct path within the directory structure.
#file://.* http://someserver.tld/share/sstate/PATH;downloadfilename=PATH \n \
#file://.* file:///some/local/dir/sstate/PATH"
# Qemu configuration
# By default qemu will build with a builtin VNC server where graphical output can be
# seen. The two lines below enable the SDL backend too. By default libsdl-native will
# be built, if you want to use your host's libSDL instead of the minimal libsdl built
# by libsdl-native then uncomment the ASSUME_PROVIDED line below.
#PACKAGECONFIG_append_pn-qemu-native = " sdl"
#PACKAGECONFIG_append_pn-nativesdk-qemu = " sdl"
#ASSUME_PROVIDED += "libsdl-native"
# CONF_VERSION is increased each time build/conf/ changes incompatibly and is used to
# track the version of this file when it was generated. This can safely be ignored if
# this doesn't mean anything to you.
DISTRO = "oryx"
# Generate mirror tarballs for sources fetched from version control systems.
# This helps us to create a full source mirror.
export ORYX_BASE=$(realpath $(dirname $BASH_SOURCE)/../..)
export TOPDIR="${ORYX_BASE}/build"
TCMODE all_proxy ftp_proxy ftps_proxy http_proxy https_proxy no_proxy \
export PATH="${ORYX_BASE}/openembedded-core/scripts:${ORYX_BASE}/bitbake/bin:${PATH}"
cd "${TOPDIR}"
#! /bin/bash
set -e
set -o pipefail
if [[ -z "$VERSION" ]]; then
# Run a really clean build to ensure that all necessary sources end up in the
# downloads directory
rm -rf downloads sstate-cache
do_oryx_full() {
./scripts/ -C -L -V $VERSION -M $MACHINE -A host
./scripts/ -C -L -V $VERSION -M $MACHINE -A host-test
./scripts/ -C -L -V $VERSION -M $MACHINE -S guest -A full-cmdline
./scripts/ -C -L -V $VERSION -M $MACHINE -S guest -A minimal
do_oryx_full qemux86
do_oryx_full raspberrypi3
do_oryx_full raspberrypi3-64
do_oryx_full beaglebone-yocto
# Sync the relevant files from the downloads directory into a "sources"
# directory
mkdir -p pub/$VERSION/downloads
find downloads -maxdepth 1 -type f -not -name *.done -exec ln -f {} pub/$VERSION/{} \;
mv pub/$VERSION/downloads pub/$VERSION/sources
# Capture bitbake and layers
git -C .. archive-all --prefix oryx-$VERSION/ build/pub/$VERSION/oryx-$VERSION.tar.xz
#! /usr/bin/env python3
import argparse
import glob
import os
import subprocess
import shutil
import sys
import tarfile
def msg(message):
print(message, flush=True)
def get_subfolder(args):
return "%s/%s/%s/%s/" % (args.version, args.machine, args.system_profile, args.application_profile)
def run_build(args):
"""Run a build using the configuration given in the args namespace"""
% (args.version, args.machine, args.system_profile, args.application_profile))
os.environ['ORYX_VERSION'] = args.version
os.environ['MACHINE'] = args.machine
os.environ['ORYX_SYSTEM_PROFILE'] = args.system_profile
os.environ['ORYX_APPLICATION_PROFILE'] = args.application_profile
subfolder = get_subfolder(args)
bitbake_args = ""
if args.bitbake_continue:
bitbake_args += " -k"
bitbake_status ="bitbake %s oryx-publish" % (bitbake_args), shell=True)
# Copy the contents of the output files out of the tmp folder. The
# destination folder must not already exist for copytree to work.
folder = "tmp/deploy/oryx/" + subfolder
newfolder = "pub/" + subfolder
if os.path.exists(newfolder):
if os.path.exists(folder):
shutil.copytree(folder, newfolder)
# At least create an empty folder in case we need to create a FAILED
# file
# Create FAILED file if bitbake failed
if bitbake_status != 0:
failedfile = "%s/FAILED" % (newfolder)
return bitbake_status
def capture_logs(args):
"""Capture log files and archive them"""
directorylist = ['tmp/work/*/*/*/temp/run.*', 'tmp/work/*/*/*/temp/log.*', 'tmp/log/*']
globlist = []
for directory in directorylist:
for filename in glob.iglob(directory):
subfolder = get_subfolder(args)
tarlocation = "pub/%s/logs.tar.gz" % (subfolder)
tar =, "w:gz")
for item in globlist:
def parse_args():
"""Parse command line arguments into an args namespace"""
parser = argparse.ArgumentParser(
description="Build script for Oryx Embedded Linux"
parser.add_argument("-V", dest="version", metavar="VERSION", default="dev",
help="Version string used to identify this build")
parser.add_argument("-S", dest="system_profile", metavar="SYSTEM_PROFILE", default="native",
help="System profile selection")
parser.add_argument("-A", dest="application_profile", metavar="APPLICATION_PROFILE", default="minimal",
help="Application profile selection")
parser.add_argument("-M", dest="machine", metavar="MACHINE", default="qemux86",
help="Machine selection")
parser.add_argument("-C", "--clean", help="Performs a clean build", action="store_true")
parser.add_argument("-L", "--logs", help="Captures and archives log files", action="store_true")
parser.add_argument("-k", "--continue", dest="bitbake_continue", action="store_true",
help="Continue as much as possible after an error")
return parser.parse_args()
def main():
args = parse_args()
if args.clean and os.path.exists("tmp"):
msg(">>> Cleaning")
exitcode = run_build(args)
if args.logs:
msg(">>> Capturing logs")
# Configuration modifications for CI
INHERIT += "rm_work"
DL_DIR = "${HOME}/ci-cache/downloads"
SSTATE_DIR = "${HOME}/ci-cache/sstate-cache"
#! /usr/bin/env python3
import argparse
import shutil
import subprocess
import sys
( 'native' , 'host' ),
( 'native' , 'host-test' ),
( 'guest' , 'minimal' ),
( 'guest' , 'full-cmdline' )
def msg(message):
print(message, flush=True)
def parse_args():
parser = argparse.ArgumentParser(
description='CI script for Oryx Embedded Linux')
parser.add_argument('version', help='Version string used to identify this build')
parser.add_argument('machine', help='Machine to test')
return parser.parse_args()
def prepare():
msg('>>> Copying CI configuration')
shutil.copyfile('ci/auto.conf', 'build/conf/auto.conf')
def run_build(version, machine, sys_profile, app_profile):
script = 'source build/conf/setenv && ./scripts/ -k -V %s -M %s -S %s -A %s' % (
version, machine, sys_profile, app_profile)
cp =, shell=True, executable='/bin/bash')
return cp.returncode
def run_all_builds(version, machine):
failures = 0
for i, (sys_profile, app_profile) in enumerate(BUILDS, 1):
msg('>>> Running build %d of %d' % (i, N_BUILDS))
rc = run_build(version, machine, sys_profile, app_profile)
if rc == 0:
msg('>>> Build succeeded')
msg('>>> Build failed')
failures += 1
msg('>>> Completed %d builds with %d failures' % (N_BUILDS, failures))
if failures > 0:
return 1
return 0
def cleanup():
msg('>>> Removing tmp & cache directories')
def main():
args = parse_args()
rc = run_all_builds(args.version, args.machine)
if __name__ == '__main__':
meta-openembedded @ d0060681
Subproject commit cca27b5ea7569d2730ee5da7ee7f47b39d775d89
Subproject commit d0060681f6b56e46c2c0e100bd579c6affd1d869
Subproject commit ba8191738e15757fddefd31293376f4216e44be3
Subproject commit 38964828d4131b49cf853651417e3ac85ee1c024
Subproject commit d5d92f2cb7f2bf7ef3322da560f35deb59deedf6
Subproject commit 1b990fb8e2c21fdfac97c72d471e658087ce7026
meta-virtualization @ 2a44defc
Subproject commit d051460bac9a0a8999703aa7feeb546e9026385c
Subproject commit 2a44defc697cdf16521e5c11e894bd93c926f059
meta-yocto @ 7b2ba7e2
Subproject commit e9a9591f36d1e7a602fdb245a1f2d1707958e0e5
Subproject commit 7b2ba7e227ac582aafa0fc45049672974c3e1565
openembedded-core @ 28e631d6
Subproject commit 748f946ee74f7480200a7eb0bb0b695467b08f0a
Subproject commit 28e631d6dbc0a126253c0a072b8f39ff683bfa3a
#! /bin/bash
set -e
shopt -s nullglob
REPOLIST="bitbake openembedded-core meta-openembedded meta-raspberrypi meta-virtualization meta-yocto"
PATCHES_DIR=$(realpath $(dirname $0))
for repo in $REPOLIST; do
for p in "$PATCHES_DIR/$repo"/*; do
echo "Applying $p"
git -C $repo am "$p"
From 0e0ac65b232dfcd9831cedb80fa80f0beef7812a Mon Sep 17 00:00:00 2001
From: Paul Barker <>
Date: Sat, 9 Mar 2019 22:33:26 +0000
Subject: [PATCH] kernel-yocto.bbclass: Fix find_patches function
Signed-off-by: Paul Barker <>
meta/classes/kernel-yocto.bbclass | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/meta/classes/kernel-yocto.bbclass b/meta/classes/kernel-yocto.bbclass
index bd7a957583..24137ce026 100644
--- a/meta/classes/kernel-yocto.bbclass
+++ b/meta/classes/kernel-yocto.bbclass
@@ -15,10 +15,7 @@ def find_patches(d,subdir):
patchdir = ''
if "patchdir" in parm:
patchdir = parm["patchdir"]
- if patchdir:
- if subdir == patchdir:
- patch_list.append(local)
- else:
+ if subdir == patchdir:
return patch_list
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment