Replace build-submodules with Python based package builder

Support building individual VyOS packages by this modules. Call
scripts/build-packages -l to list all available packages which will be
build when invoking scripts/build-packages.
This commit is contained in:
Christian Poessinger 2019-04-09 20:15:57 +02:00
parent a0b3251dba
commit 63493afa4c
5 changed files with 557 additions and 544 deletions

136
Jenkinsfile vendored
View File

@ -1,22 +1,76 @@
#!/usr/bin/env groovy
// Copyright (C) 2018 VyOS maintainers and contributors
//
// This program is free software; you can redistribute it and/or modify
// in order to easy exprort images built to "external" world
// it under the terms of the GNU General Public License version 2 or later as
// published by the Free Software Foundation.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@NonCPS
def getGitBranchName() {
return scm.branches[0].name
}
def getGitRepoURL() {
return scm.userRemoteConfigs[0].url
}
// Returns true if this is a custom build launched on any project fork,
// returns false if this is build from git@github.com:vyos/vyos-build.git
def isCustomBuild() {
def gitURI = "git@github.com:vyos/vyos-build.git"
def httpURI = "https://github.com/vyos/vyos-build.git"
return ! ((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI))
}
def setDescription() {
def item = Jenkins.instance.getItemByFullName(env.JOB_NAME)
item.setDescription("VyOS image build using a\nPipeline build inside Docker container.")
// build up the main description text
def description = ""
description += "<h2>Build VyOS ISO image</h2>"
description += "All required Vyatta/VyOS packages are build from source prior to assembling the ISO."
if (isCustomBuild()) {
description += "<p style='border: 3px dashed red; width: 50%;'>"
description += "<b>Build not started from official Git repository!</b><br>"
description += "<br>"
description += "Repository: <font face = 'courier'>" + getGitRepoURL() + "</font><br>"
description += "Branch: <font face = 'courier'>" + getGitBranchName() + "</font><br>"
description += "</p>"
} else {
description += "Sources taken from Git branch: <font face = 'courier'>" + getGitBranchName() + "</font><br>"
}
item.setDescription(description)
item.save()
}
setDescription()
/* Only keep the 10 most recent builds. */
def projectProperties = [
[$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '5']],
[$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '1']],
]
properties(projectProperties)
setDescription()
pipeline {
options {
disableConcurrentBuilds()
timeout(time: 4, unit: 'HOURS')
parallelsAlwaysFailFast()
}
agent {
dockerfile {
filename 'Dockerfile'
@ -27,23 +81,71 @@ pipeline {
}
stages {
stage('Submodule Init') {
stage('VyOS Packages') {
steps {
sh '''
git submodule update --init --recursive --remote
'''
}
}
stage('Build Packages') {
steps {
sh '''
#!/bin/sh
scripts/build-submodules --verbose
'''
script {
def build = [:]
// get a list of available package from scripts/build-packages
packageList = sh(
script: "scripts/build-packages -l | grep '*' | sed -e 's/ \\* //'",
returnStdout: true
).split("\r?\n")
packageList.each { pkg ->
skipList = ['vyos-kernel', 'vyos-wireguard', 'vyos-accel-ppp']
if (pkg in skipList) {
return
}
// add each object from 'packageList' to the 'build' array
build[pkg] = {
// we're already in the script{} block, so do our advanced stuff here
sh(
script: "scripts/build-packages -vvv -b ${pkg}",
returnStdout: true
)
}
}
// Still within the 'Script' block, run the parallel array object
parallel build
}
}
}
stage('Build ISO') {
stage('Kernel') {
steps {
sh "scripts/build-packages -vvv -b vyos-kernel"
}
}
stage('Kernel Modules') {
steps {
script {
def build = [:]
kernelModules = ['vyos-wireguard', 'vyos-accel-ppp']
kernelModules.each { pkg ->
// add each object from 'packageList' to the 'build' array
build[pkg] = {
// we're already in the script{} block, so do our advanced stuff here
sh(
script: "scripts/build-packages -vvv -b ${pkg}",
returnStdout: true
)
}
}
// Still within the 'Script' block, run the parallel array object
parallel build
}
}
}
stage('Intel Drivers') {
steps {
sh "KSRC=\$(pwd)/packages/vyos-kernel scripts/build-intel-drivers"
}
}
stage('ISO Image') {
steps {
sh '''
#!/bin/sh

102
README.md
View File

@ -106,47 +106,18 @@ Run following command after building the QEMU image.
$ make vmware
```
## Building the ISO image inside a docker container
# Building ISO or individual packages via Docker
Using our [Dockerfile](docker/Dockerfile) you create your own Docker container
that is used to build a VyOS ISO image or other required VyOS packages. The
[Dockerfile](docker/Dockerfile) contains some of the most used packages needed
to build a VyOS ISO, a qemu image, and several of the submodules. Please note
that this is not complete and only gives you a brief overview!
to build a VyOS ISO, a qemu image, and several of the submodules.
```
squashfs-tools # Required for squashfs file system
git # Required, for cloning the source
autoconf # Required, for generating build scripts
dpkg-dev # Required, used in build scripts
live-build # Required, for ISO build
syslinux # Required, for ISO build
genisoimage # Required, for ISO build
make # Required, for ISO build
lsb-release # Required, used by configure script
fakechroot # Required, for ISO build
devscripts # Optional, for building submodules (kernel etc)
kernel-package # Optional, for building the kernel
libtool # Optional, for building certain packages (vyatta-op-vpn)
libglib2.0-dev # Optional, for building vyatta-cfg
libboost-filesystem-dev # Optional, for building vyatta-cfg
libapt-pkg-dev # Optional, for building vyatta-cfg
flex # Optional, for building vyatta-cfg
bison # Optional, for building vyatta-cfg
libperl-dev # Optional, for building vyatta-cfg
libnfnetlink-dev # Optional, for building vyatta-cfg-vpn
vim # Optional, vim, vi, nano or other text editor
jq # Optional, for qemu build
qemu-system-x86 # Optional, for qemu build
qemu-utils # Optional, for qemu build
packer # Optional, for qemu build
quilt # Optional, for building vyos-1x
python3-lxml # Optional, for building vyos-1x
python3-setuptools # Optional, for building vyos-1x
python3-nose # Optional, for building vyos-1x
python3-coverage # Optional, for building vyos-1x
...
```
If you are interested which individual packages are required please check our
[Dockerfile](docker/Dockerfile) which holds the most complete documentation
of required packages. Listing individual packages here tend to be always
outdated. We try to list required packages in groups through their inheritance
in the [Dockerfile](docker/Dockerfile).
To build the docker image ensure you have a working [Docker](https://www.docker.com)
environment and then run the following commands:
@ -155,7 +126,7 @@ environment and then run the following commands:
$ docker build -t vyos-builder docker
```
Run the newly built container:
Run newly built container:
```bash
$ docker run --rm -it --privileged -v $(pwd):/vyos -w /vyos vyos-builder bash
```
@ -183,32 +154,27 @@ order to build the VyOS ISO image.
## Building subpackages inside Docker
Prior to building packages you need to checkout and update the submodules you want to compile
[scripts/build-packages](scripts/build-packages) provides an easy interface
for automate the process of building all VyOS related packages that are not part
of the upstream Debian version. Execute it in the root of the `vyos-build`
directory to start compilation.
```bash
$ git submodule update --init packages/PACKAGENAME
$ cd packages/PACKAGENAME
$ git checkout BRANCH
$ scripts/build-packages -h
usage: build-packages [-h] [-v] [-c] [-l] [-b BUILD [BUILD ...]] [-f]
optional arguments:
-h, --help show this help message and exit
-v, --verbose Increase logging verbosity for each occurance
-c, --clean Re-clone required Git repositories
-l, --list-packages List all packages to build
-b BUILD [BUILD ...], --build BUILD [BUILD ...]
Whitespace separated list of packages to build
-f, --fetch Fetch sources only, no build
```
`PACKAGENAME` is the name of the package you want to compile
`BRANCH` is `crux` for VyOS 1.2.x, latest rolling releases use `current`
Fetching all submodules at once and update them to the recent remote branches
`HEAD` is done by calling:
```bash
$ git submodule update --init --recursive
$ git submodule update --remote
```
### Building packages
The [scripts/build-submodules](scripts/build-submodules) script is used to
automate the process of building (in the future) all VyOS related packages that
are not part of the upstream Debian version. Execute it in the root of the
`vyos-build` directory to start compilation on all supported packages that are
checked out.
Git repositoriers are automatically fetched and build on demand. If you wan't to
work offline you can fetch all source code first with the `-f` option.
The easiest way to compile is with the above mentioned [Docker](docker/Dockerfile)
container, it includes all dependencies for compiling supported packages.
@ -217,7 +183,7 @@ container, it includes all dependencies for compiling supported packages.
$ docker run --rm -it -v $(pwd):/vyos -w /vyos \
--sysctl net.ipv6.conf.lo.disable_ipv6=0 \
vyos-builder \
./scripts/build-submodules
./scripts/build-packages
```
**NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is required to build the
@ -237,29 +203,15 @@ Executed from the root of `vyos-build`
$ docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/PACKAGENAME \
--sysctl net.ipv6.conf.lo.disable_ipv6=0 \
vyos-builder \
dpkg-buildpackage -uc -us -tc -b
./scripts/build-packages -b <package>
```
**NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is only needed when
building `vyos-strongswan` and can be ignored on other packages.
**NOTE:** Prior to executing this you need to checkout and update the submodules
you want to recompile!
**NOTE:** `vyos-strongswan` will only compile on a Linux system, running on macOS
or Windows might result in a unittest deadlock (it never exits).
Packages that are known to not build using this procedure (as of now):
```
vyatta-util - Not needed anymore
vyatta-quagga - Not needed anymore
vyos-1x - Unmet build dependencies: whois libvyosconfig0
vyos-frr - A lot of requirements, scary stuff...
vyos-kernel - Need special build instructions
vyos-wireguard - Needs special build instructions
```
# Development process
## Git branches

79
scripts/build-intel-drivers Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash -x
basedir=$(pwd)
if [ -z "$KSRC" ]; then
echo "Kernel source KSRC= not specified"
exit 1
fi
VERSION=$(grep "^VERSION" $KSRC/Makefile | grep -Eo '[0-9]{1,4}')
PATCHLEVEL=$(grep "^PATCHLEVEL" $KSRC/Makefile | grep -Eo '[0-9]{1,4}')
SUBLEVEL=$(grep "^SUBLEVEL" $KSRC/Makefile | grep -Eo '[0-9]{1,4}')
KERNEL_VER="$VERSION.$PATCHLEVEL.$SUBLEVEL-amd64-vyos"
pkgdir="$basedir/packages"
if [ ! -d "$basedir/packages" ]; then
mkdir $basedir/packages
fi
URLS=" \
https://downloadmirror.intel.com/14687/eng/ixgbe-5.5.5.tar.gz \
https://downloadmirror.intel.com/13663/eng/igb-5.3.5.22.tar.gz \
https://downloadmirror.intel.com/15817/eng/e1000e-3.4.2.1.tar.gz \
https://downloadmirror.intel.com/24411/eng/i40e-2.7.29.tar.gz \
https://downloadmirror.intel.com/18700/eng/ixgbevf-4.5.2.tar.gz \
https://downloadmirror.intel.com/24693/eng/i40evf-3.6.15.tar.gz \
"
# The intel IGBVF driver can't be compiled with recent Kernel versions
# due to interlanl API changes. Driver is from 10.09.2015
# Source code is here: https://downloadmirror.intel.com/18298/eng/igbvf-2.3.9.6.tar.gz
for URL in $URLS
do
cd $pkgdir
filename=${URL##*/}
dirname_full=$(echo $filename | awk -F".tar.gz" '{print $1}')
dirname=$(echo $dirname_full | awk -F- '{print $1}')
version="$(echo $dirname_full | awk -F- '{print $2}')-0"
deb_pkg=${dirname}_${version}_amd64
deb_pkg_dir=$pkgdir/vyos-intel-${deb_pkg}
if [ -d "$dirname_full" ]; then
rm -rf $dirname_full
fi
if [ -d "$deb_pkg_dir" ]; then
rm -rf $deb_pkg_dir
fi
if [ ! -f "$filename" ]; then
wget ${URL}
ret=$?
if [ "$ret" != "0" ]; then
echo "Download of ${URL} failed!"
exit $ret
fi
fi
tar xf $filename
cd $dirname_full/src
KSRC=$KSRC INSTALL_MOD_PATH=$deb_pkg_dir \
make -j $(cat /proc/cpuinfo | grep processor | wc -l) install
mkdir -p $deb_pkg_dir/DEBIAN
echo "Package: vyos-intel-$dirname" >$deb_pkg_dir/DEBIAN/control
echo "Version: $version" >>$deb_pkg_dir/DEBIAN/control
echo "Section: kernel" >>$deb_pkg_dir/DEBIAN/control
echo "Priority: extra" >>$deb_pkg_dir/DEBIAN/control
echo "Architecture: amd64" >>$deb_pkg_dir/DEBIAN/control
echo "Maintainer: VyOS Package Maintainers <maintainers@vyos.net>" >>$deb_pkg_dir/DEBIAN/control
echo "Installed-Size: 9" >>$deb_pkg_dir/DEBIAN/control
echo "Depends: linux-image" >>$deb_pkg_dir/DEBIAN/control
echo "Description: Intel Vendor driver for $dirname" >>$deb_pkg_dir/DEBIAN/control
echo " Replacement for the in Kernel drivers" >>$deb_pkg_dir/DEBIAN/control
# Cleanup files which might also be present in linux-image-4.19.20-amd64-vyos
rm -rf $deb_pkg_dir/usr $deb_pkg_dir/lib/modules/$KERNEL_VER/modules.*
cd $pkgdir
dpkg-deb --build $(basename $deb_pkg_dir)
done

332
scripts/build-packages Executable file
View File

@ -0,0 +1,332 @@
#!/usr/bin/env python3
import os
import subprocess
import sys
import shutil
import argparse
import logging
current_working_directory = os.getcwd()
repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n')
repo_sha = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n')
def add_package(name, url=None, commit='HEAD', branch='current', tag=None, custombuild_cmd=None):
"""
Build up source package with URL and build commands executed during the later
called build_package step.
If no additional information is passed we will use the latest commit from current
branch
If no URL is passed we assume it's a regular VyOS package from the VyOS Github
namespace at https://github.com/vyos
"""
if not url:
url = 'https://github.com/vyos/' + name + '.git'
package = {
'name': name,
'url': url,
'commit': commit,
'tag': tag,
'branch': branch,
'path': repo_root + '/packages/' + name,
'custombuild_cmd': custombuild_cmd
}
return package
def call(bashCommand, log, **kwargs):
"""
Run command with arguments. Wait for command to complete.
Send output to logging module passed as 'log'.
"""
from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError
from tempfile import TemporaryFile
from time import sleep
log.debug("Executing '{}'".format(bashCommand))
# code borrowsed from:
# https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p
# the temp file will be automatically cleaned up
output = TemporaryFile()
error = TemporaryFile()
kwargs['stdout'] = output
kwargs['stderr'] = error
kwargs['shell'] = True
kwargs['universal_newlines'] = True
sub = Popen(bashCommand, **kwargs)
while sub.poll() is None:
where = output.tell()
lines = output.readline()
if not lines:
sleep(0.3)
output.seek(where)
else:
log.debug(lines.decode().rstrip('\n'))
where = error.tell()
lines = error.readline()
if not lines:
sleep(0.3)
error.seek(where)
else:
log.info(lines.decode().rstrip('\n'))
error.close()
output.close()
return sub.returncode
def clone_package(pkg, log):
"""
Clone Git repository from URL embedded in pkg to local disk
First cleanup any possible leftovers from previous builds
"""
if args.clean:
# delete repository from disk
if os.path.isdir(pkg['path']):
log.debug("Cleaning '{}'".format(pkg['path']))
shutil.rmtree(pkg['path'])
else:
if os.path.isdir(pkg['path']):
# Change current directory into Git repo for this package
os.chdir(pkg['path'])
bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit']
return call(bashCommand, log)
# resolve given tag to commit id to use shallow clone
bashCommand = 'git clone ' + pkg['url']
if pkg['tag']:
bashCommand += ' --branch ' + pkg['tag']
elif pkg['branch']:
bashCommand += ' --depth 1 --branch ' + pkg['branch']
bashCommand += ' ' + pkg['path']
return call(bashCommand, log)
def build_package(pkg, log=None):
"""
Generate Debian package from passed 'pkg'
"""
# Change current directory into Git repo for this package
os.chdir(pkg['path'])
# Overwrite custom build command if required, e.g. libyang
bashCommand = ''
if pkg['custombuild_cmd']:
bashCommand = pkg['custombuild_cmd']
else:
# Build package
bashCommand = 'dpkg-buildpackage -uc -us -tc -b'
if args.parallel:
bashCommand += ' -j' + str(os.cpu_count())
return call(bashCommand, log)
# a List of all Vyatta/VyOS based packages
vyos_packages = ['vyatta-bash',
'vyatta-cfg',
'vyatta-op',
'vyatta-cfg-system',
'vyatta-cfg-firewall',
'vyatta-op-firewall',
'vyatta-cfg-vpn',
'vyatta-op-vpn',
'vyatta-cfg-qos',
'vyatta-op-qos',
'vyatta-cfg-op-pppoe',
'vyatta-openvpn',
'vyatta-conntrack',
'vyatta-conntrack-sync',
'vyatta-nat',
'vyatta-config-mgmt',
'vyatta-config-migrate',
'vyatta-zone',
'vyatta-cluster',
'vyatta-eventwatch',
'vyatta-webproxy',
'vyatta-cfg-quagga',
'vyatta-op-quagga',
'vyatta-op-dhcp-server',
'vyatta-wireless',
'vyatta-wirelessmodem',
'vyatta-wanloadbalance',
'vyatta-netflow',
'vyatta-lldp',
'vyatta-ipv6-rtradv',
'vyatta-ravpn',
'vyos-nhrp',
'vyos-world',
'vyos-1x',
'vyatta-iproute',
'vyatta-quagga',
'vyos-strongswan',
'vyos-vmwaretools-scripts',
'vyos-netplug',
'vyos-xe-guest-utilities',
'vyatta-biosdevname',
'vyos-opennhrp',
'vyos-salt-minion',
'xl2tpd',
'mdns-repeater',
'udp-broadcast-relay',
'pmacct',
'ddclient',
'igmpproxy',
'eventwatchd',
'conntrack-tools']
# Special packages mean packages which are located no in the VyOS namespace
# or require fancy build instructions
pkg_special = []
# libvyosconfig/ipaddrcheck uses a different default branch
libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \
"dpkg-buildpackage -b -us -uc -tc"
pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd))
pkg_special.append( add_package('ipaddrcheck', branch='master'))
# Packages where we directly build the upstream source
pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') )
pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') )
#
# FreeRangeRouting (FRR) packages
#
pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') )
frr_build_cmd = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d'
pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) )
#libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages'
#pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) )
#
# Linux (VyOS) Kernel
#
kernel_build_cmd = "export VERSION=$(grep '^VERSION' Makefile | grep -Eo '[0-9]{1,4}') && " \
"export PATCHLEVEL=$(grep '^PATCHLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \
"export SUBLEVEL=$(grep '^SUBLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \
"echo ${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-amd64-vyos > " + repo_root + "/data/kernel_version && " \
"make x86_64_vyos_defconfig && " \
"LOCALVERSION='' make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos " \
" --revision=${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-0 kernel_headers kernel_image -j" + str(os.cpu_count())
pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) )
#
# WireGuard Kernel Module
#
wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/kernel_version) > debian/wireguard-modules.install && " \
"KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) )
#
# Accell-PPP Package and Kernel Module
#
accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/kernel_version)'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \
"sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/kernel_version)'#g' debian/rules && " \
"KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) )
# A list of all packages we will build in the end
pkg_build = []
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance')
parser.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories')
parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build')
parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build')
parser.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build')
parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs')
args = parser.parse_args()
levels = [ logging.INFO, logging.WARNING, logging.DEBUG ]
level = levels[min(len(levels)-1,args.verbose)] # capped to number of levels
logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s")
print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha))
if args.list_packages:
print("Individual packages available for build:")
for pkg in vyos_packages:
print(' * ' + pkg)
for pkg in pkg_special:
print(' * ' + pkg['name'])
sys.exit(0)
#
# Only add selective packages to the build list
#
if args.build:
# NOTE: remove double added packages from list
for target in args.build:
if target in vyos_packages:
pkg_build.append(add_package( target ))
continue
found = False
for pkg in pkg_special:
if target == pkg['name']:
found = True
# package already formed
pkg_build.append( pkg )
break
if not found:
print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!")
sys.exit(1)
else:
# Add all VyOS packages to the package list
for pkg in vyos_packages:
pkg_build.append(add_package( pkg ))
# We also wan't to build all of our special packages
for pkg in pkg_special:
pkg_build.append( pkg )
# Build all VyOS packages (packages found on https://github.com/vyos
# and referenced in vyos_packages)
for pkg in pkg_build:
# Create a logging instance per package
log = logging.getLogger(pkg['name'])
ret = clone_package(pkg, log)
if ret:
log.error("ERROR cloning source")
sys.exit(1)
else:
# only build packages if fetch flag is not set
if not args.fetch:
ret = build_package(pkg, log)
if ret:
log.error("ERROR building source")
sys.exit(1)
sys.exit(0)

View File

@ -1,452 +0,0 @@
#!/bin/bash
if [ ! -d "packages" ]; then
echo "This script needs to be executed inside the top root of vyos-build"
exit 1
fi
print_help() {
echo "Script for building all subpackages to vyos"
echo "Execute this sctipt from the root of the vyos-build directory"
echo ""
echo "This script could be executed from a Debian Jessie installation with all dependencies"
echo "or from the vyos-builder docker container."
echo ""
echo "Build the container:"
echo " docker build -t vyos-builder ."
echo "Compile packages:"
echo " docker run --rm -it -v $(pwd):/vyos -w /vyos --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder scripts/build-submodules"
echo ""
echo "Parameters:"
echo " --init-packages - Initiate all subpackages before building"
echo " --verbose - Enable verbose output"
echo " --build-[packagename] - build only selected packages"
}
BUILDLIST=""
VERBOSE=0
while test $# -gt 0
do
case "$1" in
-h|-?|--help)
print_help
exit 0
;;
--init-packages)
INIT_PACKAGES=1
;;
--verbose)
VERBOSE=1
;;
--build-*)
VAL=$(sed s/^--build-//g <<< $1)
BUILDLIST="$BUILDLIST $VAL"
;;
*)
(>&2 echo "Error: Argument $1 is not valid")
echo ""
print_help
exit 1
;;
esac
shift
done
status_start() {
echo -ne "[ ] $1"
}
status_ok() {
echo -ne "\r[\e[32m OK \e[39m]\n"
}
status_fail() {
echo -ne "\r[\e[31mFAIL\e[39m]\n"
}
status_skip() {
echo -ne "\r[SKIP] $1\033[K\n"
}
error_msg() {
echo -ne " $1\n"
}
verbose_msg() {
if [ $VERBOSE -ne 0 ]; then
echo "Current Environment:"
env
if [ ! -z "$1" ]; then
echo "Logfile:"
cat $1
fi
fi
}
ROOTDIR="$(pwd)"
PKGDIR="$ROOTDIR/packages"
SCRIPTDIR="$ROOTDIR/scripts"
# Source OPAM environment if not already set
if [ -z "$OPAMROOT" ]; then
if [ -x "$(command -v opam)" ]; then
eval $(opam env --root=/opt/opam --set-root)
else
echo "WARNING: 'opam' not installed, can't build VyConf and libvyosconfig"
fi
fi
package_in_buildlist() {
# Return true if buildlist is not set
if [ -z "$BUILDLIST" ]; then
return 0
fi
if [[ $BUILDLIST =~ (^|[[:space:]])$1($|[[:space:]]) ]]; then
return 0
fi
return 1
}
build_package() {
PKG=$1
COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
if ! package_in_buildlist $1; then
return 0
fi
status_start "Building package: $PKG Commit id: $COMMITID"
if [ ! -d "$PKGDIR/$PKG/debian" ]; then
status_skip "No source for: $PKG"
return 1
fi
( set -e; set -x
cd $PKGDIR/$PKG
dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
) >>$PKGDIR/$PKG.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
verbose_msg "$PKGDIR/$PKG.buildlog"
error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
return 2
fi
status_ok
}
echo "Cleaning up buildfiles..."
rm -rf $PKGDIR/*.deb
rm -rf $PKGDIR/*.changes
rm -rf $PKGDIR/*.buildlog
COMMITID=$(cd $PKGDIR; git rev-parse --short=10 HEAD)
echo "-----------------------------------------------------"
echo "Starting build process for all packages"
echo "vyos-build Commit ID: $COMMITID"
initialize_packages() {
status_start "Initializing packages"
(
set -x
git submodule update --init --recursive
git submodule update --remote
) >>$PKGDIR/init-packages.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
verbose_msg "$PKGDIR/init-packages.buildlog"
error_msg "Failed to update all package, look in init-packages.buildlog to examine the fault\n"
return 1
fi
status_ok
}
if [ $INIT_PACKAGES ]; then
initialize_packages
fi
build_libyang() {
PKG=libyang
COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
if ! package_in_buildlist $1; then
return 0
fi
status_start "Building package: $PKG Commit id: $COMMITID"
if [ ! -f "$PKGDIR/$PKG/README.md" ]; then
status_skip "No source for: $PKG"
return 1
fi
( set -e; set -x
cd $PKGDIR/$PKG
git checkout 179da47f2e8de
git clean -dxf
git reset --hard
mkdir build
cd build
cmake ..
make build-deb
cp debs/* $PKGDIR
) >>$PKGDIR/$PKG.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
return 2
fi
status_ok
}
build_libyang
build_frr() {
PKG=frr
COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
if ! package_in_buildlist $1; then
return 0
fi
status_start "Building package: $PKG Commit id: $COMMITID"
if [ ! -f "$PKGDIR/$PKG/README.md" ]; then
status_skip "No source for: $PKG"
return 1
fi
( set -e; set -x
cd $PKGDIR/$PKG
git clean -dxf
git reset --hard
# Run bootstrap.sh and make a dist tarball
# http://docs.frrouting.org/projects/dev-guide/en/latest/packaging-debian.html
./bootstrap.sh
./configure --with-pkg-extra-version=-vyos
make dist
# Create backports debian sources
mv debianpkg debian
make -f debian/rules backports
# Create a new directory to build the package and populate with package source.
mkdir frrpkg
cd frrpkg
tar xf ../frr_*.orig.tar.gz
cd frr*
source /etc/os-release
tar xf ../../frr_*${ID}${VERSION_ID}*.debian.tar.xz
# Build Debian Package
debuild --no-lintian --set-envvar=WANT_SNMP=1 --set-envvar=WANT_RPKI=1 \
--set-envvar=WANT_CUMULUS_MODE=0 -b -uc -us \
-j$(cat /proc/cpuinfo | grep processor | wc -l)
mv ../frr_*.deb $PKGDIR
) >>$PKGDIR/$PKG.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
return 2
fi
status_ok
}
build_frr
for PKG in mdns-repeater \
pmacct \
udp-broadcast-relay \
conntrack-tools \
eventwatchd \
ddclient \
rtrlib \
hvinfo \
igmpproxy \
ipaddrcheck \
lldpd \
libvyosconfig \
vyatta-bash \
vyatta-biosdevname \
vyatta-cfg \
vyatta-cfg-firewall \
vyatta-cfg-op-pppoe \
vyatta-cfg-qos \
vyatta-cfg-quagga \
vyatta-cfg-system \
vyatta-cfg-vpn \
vyatta-cluster \
vyatta-config-mgmt \
vyatta-config-migrate \
vyatta-conntrack \
vyatta-conntrack-sync \
vyatta-eventwatch \
vyatta-iproute \
vyatta-ipv6-rtradv \
vyatta-lldp \
vyatta-nat \
vyatta-netflow \
vyatta-op \
vyatta-op-dhcp-server \
vyatta-op-firewall \
vyatta-op-qos \
vyatta-op-quagga \
vyatta-op-vpn \
vyatta-openvpn \
vyatta-ravpn \
vyatta-wanloadbalance \
vyatta-webgui \
vyatta-webproxy \
vyatta-wireless \
vyatta-wirelessmodem \
vyatta-zone \
vyos-xe-guest-utilities \
vyos-keepalived \
vyos-netplug \
vyos-nhrp \
vyos-opennhrp \
vyos-salt-minion \
vyos-strongswan \
vyos-vmwaretools-scripts \
vyos-world \
vyos-1x \
; do
build_package "$PKG"
ERRCODE=$?
if [ "$ERRCODE" -ne "0" ]; then
exit $ERRCODE
fi
done
# KERNEL
build_kernel() {
PKG="vyos-kernel"
if ! package_in_buildlist "vyos-kernel"; then
return 0
fi
COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
status_start "Building-package: vyos-kernel Commit ID:$COMMITID"
if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then
status_skip "No source for: vyos-kernel"
return 0
fi
( set -e; set -x
cd $PKGDIR/vyos-kernel > /dev/null
bash -c "$SCRIPTDIR/build-kernel"
) >>$PKGDIR/vyos-kernel.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
verbose_msg "$PKGDIR/vyos-kernel.buildlog"
error_msg "Failed to build package vyos-kernel, look in vyos-kernel.buildlog to examine the fault\n"
return 1
fi
VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
ARCH=$(dpkg --print-architecture)
status_ok
}
build_kernel
ERRCODE=$?
if [ "$ERRCODE" -ne "0" ]; then
exit $ERRCODE
fi
# WIREGUARD
build_wireguard() {
PKG="vyos-wireguard"
if ! package_in_buildlist "vyos-wireguard"; then
return 0
fi
COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
status_start "Building package: vyos-wireguard Commit ID:$COMMITID"
if [ ! -d "$PKGDIR/vyos-wireguard/debian" ]; then
status_skip "No source for: vyos-wireguard"
return 0
fi
if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then
status_fail
error_msg "No Makefile found in kernel package"
return 1
fi
if ! grep -q "KBUILD_OUTPUT" $PKGDIR/vyos-kernel/Makefile; then
status_fail
error_msg "Failed to build package vyos-wireguard, no kernel source found\n"
return 1
fi
VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
ARCH=$(dpkg --print-architecture)
# Collect kernel information
( set -e; set -x
cd $PKGDIR/vyos-wireguard
echo "src/wireguard.ko /lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra" | tee debian/wireguard-modules.install
KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
) >>$PKGDIR/vyos-wireguard.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
verbose_msg "$PKGDIR/vyos-wireguard.buildlog"
error_msg "Failed to build package vyos-wireguard, look in vyos-wireguard.buildlog to examine the fault\n"
return 2
fi
status_ok
}
build_wireguard
ERRCODE=$?
if [ "$ERRCODE" -ne "0" ]; then
exit $ERRCODE
fi
# ACCEL-PPP
build_accel-ppp() {
PKG="vyos-accel-ppp"
if ! package_in_buildlist "accel-ppp"; then
return 0
fi
COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
status_start "Building package: vyos-accel-ppp Commit ID: $COMMITID"
if [ ! -d "$PKGDIR/vyos-accel-ppp/debian" ]; then
status_skip "No source for: vyos-accel-ppp"
return 0
fi
if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then
status_fail
error_msg "No Makefile found in kernel package"
return 1
fi
if ! grep -q "KBUILD_OUTPUT" $PKGDIR/vyos-kernel/Makefile; then
status_fail
error_msg "Failed to build package vyos-accel-ppp, no kernel source found\n"
fi
# Collect kernel information
VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
ARCH=$(dpkg --print-architecture)
( set -e; set -x
cd $PKGDIR/vyos-accel-ppp
echo "lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra/*.ko" | tee debian/vyos-accel-ppp-ipoe-kmod.install
sed -i "s#[0-9].[0-9][0-9].[0-9]-amd64-vyos#$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos#g" debian/rules
KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
) >>$PKGDIR/vyos-accel-ppp.buildlog 2>&1
if [ $? -ne 0 ]; then
status_fail
verbose_msg "$PKGDIR/vyos-accel-ppp.buildlog"
error_msg "Failed to build package vyos-accel-ppp, look in vyos-accel-ppp.buildlog to examine the fault\n"
return 1
fi
status_ok
}
build_accel-ppp
ERRCODE=$?
if [ "$ERRCODE" -ne "0" ]; then
exit $ERRCODE
fi