vyos-build/scripts/build-packages

#!/usr/bin/env python3

import os
import subprocess
import sys
import shutil
import argparse
import logging

current_working_directory = os.getcwd()
repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n')
repo_sha  = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n')

def add_package(name, url=None, commit='HEAD', branch='equuleus', tag=None, custombuild_cmd=None):
    """
    Build up source package with URL and build commands executed during the later
    called build_package step.

    If no additional information is passed we will use the latest commit from current
    branch

    If no URL is passed we assume it's a regular VyOS package from the VyOS Github
    namespace at https://github.com/vyos
    """

    if not url:
        url = 'https://github.com/vyos/' + name + '.git'

    package = {
        'name': name,
        'url': url,
        'commit': commit,
        'tag': tag,
        'branch': branch,
        'path': repo_root + '/packages/' + name,
        'custombuild_cmd': custombuild_cmd
    }

    return package



def call(bashCommand, log, **kwargs):
    """
    Run command with arguments. Wait for command to complete.

    Send output to logging module passed as 'log'.
    """

    from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError
    from tempfile import TemporaryFile
    from time import sleep

    log.debug("Executing '{}'".format(bashCommand))

    # code borrowsed from:
    # https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p
    # the temp file will be automatically cleaned up
    output = TemporaryFile()
    error = TemporaryFile()

    kwargs['stdout'] = output
    kwargs['stderr'] = error
    kwargs['shell'] = True
    kwargs['universal_newlines'] = True

    sub = Popen(bashCommand, **kwargs)
    while sub.poll() is None:
        where = output.tell()
        lines = output.readline()
        if not lines:
            sleep(0.3)
            output.seek(where)
        else:
            log.debug(lines.decode().rstrip('\n'))

        where = error.tell()
        lines = error.readline()
        if not lines:
            sleep(0.3)
            error.seek(where)
        else:
           log.info(lines.decode().rstrip('\n'))


    # A last write needed after subprocess ends
    log.debug(output.read().decode().rstrip('\n'))
    log.info(error.read().decode().rstrip('\n'))

    error.close()
    output.close()
    return sub.returncode

def clone_package(pkg, log):
    """
    Clone Git repository from URL embedded in pkg to local disk

    First cleanup any possible leftovers from previous builds
    """

    if args.keep:
        log.debug("Keep possibly modified package '{}'".format(pkg['path']))
        return False
    elif args.clean:
        # delete repository from disk
        if os.path.isdir(pkg['path']):
            log.debug("Cleaning '{}'".format(pkg['path']))
            shutil.rmtree(pkg['path'])
    else:
        if os.path.isdir(pkg['path']):
            # Change current directory into Git repo for this package
            os.chdir(pkg['path'])

            bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit']
            return call(bashCommand, log)

    # resolve given tag to commit id to use shallow clone
    bashCommand = 'git clone ' + pkg['url']
    if pkg['tag']:
        bashCommand += ' --branch ' + pkg['tag']
    elif pkg['branch']:
        bashCommand += ' --depth 1 --branch ' + pkg['branch']

    bashCommand += ' ' + pkg['path']
    return call(bashCommand, log)


def build_package(pkg, log=None):
    """
    Generate Debian package from passed 'pkg'
    """

    # Change current directory into Git repo for this package
    os.chdir(pkg['path'])

    # Overwrite custom build command if required, e.g. libyang
    bashCommand = ''
    if pkg['custombuild_cmd']:
        bashCommand = pkg['custombuild_cmd']
    else:
        # Build package
        bashCommand = 'dpkg-buildpackage -uc -us -tc -b'
        if args.parallel:
            bashCommand += ' -j' + str(os.cpu_count())

    return call(bashCommand, log)

# a List of all Vyatta/VyOS based packages
vyos_packages = ['vyatta-bash',
                 'vyatta-cfg',
                 'vyatta-op',
                 'vyatta-cfg-system',
                 'vyatta-cfg-firewall',
                 'vyatta-op-firewall',
                 'vyatta-cfg-vpn',
                 'vyatta-op-vpn',
                 'vyatta-cfg-qos',
                 'vyatta-op-qos',
                 'vyatta-cfg-op-pppoe',
                 'vyatta-nat',
                 'vyatta-config-mgmt',
                 'vyatta-config-migrate',
                 'vyatta-zone',
                 'vyatta-cluster',
                 'vyatta-eventwatch',
                 'vyatta-webproxy',
                 'vyatta-cfg-quagga',
                 'vyatta-op-quagga',
                 'vyatta-wireless',
                 'vyatta-wirelessmodem',
                 'vyatta-wanloadbalance',
                 'vyatta-netflow',
                 'vyatta-lldp',
                 'vyatta-ipv6-rtradv',
                 'vyatta-ravpn',
                 'vyos-replace',
                 'vyos-nhrp',
                 'vyos-world',
                 'vyatta-iproute',
                 'vyos-vmwaretools-scripts',
                 'vyatta-biosdevname',
                 'vyos-opennhrp',
                 'vyos-salt-minion',
                 'mdns-repeater',
                 'udp-broadcast-relay',
                 'vyos-1x',
                 'vyatta-conntrack',
                 'vyatta-conntrack-sync',
                 'vyos-xe-guest-utilities',
                 'vyos-netplug',
                 'pmacct',
                 'ddclient',
                 'igmpproxy',
                 'eventwatchd',
                 'live-boot',
                 'conntrack-tools']

# Special packages mean packages which are located no in the VyOS namespace
# or require fancy build instructions
pkg_special = []

# libvyosconfig/ipaddrcheck uses a different default branch
libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \
                          "dpkg-buildpackage -b -us -uc -tc"
pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd))
pkg_special.append( add_package('ipaddrcheck', branch='master'))

# Packages where we directly build the upstream source
pkg_special.append( add_package('hvinfo',  url='https://github.com/dmbaturin/hvinfo.git', branch='master') )

# VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default
vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \
                            "autoreconf -i && ./configure --enable-python-eggs && " \
                            "cd src/libcharon/plugins/vici/python && make && " \
                            "python3 setup.py --command-packages=stdeb.command bdist_deb && " \
                            "mv ./deb_dist/*.deb " +  repo_root + "/packages"
pkg_special.append( add_package('vyos-strongswan', custombuild_cmd=vyos_strongswan_build_cmd) )

#
# FreeRangeRouting (FRR) packages
#
pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') )

frr_build_cmd  = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d'
pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) )
#libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages'
#pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) )


#
# We use keepalived from Debian Buster
#
keepalived_build_cmd = "sed -i 's/debhelper (>= 11)/debhelper (>= 9)/' debian/control && " \
                       "echo 9 > debian/compat && " \
                       "dpkg-buildpackage -b -us -uc -tc -j$(getconf _NPROCESSORS_ONLN)"
pkg_special.append( add_package('keepalived', url='https://salsa.debian.org/ipvs-team/pkg-keepalived.git', branch='master', commit='eae91c81', custombuild_cmd=keepalived_build_cmd) )

#
# Net-SNMP
#
# net-snmp conflicts with already installed libsnmp-dev thus we need to build with -d
net_snmp_build_cmd = "dpkg-buildpackage -b -us -uc -tc -d -j$(getconf _NPROCESSORS_ONLN)"
pkg_special.append( add_package('net-snmp', custombuild_cmd=net_snmp_build_cmd) )

#
# Linux (VyOS) Kernel
#
kernel_build_cmd = "make x86_64_vyos_defconfig && " \
                   "sed -i 's/\"kernel_version\": \"[0-9].[0-9][0-9].[0-9]*\"/\"kernel_version\": \"'$(make kernelversion)'\"/' " + repo_root + "/data/defaults.json && " \
                   "make bindeb-pkg LOCALVERSION='-amd64-vyos' KDEB_PKGVERSION=$(make kernelversion)-1 -j $(getconf _NPROCESSORS_ONLN)"
pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) )



#
# WireGuard Kernel Module
#
wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra > debian/wireguard-modules.install && " \
                      "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) )



#
# Accell-PPP Package and Kernel Module
#
accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra/*.ko > debian/vyos-accel-ppp-ipoe-kmod.install && " \
                      "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'-amd64-vyos#g' debian/rules && " \
                      "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) )


# A list of all packages we will build in the end
pkg_build = []




if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    exclusive = parser.add_mutually_exclusive_group(required=False)
    exclusive.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories')
    exclusive.add_argument('-k', '--keep', action='store_true', help='Keep modified Git repositories')
    exclusive.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build')

    parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance')
    parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build')
    parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build')
    parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs')
    parser.add_argument('--blacklist', nargs='+', help='Do not build/report packages when calling --list')


    args = parser.parse_args()

    levels = [ logging.INFO, logging.WARNING, logging.DEBUG ]
    level = levels[min(len(levels)-1,args.verbose)]  # capped to number of levels
    logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s")

    print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha))

    #
    # Exclude packages from build process,
    # also do not list them on --list
    #
    if args.blacklist:
        for exclude in args.blacklist:
            if exclude in vyos_packages:
                vyos_packages.remove(exclude)
                continue

            found = False
            for pkg in pkg_special:
                if exclude == pkg['name']:
                    found = True
                    # package already formed
                    pkg_special.remove(pkg)
                    break

            if not found:
                print("Invalid choice '" + exclude + "', --list-packages for complete list!")
                sys.exit(1)

    #
    # List all available (to be build) packages
    #
    if args.list_packages:
        print("Individual packages available for build:")
        for pkg in vyos_packages:
            print(' * ' + pkg)
        for pkg in pkg_special:
            print(' * ' + pkg['name'])

        sys.exit(0)

    #
    # Only add selective packages to the build list
    #
    if args.build:
        # NOTE: remove double added packages from list
        for target in args.build:
            if target in vyos_packages:
                pkg_build.append(add_package( target ))
                continue

            found = False
            for pkg in pkg_special:
                if target == pkg['name']:
                    found = True
                    # package already formed
                    pkg_build.append( pkg )
                    break

            if not found:
                print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!")
                sys.exit(1)

    else:
        # Add all VyOS packages to the package list
        for pkg in vyos_packages:
            pkg_build.append(add_package( pkg ))

        # We also wan't to build all of our special packages
        for pkg in pkg_special:
            pkg_build.append( pkg )

    # Build all VyOS packages (packages found on https://github.com/vyos
    # and referenced in vyos_packages)
    for pkg in pkg_build:
        # Create a logging instance per package
        log = logging.getLogger(pkg['name'])

        ret = clone_package(pkg, log)
        if ret:
            log.error("ERROR cloning source")
            sys.exit(1)
        else:
            # only build packages if fetch flag is not set
            if not args.fetch:
                ret = build_package(pkg, log)
                if ret:
                    log.error("ERROR building source")
                    sys.exit(1)

    sys.exit(0)