Merge branch 'current' of github.com:c-po/vyos-build into current

* 'current' of github.com:c-po/vyos-build:
  Jenkins: cleanup
  Docker: Make OPAM system wide available
  When building linux-kernel use number of host CPUs on parallel build
  When building parse Git commit ID
  When building frr clean out Git repository first
  Fix --verbose option when building submodules
  Build submodules in parallel with number of host CPUs
  submodules: add igmpproxy
  submodules: add vyos-xe-guest-utilities

.gitmodules

@@ -233,3 +233,11 @@
 	path = packages/vyos-netplug
 	url = https://github.com/vyos/vyos-netplug.git
 	branch = current
+[submodule "packages/vyos-xe-guest-utilities"]
+	path = packages/vyos-xe-guest-utilities
+	url = http://github.com/vyos/vyos-xe-guest-utilities.git
+	branch = current
+[submodule "packages/igmpproxy"]
+	path = packages/igmpproxy
+	url = https://github.com/vyos/igmpproxy.git
+	branch = current

Jenkinsfile

@@ -30,14 +30,16 @@ pipeline {
         stage('Submodule Init') {
             steps {
                 sh '''
-                    git submodule update --init --recursive
-                    git submodule update --remote
+                    git submodule update --init --recursive --remote
                 '''
             }
         }
         stage('Build Packages') {
             steps {
-                sh 'scripts/build-submodules --verbose'
+                sh '''
+                    #!/bin/sh
+                    scripts/build-submodules --verbose
+                '''
             }
         }
 
@@ -68,7 +70,12 @@ pipeline {
             echo 'One way or another, I have finished'
             // the 'build' directory got elevated permissions during the build
             // cdjust permissions so it can be cleaned up by the regular user
-            sh 'sudo chmod -R 777 build/'
+            sh '''
+                #!/bin/bash
+                if [ -d build ]; then
+                    sudo chmod -R 777 build/
+                fi
+            '''
             deleteDir() /* cleanup our workspace */
         }
     }

docker/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y \
       curl \
       make \
       sudo \
+      mc \
       live-build \
       pbuilder \
       devscripts \
@@ -190,10 +191,13 @@ RUN apt-get update && apt-get install -y \
 # Packages needed for libvyosconfig
 RUN curl https://raw.githubusercontent.com/ocaml/opam/2.0.2/shell/install.sh --output /tmp/opam_install.sh && \
     sed -i 's/read BINDIR/BINDIR=""/' /tmp/opam_install.sh && sh /tmp/opam_install.sh && \
-    opam init --disable-sandboxing && \
-    eval $(opam env) && opam switch create 4.07.0 && \
-    eval $(opam env) && opam install -y oasis && \
-    eval $(opam env) && opam install -y \
+    opam init --root=/opt/opam --comp=4.07.0 --disable-sandboxing
+
+RUN eval $(opam env --root=/opt/opam --set-root) && \
+    opam install -y \
+      oasis
+
+RUN eval $(opam env --root=/opt/opam --set-root) && opam install -y \
       fileutils \
       lwt \
       lwt_ppx \
@@ -210,21 +214,21 @@ RUN curl https://raw.githubusercontent.com/ocaml/opam/2.0.2/shell/install.sh --o
       ctypes-foreign \
       menhir
 
-RUN eval $(opam env) && opam install -y \
+RUN eval $(opam env --root=/opt/opam --set-root) && opam install -y \
       ctypes
 
 # Build VyConf which is required to build libvyosconfig
-RUN eval $(opam env) && \
-    git clone https://github.com/vyos/vyconf.git && \
-    cd vyconf && \
-    git checkout 0f121c12a84200 && \
+RUN eval $(opam env --root=/opt/opam --set-root) && \
+    git clone https://github.com/vyos/vyconf.git /tmp/vyconf && \
+    cd /tmp/vyconf && \
+    git checkout v0.1 && \
     opam pin add vyconf .
 
 # Build libvyosconfig
-RUN eval $(opam env) && \
+RUN eval $(opam env --root=/opt/opam --set-root) && \
     git clone https://github.com/vyos/libvyosconfig && \
     cd libvyosconfig && \
-    git checkout e75e4ae638c49e && \
+    git checkout v0.0.4 && \
     dpkg-buildpackage -uc -us -tc -b && \
     dpkg -i ../libvyosconfig0_*_amd64.deb
 
@@ -232,6 +236,10 @@ RUN eval $(opam env) && \
 RUN apt-get update && apt-get install -y \
       whois
 
+# Packages needed for vyos-xe-guest-utilities
+RUN apt-get update && apt-get install -y \
+      golang
+
 # Update live-build
 RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list && \
     apt-get update && apt-get install -y -t stretch live-build && \

docker/entrypoint.sh

@@ -23,6 +23,10 @@ if ! grep -q $NEW_GID /etc/group; then
     groupadd --gid $NEW_GID $USER_NAME
 fi
 
+# libvyosconfig depends on OCaml/OPAM so we make sure
+# we can use it
+echo "eval \$(opam env --root=/opt/opam --set-root)" >> /etc/skel/.bashrc
+
 useradd --shell /bin/bash --uid $NEW_UID --gid $NEW_GID --non-unique --create-home $USER_NAME
 usermod --append --groups sudo $USER_NAME
 sudo chown $NEW_UID:$NEW_GID /home/$USER_NAME

packages/igmpproxy

@@ -0,0 +1 @@
+Subproject commit 1f87c675725bff8f380bf7364c3961ee5a5a9e4f

packages/vyos-xe-guest-utilities

@@ -0,0 +1 @@
+Subproject commit a31c092661c65e177967535e8c53dc5b7a04b5cd

scripts/build-kernel

@@ -27,7 +27,7 @@ case "$ARCH" in
         if [ ${PATCHLEVEL} -lt 14 ]; then
             TARGETS+=" kernel_manual kernel_doc"
         fi
-        CONCURRENCY_LEVEL=4 LOCALVERSION="" make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos --revision=$VERSION.$PATCHLEVEL.$SUBLEVEL-1+vyos1+current1 -j4 ${TARGETS}
+        CONCURRENCY_LEVEL=4 LOCALVERSION="" make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos --revision=$VERSION.$PATCHLEVEL.$SUBLEVEL-1+vyos1+current1 -j$(cat /proc/cpuinfo | grep processor | wc -l) ${TARGETS}
     ;;
 
     armhf)

scripts/build-submodules

@@ -83,10 +83,8 @@ package_in_buildlist() {
   if [[ $BUILDLIST =~ (^|[[:space:]])$1($|[[:space:]]) ]]; then
     return 0
   fi
-  
-  return 1
 
- 
+  return 1
 }
 
 build_package() {
@@ -102,7 +100,7 @@ build_package() {
   fi
   ( set -e; set -x
     cd $PKGDIR/$PKG
-    dpkg-buildpackage -uc -us -tc -b
+    dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
   ) >>$PKGDIR/$PKG.buildlog 2>&1
   if [ $? -ne 0 ]; then
     status_fail
@@ -130,7 +128,7 @@ initialize_packages() {
   ) >>$PKGDIR/init-packages.buildlog 2>&1
   if [ $? -ne 0 ]; then
       status_fail
-      if [ $VERBOSE ]; then
+      if [ $VERBOSE -eq 1 ]; then
           cat $PKGDIR/init-packages.buildlog
       fi
       error_msg "Failed to update all package, look in init-packages.buildlog to examine the fault\n"
@@ -144,10 +142,11 @@ fi
 
 build_frr() {
   PKG=frr
-  if ! package_in_buildlist "frr"; then
+  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
+  if ! package_in_buildlist $1; then
     return 0
   fi
-  status_start "Building package: $PKG (Git commit/tag: $FRR_VER)"
+  status_start "Building package: $PKG Commit id: $COMMITID"
   if [ ! -f "$PKGDIR/$PKG/README.md" ]; then
     status_skip "No source for: $PKG"
     return 1
@@ -155,6 +154,8 @@ build_frr() {
 
   ( set -e; set -x
     cd $PKGDIR/$PKG
+    git clean -dxf
+    git reset --hard
 
     # Run bootstrap.sh and make a dist tarball
     # http://docs.frrouting.org/projects/dev-guide/en/latest/packaging-debian.html
@@ -172,7 +173,6 @@ build_frr() {
     tar xf ../frr_*.orig.tar.gz
     cd frr*
     source /etc/os-release
-    ls -al ../../frr_*${ID}${VERSION_ID}*
     tar xf ../../frr_*${ID}${VERSION_ID}*.debian.tar.xz
 
     # As libyang is not (yet) build and installed as native Debian package FRR
@@ -187,7 +187,8 @@ build_frr() {
 
     # Build Debian Package
     debuild --no-lintian --set-envvar=WANT_SNMP=1 --set-envvar=WANT_RPKI=1 \
-        --set-envvar=WANT_CUMULUS_MODE=0 -b -uc -us
+        --set-envvar=WANT_CUMULUS_MODE=0 -b -uc -us \
+        -j$(cat /proc/cpuinfo | grep processor | wc -l)
 
     mv ../frr_*.deb $PKGDIR
   ) >>$PKGDIR/$PKG.buildlog 2>&1
@@ -207,6 +208,7 @@ for PKG in mdns-repeater \
            eventwatchd \
            ddclient \
            rtrlib \
+           igmpproxy \
            libvyosconfig \
            vyatta-bash \
            vyatta-biosdevname \
@@ -242,6 +244,7 @@ for PKG in mdns-repeater \
            vyatta-wireless \
            vyatta-wirelessmodem \
            vyatta-zone \
+           vyos-xe-guest-utilities \
            vyos-keepalived \
            vyos-netplug \
            vyos-nhrp \
@@ -277,7 +280,7 @@ build_kernel() {
   ) >>$PKGDIR/vyos-kernel.buildlog 2>&1
   if [ $? -ne 0 ]; then
     status_fail
-    if [ $VERBOSE ]; then
+    if [ $VERBOSE -eq 1 ]; then
         cat $PKGDIR/vyos-kernel.buildlog
     fi
     error_msg "Failed to build package vyos-kernel, look in vyos-kernel.buildlog to examine the fault\n"
@@ -332,11 +335,11 @@ build_wireguard() {
   ( set -e; set -x
     cd $PKGDIR/vyos-wireguard
     echo "src/wireguard.ko /lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra" | tee debian/wireguard-modules.install
-    KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b
+    KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
   ) >>$PKGDIR/vyos-wireguard.buildlog 2>&1
   if [ $? -ne 0 ]; then
     status_fail
-    if [ $VERBOSE ]; then
+    if [ $VERBOSE -eq 1 ]; then
         cat $PKGDIR/vyos-wireguard.buildlog
     fi
     error_msg "Failed to build package vyos-wireguard, look in vyos-wireguard.buildlog to examine the fault\n"
@@ -383,11 +386,11 @@ build_accel-ppp() {
     cd $PKGDIR/vyos-accel-ppp
     echo "lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra/*.ko" | tee debian/vyos-accel-ppp-ipoe-kmod.install
     sed -i "s#[0-9].[0-9][0-9].[0-9]-amd64-vyos#$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos#g" debian/rules
-    KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b
+    KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
   ) >>$PKGDIR/vyos-accel-ppp.buildlog 2>&1
   if [ $? -ne 0 ]; then
     status_fail
-    if [ $VERBOSE ]; then
+    if [ $VERBOSE -eq 1 ]; then
         cat $PKGDIR/vyos-accel-ppp.buildlog
     fi
     error_msg "Failed to build package vyos-accel-ppp, look in vyos-accel-ppp.buildlog to examine the fault\n"