Docker: T2640: Added ability to build Docker images with VyOS

This implementation works, but it is on the proof-of-concept stage and requires extended testing.

docker-vyos/Dockerfile

@@ -0,0 +1,68 @@
+# syntax = docker/dockerfile:1
+
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Define arguments for VyOS image
+ARG VYOS_VERSION
+ARG BUILD_DATE
+ARG DEBIAN_VERSION
+
+# Use Debian as base layer
+FROM debian:${DEBIAN_VERSION}-slim
+# Copy installer script and default build settings
+COPY [ "data/defaults.json", "data/live-build-config/archives/*", "docker-vyos/vyos_install_common.sh", "docker-vyos/vyos_install_stage_01.sh", "/tmp/" ]
+COPY [ "data/live-build-config/hooks/live/*", "/tmp/hooks/" ]
+
+
+# Install VyOS dependencies
+WORKDIR /tmp
+RUN bash /tmp/vyos_install_stage_01.sh
+
+
+# Install VyOS specific software
+COPY [ "data/defaults.json", "docker-vyos/vyos_install_common.sh", "docker-vyos/vyos_install_stage_02.sh", "/tmp/" ]
+RUN bash /tmp/vyos_install_stage_02.sh
+
+
+# Tune system for VyOS
+COPY [ "docker-vyos/vyos_install_common.sh", "docker-vyos/vyos_install_stage_03.sh", "/tmp/" ]
+# Copy skel for bash profile
+COPY data/live-build-config/includes.chroot/etc/skel/.bashrc /etc/skel/.bashrc
+# Copy default config
+COPY data/live-build-config/includes.chroot/opt/vyatta/etc/config.boot.default /opt/vyatta/etc/
+
+RUN bash /tmp/vyos_install_stage_03.sh
+
+# Delete installer scripts
+RUN rm -rf /tmp/*
+
+
+# Make changes specific to the container environment
+
+# Tell systemd that we are inside container
+ENV container=docker
+
+# Set proper STOPSIGNAL
+STOPSIGNAL SIGRTMIN+3
+
+# Run VyOS
+CMD [ "/lib/systemd/systemd" ]
+
+# Describe this image
+LABEL maintainer="support@vyos.io" \
+      description="VyOS for Docker" \
+      vendor="Sentrium S.L." \
+      version=${VYOS_VERSION} \
+      io.vyos.build-date=${BUILD_DATE}

docker-vyos/README.md

@@ -0,0 +1,59 @@
+# VyOS as Docker container
+
+VyOS can be run as a Docker container on a Linux host with a compatible kernel.
+
+
+## Building Docker image
+
+To build a Docker image you need to have the whole `vyos-build` repository, not only a folder with Dockerfile, because some files from this repository are required for building.
+Docker image with VyOS can be built on Linux host with the next command:
+
+```
+docker build --compress -f Dockerfile -t vyos:version-`date -u +%Y%m%d%H%M%S` --build-arg BUILD_DATE="`date -u --rfc-3339=seconds`" --build-arg VYOS_VERSION=version --build-arg DEBIAN_VERSION=debian --progress plain ..
+```
+
+Or, if you want to rebuild completely from the scratch (without cache):
+
+```
+docker build --no-cache --pull --compress -f Dockerfile -t vyos:version-`date -u +%Y%m%d%H%M%S` --build-arg BUILD_DATE="`date -u --rfc-3339=seconds`" --build-arg VYOS_VERSION=version --build-arg DEBIAN_VERSION=debian --progress plain ..
+```
+
+> **NOTE:** You must use proper version value for `DEBIAN_VERSION` variable. It can be only `jessie` (for VyOS 1.2) or `buster` (for VyOS 1.3).
+
+## Running Docker image
+
+Docker container with VyOS can be running with the next command:
+
+```
+docker run -v /lib/modules:/lib/modules --privileged --name vyos_inside_docker -d vyos:version
+```
+
+You need to use the `--privileged` flag because the system actively interacts with a host kernel to perform routing operations and tune networking options.
+
+
+**Experimantal:** You can limit access to some system resources with:
+
+```
+docker run --tmpfs /tmp --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /lib/modules:/lib/modules --privileged --name vyos_inside_docker -d vyos:version
+```
+
+## Logging into a VyOS container
+
+To open VyOS CLI, you can use SSH connection to the Docker container or run on host:
+
+```
+docker exec -it vyos_inside_docker su vyos
+```
+
+
+## Troubleshooting
+
+If in VyOS appears IPv6-related errors, for example, it cannot assign an IPv6 for an interface, it is necessary to enable IPv6 support in Docker. This can be done, by editing `/etc/docker/daemon.json`:
+
+```
+{
+    "ipv6": true,
+    "fixed-cidr-v6": "fe80::/64"
+}
+
+```

docker-vyos/vyos_install_common.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Set environment variables
+export DEBIAN_FRONTEND="noninteractive"
+
+# Prepare for further tasks
+function prepare_apt() {
+    # Update packages list
+    apt-get update
+
+    # Install jq (required to easily extract variables from defaults.json)
+    apt-get install -y --no-install-recommends jq gnupg
+
+    # Add VyOS repository to the system
+    local APT_VYOS_MIRROR=`jq --raw-output .vyos_mirror /tmp/defaults.json`
+    local APT_VYOS_BRANCH=`jq --raw-output .vyos_branch /tmp/defaults.json`
+    local APT_ADDITIONAL_REPOS=`jq --raw-output .additional_repositories[] /tmp/defaults.json`
+    local RELEASE_TRAIN=`jq --raw-output .release_train /tmp/defaults.json`
+
+    if [[ "${RELEASE_TRAIN}" == "crux" ]]; then
+        echo -e "deb ${APT_VYOS_MIRROR}/vyos ${APT_VYOS_BRANCH} main\ndeb ${APT_VYOS_MIRROR}/debian ${APT_VYOS_BRANCH} main\n${APT_ADDITIONAL_REPOS}" > /etc/apt/sources.list.d/vyos.list
+    fi
+
+    if [[ "${RELEASE_TRAIN}" == "equuleus" ]]; then
+        echo -e "deb ${APT_VYOS_MIRROR} ${APT_VYOS_BRANCH} main\n${APT_ADDITIONAL_REPOS}" > /etc/apt/sources.list.d/vyos.list
+        # Add backports repository
+        echo -e "deb http://deb.debian.org/debian buster-backports main\ndeb http://deb.debian.org/debian buster-backports non-free" >> /etc/apt/sources.list.d/vyos.list
+    fi
+
+    # Copy additional repositories and preferences, if persented
+    if grep -sq deb /tmp/*.list.chroot; then
+        cat /tmp/*list.chroot >> /etc/apt/sources.list.d/vyos.list
+    fi
+    if grep -sq Package /tmp/*.pref.chroot; then
+        cat /tmp/*pref.chroot >> /etc/apt/preferences.d/10vyos
+    fi
+
+    # Add GPG keys
+    if [[ ! -e /etc/apt/trusted.gpg.d/vyos.gpg ]]; then
+        echo "Adding GPG keys to the system"
+        cat /tmp/*.key.chroot | apt-key --keyring /etc/apt/trusted.gpg.d/vyos.gpg add -
+    fi
+
+    # Update packages list
+    apt-get -o Acquire::Check-Valid-Until=false update
+}
+
+# Cleanup APT after finish
+function cleanup_apt() {
+    # delete jq tool
+    dpkg -P jq
+    # Clear APT cache
+    apt-get clean
+    rm -rf /var/lib/apt/lists/*
+    rm /etc/apt/sources.list.d/vyos.list
+    if [[ -e /etc/apt/preferences.d/10vyos ]]; then
+        rm /etc/apt/preferences.d/10vyos
+    fi
+}
+
+# Filter list elements
+function filter_list() {
+    local list_elements=("${!1}")
+    local filtered_elements=("${!2}")
+    local list_elements_filtered
+
+    for list_element in "${list_elements[@]}"; do
+        local filtered=""
+
+        for filtered_element in "${filtered_elements[@]}"; do
+            if [[ ${list_element} =~ ${filtered_element} ]]; then
+                filtered=True
+            fi
+        done
+
+        if [[ -z "${filtered}" ]]; then
+            list_elements_filtered+=("${list_element}")
+        fi
+    done
+    echo ${list_elements_filtered[@]}
+}

docker-vyos/vyos_install_stage_01.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Stage 1 - install dependencies
+
+# load common functions
+. vyos_install_common.sh
+
+echo "Configuring APT repositories"
+prepare_apt
+
+# Get list of VyOS packages
+vyos_packages=(`apt-cache -i depends vyos-world | awk '/Depends:/ { printf("%s ", $2) }'`)
+
+# Do not analyze packages, which we do not need in Docker
+vyos_packages_filter=(
+    "vyos-intel*"
+    )
+vyos_packages_filtered=("$(filter_list vyos_packages[@] vyos_packages_filter[@])")
+echo "Packages for dependency analyzing: ${vyos_packages_filtered[@]}"
+
+# Get list of all dependencies
+vyos_dependencies=(`apt-get -s install --no-install-recommends ${vyos_packages_filtered[@]} | awk '/Inst/ { printf("%s ", $2) }'`)
+
+# Do not install unnecessary
+ignore_list=(
+    "dosfstools"
+    "parted"
+    "libparted*"
+    "efibootmgr"
+    "gdisk"
+    "grub-*"
+    "laptop-detect"
+    "installation-report"
+    "tshark"
+    "wireshark*"
+    "mdadm"
+    "keepalived"
+    "libheartbeat2"
+    "bmon"
+    "crda"
+    "ipvsadm"
+    "iw"
+    "pptpd"
+    "cluster-glue"
+    "resource-agents"
+    "heartbeat"
+    )
+
+# Get list of packages from VYOS repository
+if ls /var/lib/apt/lists/*vyos*Packages* | grep -q gz$; then
+    arch_cat="zcat"
+fi
+if ls /var/lib/apt/lists/*vyos*Packages* | grep -q lz4$; then
+    arch_cat="lz4cat"
+    echo "Installing lz4"
+    apt-get install -y --no-install-recommends lz4
+fi
+vyos_repo_packages=(`$arch_cat /var/lib/apt/lists/*vyos*Packages* | awk '/Package:/ { printf("%s\n",$2) }'`)
+if [[ "${arch_cat}" == "lz4cat" ]]; then
+    echo "Removing lz4"
+    apt-get purge -y lz4
+fi
+# Add them to ignore list - we do not need anything from VyOS in this layer of image
+ignore_list=("${ignore_list[@]}" "${vyos_repo_packages[@]}")
+
+# Remove every ignore list item from installation list
+vyos_dependencies_filtered=("$(filter_list vyos_dependencies[@] ignore_list[@])")
+
+# Add missed dependencies
+vyos_dependencies_filtered+=(
+    "liburi-perl"
+    "locales"
+    "libcap-ng0"
+    "libnss-myhostname"
+    "dbus"
+    )
+
+echo "Dependencies filtered list: ${vyos_dependencies_filtered[@]}"
+
+# Install delependencies
+echo "Installing dependencies"
+apt-get install -y --no-install-recommends ${vyos_dependencies_filtered[@]}
+
+echo "Deconfiguring APT repositories"
+cleanup_apt
+
+
+exit 0

docker-vyos/vyos_install_stage_02.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Stage 2 - install VyOS packages
+
+# load common functions
+. vyos_install_common.sh
+
+echo "Configuring APT repositories"
+prepare_apt
+
+# Get list of VyOS packages
+vyos_packages=(`apt-cache -i depends vyos-world | awk '/Depends:/ { printf("%s ", $2) }'`)
+
+# Do not analyze packages, which we do not need in Docker
+vyos_packages_filter=(
+    "vyos-intel*"
+    )
+vyos_packages_filtered=("$(filter_list vyos_packages[@] vyos_packages_filter[@])")
+
+# Add missed dependencies
+vyos_packages_filtered+=(
+    "uuid"
+    "jq"
+    )
+
+echo "Packages for installing: ${vyos_packages_filtered[@]}"
+# Install VyOS packages
+echo "Installing VyOS packages"
+apt-get install -y --no-install-recommends ${vyos_packages_filtered[@]}
+
+# Create VyOS version file
+RELEASAE_TRAIN=`jq --raw-output .release_train /tmp/defaults.json`
+apt-cache show vyos-1x | awk -v release_train=${RELEASAE_TRAIN} '{ if ($1 == "Version:") version = $2 } END { build_git = "unknown" ; built_by = "Sentrium S.L." ; built_on = strftime("%F %T UTC", systime(), utc) ; "uuid -v 4" | getline build_uuid ; printf("{\"version\": \"%s\", \"build_git\": \"%s\", \"built_on\": \"%s\", \"built_by\": \"%s\", \"build_uuid\": \"%s\", \"release_train\": \"%s\"}", version, build_git, built_on, built_by, build_uuid, release_train) }' | json_pp > /usr/share/vyos/version.json
+
+# Delete what we do not need inside Docker image (this step makes packages database inconsistent)
+echo "Deleting what is needless in containers"
+dpkg -P --force-depends dosfstools efibootmgr jq gdisk grub-common grub-efi-amd64-bin initscripts installation-report laptop-detect libossp-uuid16 libparted2 libwireshark-data libwireshark5 mdadm parted tshark uuid vyos-qat-kernel-modules wireguard-modules
+dpkg -l | awk '/linux-image-/ { system("dpkg -P --force-depends " $2) }'
+
+# Delete documentation
+rm -rf /usr/share/doc /usr/share/doc-base
+
+echo "Deconfiguring APT repositories"
+cleanup_apt
+
+
+exit 0

docker-vyos/vyos_install_stage_03.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Stage 3 - tune the system
+
+# load common functions
+. vyos_install_common.sh
+
+# Add config partition marker
+touch /opt/vyatta/etc/config/.vyatta_config
+
+# create folder for configuration mounting
+ln -s /opt/vyatta/etc/config /config
+
+# Delete SSH keys
+rm -rf /etc/ssh/ssh_host_*
+
+# Fix FUSE settings
+sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf
+
+# Configure locale
+sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/g' /etc/locale.gen
+dpkg-reconfigure locales
+update-locale LANG=en_US.UTF-8 LC_ALL=C
+
+# Tune bash and environment settings
+echo "source /etc/bash_completion" >> /root/.bashrc
+sed -i 's/set $BASH_COMPLETION_ORIGINAL_V_VALUE/builtin set $BASH_COMPLETION_ORIGINAL_V_VALUE/g' /usr/share/bash-completion/bash_completion
+
+# Run configuration hooks
+echo "Running system configuration hooks"
+hooks_list=(
+    "18-enable-disable_services.chroot"
+    "30-frr-configs.chroot"
+    )
+for hook in ${hooks_list[@]}; do
+    if [[ -e /tmp/hooks/${hook} ]]; then
+        echo "Running ${hook}"
+        /tmp/hooks/${hook}
+    fi
+done
+
+# Delete needless options from CLI
+# CLI_DELETION=(
+#     "/opt/vyatta/share/vyatta-cfg/templates/system/host-name/"
+#     )
+# rm -rf ${CLI_DELETION[@]}
+
+
+exit 0