vyos/vyos-build
1
0
Fork 0
mirror of https://github.com/vyos/vyos-build.git synced 2025-10-01 20:28:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #550 from dmbaturin/T3664-raw-flavors

Browse Source 
build: T3664: add support for building non-ISO flavors
This commit is contained in:
Christian Breunig 2024-04-20 10:01:19 +02:00 committed by GitHub
commit 671bbd09b7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 551 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "vyos-1x"]
path = vyos-1x
url = https://github.com/vyos/vyos-1x

8
Makefile
View File

@ -7,12 +7,8 @@ all:
@echo "Make what specifically?"
@echo "The most common target is 'iso'"
.PHONY: iso
.ONESHELL:
iso: clean
set -o pipefail
@./build-vyos-image iso
exit 0
%:
VYOS_TEMPLATE_DIR=`pwd`/vyos-1x/data/templates/ ./build-vyos-image $*
.PHONY: checkiso
.ONESHELL:

4
docker/Dockerfile
View File

@ -104,7 +104,9 @@ RUN apt-get update && apt-get install -y \
python3-tomli \
yq \
debootstrap \
live-build
live-build \
gdisk \
dosfstools
# Packages for TPM test
RUN apt-get update && apt-get install -y swtpm

195
scripts/image-build/build-vyos-image
View File

@ -17,11 +17,13 @@
# File: build-vyos-image
# Purpose: builds VyOS images using a fork of Debian's live-build tool
# Import Python's standard library modules
import re
import os
import sys
import uuid
import glob
import json
import shutil
import getpass
import platform
@ -30,19 +32,51 @@ import datetime
import functools
import string
import json
# Import third-party modules
try:
import tomli
import jinja2
import git
import psutil
except ModuleNotFoundError as e:
print(f"Cannot load a required library: {e}")
print("Please make sure the following Python3 modules are installed: tomli jinja2 git")
print(f"E: Cannot load required library {e}")
print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil")
# Local modules
# Initialize Git object from our repository
try:
repo = git.Repo('.', search_parent_directories=True)
repo.git.submodule('update', '--init')
# Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
build_git = repo.head.object.hexsha[:14]
# If somone played around with the source tree and the build is "dirty", mark it
if repo.is_dirty():
build_git += "-dirty"
# Retrieve git branch name or current tag
# Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD)
# Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name.
git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None)
if git_branch is None:
git_branch = repo.active_branch.name
except Exception as e:
print(f'W: Could not retrieve information from git: {repr(e)}')
build_git = ""
git_branch = ""
# Add the vyos-1x submodule directory to the Python path
# so that we can import modules from it.
VYOS1X_DIR = os.path.join(os.getcwd(), 'vyos-1x/python')
if not os.path.exists(VYOS1X_DIR):
print("E: vyos-1x subdirectory does not exist, did you initialize submodules?")
else:
sys.path.append(VYOS1X_DIR)
# Import local modules from scripts/image-build
# They rely on modules from vyos-1x
import utils
import defaults
import raw_image
# argparse converts hyphens to underscores,
# so for lookups in the original options hash we have to convert them back
@ -108,7 +142,10 @@ if __name__ == "__main__":
'devscripts',
'python3-pystache',
'python3-git',
'qemu-utils'
'qemu-utils',
'gdisk',
'kpartx',
'dosfstools'
],
'binaries': []
}
@ -117,7 +154,7 @@ if __name__ == "__main__":
try:
checker = utils.check_system_dependencies(deps)
except OSError as e:
print(e)
print(f"E: {e}")
sys.exit(1)
## Load the file with default build configuration options
@ -125,11 +162,18 @@ if __name__ == "__main__":
with open(defaults.DEFAULTS_FILE, 'rb') as f:
build_defaults = tomli.load(f)
except Exception as e:
print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e))
print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e))
sys.exit(1)
## Get a list of available build flavors
build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR)))
flavor_dir_env = os.getenv("VYOS_BUILD_FLAVORS_DIR")
if flavor_dir_env:
flavor_dir = flavor_dir_env
else:
flavor_dir = defaults.BUILD_FLAVORS_DIR
print(f"I: using build flavors directory {flavor_dir}")
build_flavors = [f[0] for f in map(os.path.splitext, os.listdir(flavor_dir)) if (f[1] == ".toml")]
## Set up the option parser
## XXX: It uses values from the default configuration file for its option defaults,
@ -158,9 +202,8 @@ if __name__ == "__main__":
else:
parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk())
# The debug option is a bit special since it different type is different
# Debug options
parser.add_argument('--debug', help='Enable debug output', action='store_true')
parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true')
# Custom APT entry and APT key options can be used multiple times
@ -168,6 +211,10 @@ if __name__ == "__main__":
parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append', default=[])
parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append', default=[])
# Options relevant for non-ISO format flavors
parser.add_argument('--reuse-iso', help='Use an existing ISO file to build additional image formats', type=str, action='store', default=None)
parser.add_argument('--disk-size', help='Disk size for non-ISO image formats', type=int, action='store', default=10)
# Build flavor is a positional argument
parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store')
@ -181,7 +228,7 @@ if __name__ == "__main__":
func = get_validator(options, k)
if func is not None:
if not func(v):
print("{v} is not a valid value for --{o} option".format(o=key, v=v))
print("E: {v} is not a valid value for --{o} option".format(o=key, v=v))
sys.exit(1)
if not args["build_flavor"]:
@ -197,7 +244,8 @@ if __name__ == "__main__":
flavor_config = {}
build_flavor = args["build_flavor"]
try:
with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f:
toml_flavor_file = make_toml_path(flavor_dir, args["build_flavor"])
with open(toml_flavor_file, 'rb') as f:
flavor_config = tomli.load(f)
pre_build_config = merge_dicts(flavor_config, pre_build_config)
except FileNotFoundError:
@ -214,7 +262,7 @@ if __name__ == "__main__":
# The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not,
# use the --debian-mirror value for both lb and pbuilder bootstrap
if pre_build_config['debian_mirror'] is None or pre_build_config['debian_security_mirror'] is None:
print("debian_mirror and debian_security_mirror cannot be empty")
print("E: debian_mirror and debian_security_mirror cannot be empty")
sys.exit(1)
if pre_build_config['pbuilder_debian_mirror'] is None:
@ -224,7 +272,7 @@ if __name__ == "__main__":
# for dev builds it hardly makes any sense
if pre_build_config['build_type'] == 'development':
if args['version'] is not None:
print("Version can only be set for release builds")
print("E: Version can only be set for release builds")
print("Use --build-type=release option if you want to set version number")
sys.exit(1)
@ -266,8 +314,26 @@ if __name__ == "__main__":
if has_nonempty_key(build_config, "architectures"):
arch = build_config["architecture"]
if arch in build_config["architectures"]:
if has_nonempty_key(build_config["architectures"], "packages"):
build_config["packages"] += build_config["architectures"][arch]["packages"]
## Check if image format is specified,
## else we have no idea what we are actually supposed to build.
if not has_nonempty_key(build_config, "image_format"):
print("E: image format is not specified in the build flavor file")
sys.exit(1)
## Add default boot settings if needed
if "boot_settings" not in build_config:
build_config["boot_settings"] = defaults.boot_settings
else:
build_config["boot_settings"] = merge_dicts(defaults.default_consolede, build_config["boot_settings"])
## Convert the image_format field to a single-item list if it's a scalar
## (like `image_format = "iso"`)
if type(build_config["image_format"]) != list:
build_config["image_format"] = [ build_config["image_format"] ]
## Dump the complete config if the user enabled debug mode
if debug:
import json
@ -276,11 +342,22 @@ if __name__ == "__main__":
## Clean up the old build config and set up a fresh copy
lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR)
print(lb_config_dir)
shutil.rmtree(lb_config_dir, ignore_errors=True)
shutil.copytree("data/live-build-config/", lb_config_dir)
os.makedirs(lb_config_dir, exist_ok=True)
# Switch to the build directory, this is crucial for the live-build work
# because the efective build config files etc. are there.
#
# All directory paths from this point must be relative to BUILD_DIR,
# not to the vyos-build repository root.
os.chdir(defaults.BUILD_DIR)
iso_file = None
if build_config["reuse_iso"]:
iso_file = build_config["reuse_iso"]
else:
## Create the version file
# Create a build timestamp
@ -293,27 +370,6 @@ if __name__ == "__main__":
# Assign a (hopefully) unique identifier to the build (UUID)
build_uuid = str(uuid.uuid4())
# Initialize Git object from our repository
try:
repo = git.Repo('.')
# Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
build_git = repo.head.object.hexsha[:14]
# If somone played around with the source tree and the build is "dirty", mark it
if repo.is_dirty():
build_git += "-dirty"
# Retrieve git branch name or current tag
# Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD)
# Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name.
git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None)
if git_branch is None:
git_branch = repo.active_branch.name
except Exception as e:
exit(f'Could not retrieve information from git: {e}')
build_git = ""
git_branch = ""
# Create the build version string
if build_config['build_type'] == 'development':
try:
@ -321,14 +377,14 @@ if __name__ == "__main__":
raise ValueError("git branch could not be determined")
# Load the branch to version mapping file
with open('data/versions') as f:
with open('../data/versions') as f:
version_mapping = json.load(f)
branch_version = version_mapping[git_branch]
version = "{0}-rolling-{1}".format(branch_version, build_timestamp)
except Exception as e:
print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e)))
print("W: Could not build a version string specific to git branch, falling back to default: {0}".format(str(e)))
version = "999.{0}".format(build_timestamp)
else:
# Release build, use the version from ./configure arguments
@ -357,26 +413,20 @@ if __name__ == "__main__":
# Multi line strings needs to be un-indented to not have leading
# whitespaces in the resulting file
os_release = f"""
PRETTY_NAME="VyOS {version} ({build_config['release_train']})"
NAME="VyOS"
VERSION_ID="{version}"
VERSION="{version} ({build_config['release_train']})"
VERSION_CODENAME={build_defaults['debian_distribution']}
ID=vyos
BUILD_ID="{build_git}"
HOME_URL="{build_defaults['website_url']}"
SUPPORT_URL="{build_defaults['support_url']}"
BUG_REPORT_URL="{build_defaults['bugtracker_url']}"
DOCUMENTATION_URL="{build_config['documentation_url']}"
PRETTY_NAME="VyOS {version} ({build_config['release_train']})"
NAME="VyOS"
VERSION_ID="{version}"
VERSION="{version} ({build_config['release_train']})"
VERSION_CODENAME={build_defaults['debian_distribution']}
ID=vyos
BUILD_ID="{build_git}"
HOME_URL="{build_defaults['website_url']}"
SUPPORT_URL="{build_defaults['support_url']}"
BUG_REPORT_URL="{build_defaults['bugtracker_url']}"
DOCUMENTATION_URL="{build_config['documentation_url']}"
"""
# Switch to the build directory, this is crucial for the live-build work
# because the efective build config files etc. are there.
#
# All directory paths from this point must be relative to BUILD_DIR,
# not to the vyos-build repository root.
os.chdir(defaults.BUILD_DIR)
# Reminder: all paths relative to the build dir, not to the repository root
chroot_includes_dir = defaults.CHROOT_INCLUDES_DIR
binary_includes_dir = defaults.BINARY_INCLUDES_DIR
vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos")
@ -393,7 +443,6 @@ DOCUMENTATION_URL="{build_config['documentation_url']}"
with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f:
print("Version: {0}".format(version), file=f)
# Define variables that influence to welcome message on boot
os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True)
with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f:
@ -409,6 +458,9 @@ DOCUMENTATION_URL="{build_config['documentation_url']}"
map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf']))
iter(os.remove, artifacts)
## Create the target ISO file path
iso_file = "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])
## Create live-build configuration files
# Add the additional repositories to package lists
@ -510,9 +562,9 @@ DOCUMENTATION_URL="{build_config['documentation_url']}"
## Pin release for VyOS packages
apt_pin = f"""Package: *
Pin: release n={build_config['release_train']}
Pin-Priority: 600
"""
Pin: release n={build_config['release_train']}
Pin-Priority: 600
"""
with open(defaults.VYOS_PIN_FILE, 'w') as f:
f.write(apt_pin)
@ -548,5 +600,22 @@ Pin-Priority: 600
sys.exit(res)
# Copy the image
shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]),
"vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"]))
shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file)
# Build additional flavors from the ISO,
# if the flavor calls for them
if build_config["image_format"] != ["iso"]:
raw_image = raw_image.create_raw_image(build_config, iso_file, "tmp/")
other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"])
for f in other_formats:
target = f"{os.path.splitext(raw_image)[0]}.{f}"
print(f"I: building {f} file {target}")
os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}")
# Some flavors require special procedures that aren't covered by qemu-img
# (most notable, the VMware OVA that requires a custom tool to make and sign the image).
# Such procedures are executed as post-build hooks.
if has_nonempty_key(build_config, "post_build_hook"):
hook_path = build_config["post_build_hook"]
os.system(f"{hook_path} {raw_image}")

9
scripts/image-build/defaults.py
View File

@ -18,6 +18,15 @@
import os
# Default boot settings
boot_settings: dict[str, str] = {
'timeout': '5',
'console_type': 'tty',
'console_num': '0',
'console_speed': '115200',
'bootmode': 'normal'
}
# Relative to the repository directory
BUILD_DIR = 'build'

215
scripts/image-build/raw_image.py Normal file
View File

@ -0,0 +1,215 @@
# Copyright (C) 2024 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# File: raw_image.py
# Purpose: Helper functions for building raw images.
import os
import sys
import shutil
import traceback
import vyos.utils.process
SQUASHFS_FILE = 'live/filesystem.squashfs'
VERSION_FILE = 'version.json'
def cmd(command):
res = vyos.utils.process.call(command, shell=True)
if res > 0:
raise OSError(f"Command '{command}' failed")
def mkdir(path):
os.makedirs(path, exist_ok=True)
class BuildContext:
def __init__(self, iso_path, work_dir, debug=False):
self.work_dir = work_dir
self.iso_path = iso_path
self.debug = debug
self.loop_device = None
def __enter__(self):
print(f"I: Setting up a raw image build directory in {self.work_dir}")
self.iso_dir = os.path.join(self.work_dir, "iso")
self.squash_dir = os.path.join(self.work_dir, "squash")
self.raw_dir = os.path.join(self.work_dir, "raw")
self.efi_dir = os.path.join(self.work_dir, "efi")
# Create mount point directories
mkdir(self.iso_dir)
mkdir(self.squash_dir)
mkdir(self.raw_dir)
mkdir(self.efi_dir)
# Mount the ISO image
cmd(f"""mount -t iso9660 -o ro,loop {self.iso_path} {self.iso_dir}""")
# Mount the SquashFS image
cmd(f"""mount -t squashfs -o ro,loop {self.iso_dir}/{SQUASHFS_FILE} {self.squash_dir}""")
return self
def __exit__(self, exc_type, exc_value, exc_tb):
print(f"I: Tearing down the raw image build environment in {self.work_dir}")
cmd(f"""umount {self.squash_dir}/dev/""")
cmd(f"""umount {self.squash_dir}/proc/""")
cmd(f"""umount {self.squash_dir}/sys/""")
cmd(f"umount {self.squash_dir}/boot/efi")
cmd(f"umount {self.squash_dir}/boot")
cmd(f"""umount {self.squash_dir}""")
cmd(f"""umount {self.iso_dir}""")
cmd(f"""umount {self.raw_dir}""")
cmd(f"""umount {self.efi_dir}""")
if self.loop_device:
cmd(f"""losetup -d {self.loop_device}""")
def create_disk(path, size):
cmd(f"""qemu-img create -f raw "{path}" {size}G""")
def read_version_data(iso_dir):
from json import load
with open(os.path.join(iso_dir, VERSION_FILE), 'r') as f:
data = load(f)
return data
def setup_loop_device(con, raw_file):
from subprocess import Popen, PIPE, STDOUT
from re import match
command = f'losetup --show -f {raw_file}'
p = Popen(command, stderr=PIPE, stdout=PIPE, stdin=PIPE, shell=True)
(stdout, stderr) = p.communicate()
if p.returncode > 0:
raise OSError(f"Could not set up a loop device: {stderr.decode()}")
con.loop_device = stdout.decode().strip()
if con.debug:
print(f"I: Using loop device {con.loop_device}")
def mount_image(con):
import vyos.system.disk
from subprocess import Popen, PIPE, STDOUT
from re import match
vyos.system.disk.filesystem_create(con.disk_details.partition['efi'], 'efi')
vyos.system.disk.filesystem_create(con.disk_details.partition['root'], 'ext4')
cmd(f"mount -t ext4 {con.disk_details.partition['root']} {con.raw_dir}")
cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}")
def install_image(con, version):
from glob import glob
vyos_dir = os.path.join(con.raw_dir, f'boot/{version}/')
mkdir(vyos_dir)
mkdir(os.path.join(vyos_dir, 'work/work'))
mkdir(os.path.join(vyos_dir, 'rw'))
shutil.copy(f"{con.iso_dir}/{SQUASHFS_FILE}", f"{vyos_dir}/{version}.squashfs")
boot_files = glob(f'{con.squash_dir}/boot/*')
boot_files = [f for f in boot_files if os.path.isfile(f)]
for f in boot_files:
print(f"I: Copying file {f}")
shutil.copy(f, vyos_dir)
with open(f"{con.raw_dir}/persistence.conf", 'w') as f:
f.write("/ union\n")
def setup_grub_configuration(build_config, root_dir) -> None:
"""Install GRUB configurations
Args:
root_dir (str): a path to the root of target filesystem
"""
from vyos.template import render
from vyos.system import grub
print('I: Installing GRUB configuration files')
grub_cfg_main = f'{root_dir}/{grub.GRUB_DIR_MAIN}/grub.cfg'
grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}'
grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}'
grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}'
grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}'
# create new files
render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {})
grub.common_write(root_dir)
grub.vars_write(grub_cfg_vars, build_config["boot_settings"])
grub.modules_write(grub_cfg_modules, [])
grub.write_cfg_ver(1, root_dir)
render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {})
render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {})
def install_grub(con, version):
from re import match
from vyos.system import disk, grub
# Mount the required virtual filesystems
os.makedirs(f"{con.raw_dir}/boot/efi", exist_ok=True)
cmd(f"mount --bind /dev {con.squash_dir}/dev")
cmd(f"mount --bind /proc {con.squash_dir}/proc")
cmd(f"mount --bind /sys {con.squash_dir}/sys")
cmd(f"mount --bind {con.raw_dir}/boot {con.squash_dir}/boot")
cmd(f"mount --bind {con.efi_dir} {con.squash_dir}/boot/efi")
DIR_DST_ROOT = con.raw_dir
setup_grub_configuration(con.build_config, DIR_DST_ROOT)
# add information about version
grub.create_structure(DIR_DST_ROOT)
grub.version_add(version, DIR_DST_ROOT)
grub.set_default(version, DIR_DST_ROOT)
grub.set_console_type(con.build_config["boot_settings"]["console_type"], DIR_DST_ROOT)
print('I: Installing GRUB to the disk image')
grub.install(con.loop_device, f'/boot/', f'/boot/efi', chroot=con.squash_dir)
# sort inodes (to make GRUB read config files in alphabetical order)
grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS}')
grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS_VERS}')
def create_raw_image(build_config, iso_file, work_dir):
from vyos.system.disk import parttable_create
if not os.path.exists(iso_file):
print(f"E: ISO file {iso_file} does not exist in the build directory")
sys.exit(1)
with BuildContext(iso_file, work_dir, debug=True) as con:
con.build_config = build_config
version_data = read_version_data(con.iso_dir)
version = version_data['version']
raw_file = f"vyos-{version}.raw"
print(f"I: Building raw file {raw_file}")
create_disk(raw_file, build_config["disk_size"])
setup_loop_device(con, raw_file)
disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024)
con.disk_details = disk_details
mount_image(con)
install_image(con, version)
install_grub(con, version)
return raw_file

1
vyos-1x Submodule

@ -0,0 +1 @@
Subproject commit c55754fd5fe69a44ea33830d60342b894768af58