Merge pull request #550 from dmbaturin/T3664-raw-flavors

build: T3664: add support for building non-ISO flavors

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vyos-1x"]
+	path = vyos-1x
+	url = https://github.com/vyos/vyos-1x

Makefile

@@ -7,12 +7,8 @@ all:
 	@echo "Make what specifically?"
 	@echo "The most common target is 'iso'"
 
-.PHONY: iso
+%:
-.ONESHELL:
+	VYOS_TEMPLATE_DIR=`pwd`/vyos-1x/data/templates/ ./build-vyos-image $*
-iso: clean
-	set -o pipefail
-	@./build-vyos-image iso
-	exit 0
 
 .PHONY: checkiso
 .ONESHELL:

docker/Dockerfile

@@ -104,7 +104,9 @@ RUN apt-get update && apt-get install -y \
       python3-tomli \
       yq \
       debootstrap \
-      live-build
+      live-build \
+      gdisk \
+      dosfstools
 
 # Packages for TPM test
 RUN apt-get update && apt-get install -y swtpm

scripts/image-build/build-vyos-image

@@ -17,11 +17,13 @@
 # File: build-vyos-image
 # Purpose: builds VyOS images using a fork of Debian's live-build tool
 
+# Import Python's standard library modules
 import re
 import os
 import sys
 import uuid
 import glob
+import json
 import shutil
 import getpass
 import platform
@@ -30,19 +32,51 @@ import datetime
 import functools
 import string
 
-import json
+# Import third-party modules
-
 try:
     import tomli
     import jinja2
     import git
+    import psutil
 except ModuleNotFoundError as e:
-    print(f"Cannot load a required library: {e}")
+    print(f"E: Cannot load required library {e}")
-    print("Please make sure the following Python3 modules are installed: tomli jinja2 git")
+    print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil")
 
-# Local modules
+# Initialize Git object from our repository
+try:
+    repo = git.Repo('.', search_parent_directories=True)
+    repo.git.submodule('update', '--init')
+
+    # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
+    build_git = repo.head.object.hexsha[:14]
+    # If somone played around with the source tree and the build is "dirty", mark it
+    if repo.is_dirty():
+        build_git += "-dirty"
+
+    # Retrieve git branch name or current tag
+    # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD)
+    # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name.
+    git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None)
+    if git_branch is None:
+        git_branch = repo.active_branch.name
+except Exception as e:
+    print(f'W: Could not retrieve information from git: {repr(e)}')
+    build_git = ""
+    git_branch = ""
+
+# Add the vyos-1x submodule directory to the Python path
+# so that we can import modules from it.
+VYOS1X_DIR = os.path.join(os.getcwd(), 'vyos-1x/python')
+if not os.path.exists(VYOS1X_DIR):
+    print("E: vyos-1x subdirectory does not exist, did you initialize submodules?")
+else:
+    sys.path.append(VYOS1X_DIR)
+
+# Import local modules from scripts/image-build
+# They rely on modules from vyos-1x
 import utils
 import defaults
+import raw_image
 
 # argparse converts hyphens to underscores,
 # so for lookups in the original options hash we have to convert them back
@@ -108,7 +142,10 @@ if __name__ == "__main__":
            'devscripts',
            'python3-pystache',
            'python3-git',
-           'qemu-utils'
+           'qemu-utils',
+           'gdisk',
+           'kpartx',
+           'dosfstools'
         ],
        'binaries': []
     }
@@ -117,7 +154,7 @@ if __name__ == "__main__":
     try:
         checker = utils.check_system_dependencies(deps)
     except OSError as e:
-        print(e)
+        print(f"E: {e}")
         sys.exit(1)
 
     ## Load the file with default build configuration options
@@ -125,11 +162,18 @@ if __name__ == "__main__":
         with open(defaults.DEFAULTS_FILE, 'rb') as f:
             build_defaults = tomli.load(f)
     except Exception as e:
-        print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e))
+        print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e))
         sys.exit(1)
 
     ## Get a list of available build flavors
-    build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR)))
+    flavor_dir_env = os.getenv("VYOS_BUILD_FLAVORS_DIR")
+    if flavor_dir_env:
+        flavor_dir = flavor_dir_env
+    else:
+        flavor_dir = defaults.BUILD_FLAVORS_DIR
+
+    print(f"I: using build flavors directory {flavor_dir}")
+    build_flavors = [f[0] for f in map(os.path.splitext, os.listdir(flavor_dir)) if (f[1] == ".toml")]
 
     ## Set up the option parser
     ## XXX: It uses values from the default configuration file for its option defaults,
@@ -158,9 +202,8 @@ if __name__ == "__main__":
         else:
             parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk())
 
-    # The debug option is a bit special since it different type is different
+    # Debug options
     parser.add_argument('--debug', help='Enable debug output', action='store_true')
-
     parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true')
 
     # Custom APT entry and APT key options can be used multiple times
@@ -168,6 +211,10 @@ if __name__ == "__main__":
     parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append', default=[])
     parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append', default=[])
 
+    # Options relevant for non-ISO format flavors
+    parser.add_argument('--reuse-iso', help='Use an existing ISO file to build additional image formats', type=str, action='store', default=None)
+    parser.add_argument('--disk-size', help='Disk size for non-ISO image formats', type=int, action='store', default=10)
+
     # Build flavor is a positional argument
     parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store')
 
@@ -181,7 +228,7 @@ if __name__ == "__main__":
         func = get_validator(options, k)
         if func is not None:
             if not func(v):
-                print("{v} is not a valid value for --{o} option".format(o=key, v=v))
+                print("E: {v} is not a valid value for --{o} option".format(o=key, v=v))
                 sys.exit(1)
 
     if not args["build_flavor"]:
@@ -197,7 +244,8 @@ if __name__ == "__main__":
     flavor_config = {}
     build_flavor = args["build_flavor"]
     try:
-        with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f:
+        toml_flavor_file = make_toml_path(flavor_dir, args["build_flavor"])
+        with open(toml_flavor_file, 'rb') as f:
             flavor_config = tomli.load(f)
             pre_build_config = merge_dicts(flavor_config, pre_build_config)
     except FileNotFoundError:
@@ -214,7 +262,7 @@ if __name__ == "__main__":
     # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not,
     # use the --debian-mirror value for both lb and pbuilder bootstrap
     if pre_build_config['debian_mirror'] is None or pre_build_config['debian_security_mirror'] is None:
-        print("debian_mirror and debian_security_mirror cannot be empty")
+        print("E: debian_mirror and debian_security_mirror cannot be empty")
         sys.exit(1)
 
     if pre_build_config['pbuilder_debian_mirror'] is None:
@@ -224,7 +272,7 @@ if __name__ == "__main__":
     # for dev builds it hardly makes any sense
     if pre_build_config['build_type'] == 'development':
         if args['version'] is not None:
-            print("Version can only be set for release builds")
+            print("E: Version can only be set for release builds")
             print("Use --build-type=release option if you want to set version number")
             sys.exit(1)
 
@@ -266,7 +314,25 @@ if __name__ == "__main__":
     if has_nonempty_key(build_config, "architectures"):
         arch = build_config["architecture"]
         if arch in build_config["architectures"]:
-            build_config["packages"] += build_config["architectures"][arch]["packages"]
+            if has_nonempty_key(build_config["architectures"], "packages"):
+                build_config["packages"] += build_config["architectures"][arch]["packages"]
+
+    ## Check if image format is specified,
+    ## else we have no idea what we are actually supposed to build.
+    if not has_nonempty_key(build_config, "image_format"):
+        print("E: image format is not specified in the build flavor file")
+        sys.exit(1)
+
+    ## Add default boot settings if needed
+    if "boot_settings" not in build_config:
+        build_config["boot_settings"] = defaults.boot_settings
+    else:
+        build_config["boot_settings"] = merge_dicts(defaults.default_consolede, build_config["boot_settings"])
+
+    ## Convert the image_format field to a single-item list if it's a scalar
+    ## (like `image_format = "iso"`)
+    if type(build_config["image_format"]) != list:
+        build_config["image_format"] = [ build_config["image_format"] ]
 
     ## Dump the complete config if the user enabled debug mode
     if debug:
@@ -276,100 +342,10 @@ if __name__ == "__main__":
 
     ## Clean up the old build config and set up a fresh copy
     lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR)
-    print(lb_config_dir)
     shutil.rmtree(lb_config_dir, ignore_errors=True)
     shutil.copytree("data/live-build-config/", lb_config_dir)
     os.makedirs(lb_config_dir, exist_ok=True)
 
-    ## Create the version file
-
-    # Create a build timestamp
-    now = datetime.datetime.today()
-    build_timestamp = now.strftime("%Y%m%d%H%M")
-
-    # FIXME: use aware rather than naive object
-    build_date = now.strftime("%a %d %b %Y %H:%M UTC")
-
-    # Assign a (hopefully) unique identifier to the build (UUID)
-    build_uuid = str(uuid.uuid4())
-
-    # Initialize Git object from our repository
-    try:
-        repo = git.Repo('.')
-
-        # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
-        build_git = repo.head.object.hexsha[:14]
-        # If somone played around with the source tree and the build is "dirty", mark it
-        if repo.is_dirty():
-            build_git += "-dirty"
-
-        # Retrieve git branch name or current tag
-        # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD)
-        # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name.
-        git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None)
-        if git_branch is None:
-            git_branch = repo.active_branch.name
-    except Exception as e:
-      exit(f'Could not retrieve information from git: {e}')
-      build_git = ""
-      git_branch = ""
-
-    # Create the build version string
-    if build_config['build_type'] == 'development':
-        try:
-            if not git_branch:
-                raise ValueError("git branch could not be determined")
-
-            # Load the branch to version mapping file
-            with open('data/versions') as f:
-                version_mapping = json.load(f)
-
-            branch_version = version_mapping[git_branch]
-
-            version = "{0}-rolling-{1}".format(branch_version, build_timestamp)
-        except Exception as e:
-            print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e)))
-            version = "999.{0}".format(build_timestamp)
-    else:
-        # Release build, use the version from ./configure arguments
-        version = build_config['version']
-
-    if build_config['build_type'] == 'development':
-        lts_build = False
-    else:
-        lts_build = True
-
-    version_data = {
-        'version': version,
-        'built_by': build_config['build_by'],
-        'built_on': build_date,
-        'build_uuid': build_uuid,
-        'build_git': build_git,
-        'build_branch': git_branch,
-        'release_train': build_config['release_train'],
-        'lts_build': lts_build,
-        'build_comment': build_config['build_comment'],
-        'bugtracker_url': build_config['bugtracker_url'],
-        'documentation_url': build_config['documentation_url'],
-        'project_news_url': build_config['project_news_url'],
-    }
-
-    # Multi line strings needs to be un-indented to not have leading
-    # whitespaces in the resulting file
-    os_release = f"""
-PRETTY_NAME="VyOS {version} ({build_config['release_train']})"
-NAME="VyOS"
-VERSION_ID="{version}"
-VERSION="{version} ({build_config['release_train']})"
-VERSION_CODENAME={build_defaults['debian_distribution']}
-ID=vyos
-BUILD_ID="{build_git}"
-HOME_URL="{build_defaults['website_url']}"
-SUPPORT_URL="{build_defaults['support_url']}"
-BUG_REPORT_URL="{build_defaults['bugtracker_url']}"
-DOCUMENTATION_URL="{build_config['documentation_url']}"
-    """
-
     # Switch to the build directory, this is crucial for the live-build work
     # because the efective build config files etc. are there.
     #
@@ -377,176 +353,269 @@ DOCUMENTATION_URL="{build_config['documentation_url']}"
     # not to the vyos-build repository root.
     os.chdir(defaults.BUILD_DIR)
 
-    chroot_includes_dir = defaults.CHROOT_INCLUDES_DIR
+    iso_file = None
-    binary_includes_dir = defaults.BINARY_INCLUDES_DIR
-    vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos")
-    os.makedirs(vyos_data_dir, exist_ok=True)
-    with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f:
-        json.dump(version_data, f)
-    with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f:
-        json.dump(version_data, f)
 
-    # For backwards compatibility with 'add system image' script from older versions
+    if build_config["reuse_iso"]:
-    # we need a file in the old format so that script can find out the version of the image
+        iso_file = build_config["reuse_iso"]
-    # for upgrade
+    else:
-    os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True)
+        ## Create the version file
-    with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f:
-        print("Version: {0}".format(version), file=f)
 
+        # Create a build timestamp
+        now = datetime.datetime.today()
+        build_timestamp = now.strftime("%Y%m%d%H%M")
 
-    # Define variables that influence to welcome message on boot
+        # FIXME: use aware rather than naive object
-    os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True)
+        build_date = now.strftime("%a %d %b %Y %H:%M UTC")
-    with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f:
-        print(os_release, file=f)
 
-    ## Clean up earlier build state and artifacts
+        # Assign a (hopefully) unique identifier to the build (UUID)
-    print("I: Cleaning the build workspace")
+        build_uuid = str(uuid.uuid4())
-    os.system("lb clean")
-    #iter(lambda p: shutil.rmtree(p, ignore_errors=True),
-    #  ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source'])
-    artifacts = functools.reduce(
-      lambda x, y: x + y,
-      map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf']))
-    iter(os.remove, artifacts)
 
-    ## Create live-build configuration files
+        # Create the build version string
+        if build_config['build_type'] == 'development':
+            try:
+                if not git_branch:
+                    raise ValueError("git branch could not be determined")
 
-    # Add the additional repositories to package lists
+                # Load the branch to version mapping file
-    print("I: Setting up additional APT entries")
+                with open('../data/versions') as f:
-    vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config)
+                    version_mapping = json.load(f)
 
-    apt_file = defaults.VYOS_REPO_FILE
+                branch_version = version_mapping[git_branch]
 
-    if debug:
+                version = "{0}-rolling-{1}".format(branch_version, build_timestamp)
-        print(f"D: Adding these entries to {apt_file}:")
+            except Exception as e:
-        print("\t", vyos_repo_entry)
+                print("W: Could not build a version string specific to git branch, falling back to default: {0}".format(str(e)))
+                version = "999.{0}".format(build_timestamp)
+        else:
+            # Release build, use the version from ./configure arguments
+            version = build_config['version']
 
-    with open(apt_file, 'w') as f:
+        if build_config['build_type'] == 'development':
-        f.write(vyos_repo_entry)
+            lts_build = False
+        else:
+            lts_build = True
 
-    # Add custom APT entries
+        version_data = {
-    if build_config.get('additional_repositories', False):
+            'version': version,
-        build_config['custom_apt_entry'] += build_config['additional_repositories']
+            'built_by': build_config['build_by'],
+            'built_on': build_date,
+            'build_uuid': build_uuid,
+            'build_git': build_git,
+            'build_branch': git_branch,
+            'release_train': build_config['release_train'],
+            'lts_build': lts_build,
+            'build_comment': build_config['build_comment'],
+            'bugtracker_url': build_config['bugtracker_url'],
+            'documentation_url': build_config['documentation_url'],
+            'project_news_url': build_config['project_news_url'],
+        }
+
+        # Multi line strings needs to be un-indented to not have leading
+        # whitespaces in the resulting file
+        os_release = f"""
+    PRETTY_NAME="VyOS {version} ({build_config['release_train']})"
+    NAME="VyOS"
+    VERSION_ID="{version}"
+    VERSION="{version} ({build_config['release_train']})"
+    VERSION_CODENAME={build_defaults['debian_distribution']}
+    ID=vyos
+    BUILD_ID="{build_git}"
+    HOME_URL="{build_defaults['website_url']}"
+    SUPPORT_URL="{build_defaults['support_url']}"
+    BUG_REPORT_URL="{build_defaults['bugtracker_url']}"
+    DOCUMENTATION_URL="{build_config['documentation_url']}"
+        """
+
+        # Reminder: all paths relative to the build dir, not to the repository root
+        chroot_includes_dir = defaults.CHROOT_INCLUDES_DIR
+        binary_includes_dir = defaults.BINARY_INCLUDES_DIR
+        vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos")
+        os.makedirs(vyos_data_dir, exist_ok=True)
+        with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f:
+            json.dump(version_data, f)
+        with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f:
+            json.dump(version_data, f)
+
+        # For backwards compatibility with 'add system image' script from older versions
+        # we need a file in the old format so that script can find out the version of the image
+        # for upgrade
+        os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True)
+        with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f:
+            print("Version: {0}".format(version), file=f)
+
+        # Define variables that influence to welcome message on boot
+        os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True)
+        with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f:
+            print(os_release, file=f)
+
+        ## Clean up earlier build state and artifacts
+        print("I: Cleaning the build workspace")
+        os.system("lb clean")
+        #iter(lambda p: shutil.rmtree(p, ignore_errors=True),
+        #  ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source'])
+        artifacts = functools.reduce(
+          lambda x, y: x + y,
+          map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf']))
+        iter(os.remove, artifacts)
+
+        ## Create the target ISO file path
+        iso_file = "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])
+
+        ## Create live-build configuration files
+
+        # Add the additional repositories to package lists
+        print("I: Setting up additional APT entries")
+        vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config)
+
+        apt_file = defaults.VYOS_REPO_FILE
 
-    if build_config.get('custom_apt_entry', False):
-        custom_apt_file = defaults.CUSTOM_REPO_FILE
-        entries = "\n".join(build_config['custom_apt_entry'])
         if debug:
-            print("D: Adding custom APT entries:")
+            print(f"D: Adding these entries to {apt_file}:")
-            print(entries)
+            print("\t", vyos_repo_entry)
-        with open(custom_apt_file, 'w') as f:
-            f.write(entries)
-            f.write("\n")
 
-    # Add custom APT keys
+        with open(apt_file, 'w') as f:
-    if has_nonempty_key(build_config, 'custom_apt_key'):
+            f.write(vyos_repo_entry)
-        key_dir = defaults.ARCHIVES_DIR
-        for k in build_config['custom_apt_key']:
-            dst_name = '{0}.key.chroot'.format(os.path.basename(k))
-            shutil.copy(k, os.path.join(key_dir, dst_name))
 
-    # Add custom packages
+        # Add custom APT entries
-    if has_nonempty_key(build_config, 'packages'):
+        if build_config.get('additional_repositories', False):
-        package_list_file = defaults.PACKAGE_LIST_FILE
+            build_config['custom_apt_entry'] += build_config['additional_repositories']
-        packages = "\n".join(build_config['packages'])
-        with open (package_list_file, 'w') as f:
-            f.write(packages)
 
-    ## Create includes
+        if build_config.get('custom_apt_entry', False):
-    if has_nonempty_key(build_config, "includes_chroot"):
+            custom_apt_file = defaults.CUSTOM_REPO_FILE
-        for i in build_config["includes_chroot"]:
+            entries = "\n".join(build_config['custom_apt_entry'])
-            file_path = os.path.join(chroot_includes_dir, i["path"])
             if debug:
-                print(f"D: Creating chroot include file: {file_path}")
+                print("D: Adding custom APT entries:")
+                print(entries)
+            with open(custom_apt_file, 'w') as f:
+                f.write(entries)
+                f.write("\n")
+
+        # Add custom APT keys
+        if has_nonempty_key(build_config, 'custom_apt_key'):
+            key_dir = defaults.ARCHIVES_DIR
+            for k in build_config['custom_apt_key']:
+                dst_name = '{0}.key.chroot'.format(os.path.basename(k))
+                shutil.copy(k, os.path.join(key_dir, dst_name))
+
+        # Add custom packages
+        if has_nonempty_key(build_config, 'packages'):
+            package_list_file = defaults.PACKAGE_LIST_FILE
+            packages = "\n".join(build_config['packages'])
+            with open (package_list_file, 'w') as f:
+                f.write(packages)
+
+        ## Create includes
+        if has_nonempty_key(build_config, "includes_chroot"):
+            for i in build_config["includes_chroot"]:
+                file_path = os.path.join(chroot_includes_dir, i["path"])
+                if debug:
+                    print(f"D: Creating chroot include file: {file_path}")
+                os.makedirs(os.path.dirname(file_path), exist_ok=True)
+                with open(file_path, 'w') as f:
+                    f.write(i["data"])
+
+        ## Create the default config
+        ## Technically it's just another includes.chroot entry,
+        ## but it's special enough to warrant making it easier for flavor writers
+        if has_nonempty_key(build_config, "default_config"):
+            file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default")
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
             with open(file_path, 'w') as f:
-                f.write(i["data"])
+                f.write(build_config["default_config"])
 
-    ## Create the default config
+        ## Configure live-build
-    ## Technically it's just another includes.chroot entry,
+        lb_config_tmpl = jinja2.Template("""
-    ## but it's special enough to warrant making it easier for flavor writers
+        lb config noauto \
-    if has_nonempty_key(build_config, "default_config"):
+                --apt-indices false \
-        file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default")
+                --apt-options "--yes -oAPT::Get::allow-downgrades=true" \
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+                --apt-recommends false \
-        with open(file_path, 'w') as f:
+                --architecture {{architecture}} \
-            f.write(build_config["default_config"])
+                --archive-areas {{debian_archive_areas}} \
+                --backports true \
+                --binary-image iso-hybrid \
+                --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
+                --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
+                --bootloaders {{bootloaders}} \
+                --checksums 'sha256 md5' \
+                --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \
+                --debian-installer none \
+                --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \
+                --distribution {{debian_distribution}} \
+                --firmware-binary false \
+                --firmware-chroot false \
+                --iso-application "VyOS" \
+                --iso-publisher "{{build_by}}" \
+                --iso-volume "VyOS" \
+                --linux-flavours {{kernel_flavor}} \
+                --linux-packages linux-image-{{kernel_version}} \
+                --mirror-binary {{debian_mirror}} \
+                --mirror-binary-security {{debian_security_mirror}} \
+                --mirror-bootstrap {{debian_mirror}} \
+                --mirror-chroot {{debian_mirror}} \
+                --mirror-chroot-security {{debian_security_mirror}} \
+                --security true \
+                --updates true
+                "${@}"
+        """)
 
-    ## Configure live-build
+        lb_config_command = lb_config_tmpl.render(build_config)
-    lb_config_tmpl = jinja2.Template("""
-    lb config noauto \
-            --apt-indices false \
-            --apt-options "--yes -oAPT::Get::allow-downgrades=true" \
-            --apt-recommends false \
-            --architecture {{architecture}} \
-            --archive-areas {{debian_archive_areas}} \
-            --backports true \
-            --binary-image iso-hybrid \
-            --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
-            --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
-            --bootloaders {{bootloaders}} \
-            --checksums 'sha256 md5' \
-            --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \
-            --debian-installer none \
-            --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \
-            --distribution {{debian_distribution}} \
-            --firmware-binary false \
-            --firmware-chroot false \
-            --iso-application "VyOS" \
-            --iso-publisher "{{build_by}}" \
-            --iso-volume "VyOS" \
-            --linux-flavours {{kernel_flavor}} \
-            --linux-packages linux-image-{{kernel_version}} \
-            --mirror-binary {{debian_mirror}} \
-            --mirror-binary-security {{debian_security_mirror}} \
-            --mirror-bootstrap {{debian_mirror}} \
-            --mirror-chroot {{debian_mirror}} \
-            --mirror-chroot-security {{debian_security_mirror}} \
-            --security true \
-            --updates true
-            "${@}"
-    """)
 
-    lb_config_command = lb_config_tmpl.render(build_config)
+        ## Pin release for VyOS packages
+        apt_pin = f"""Package: *
+    Pin: release n={build_config['release_train']}
+    Pin-Priority: 600
+    """
 
-    ## Pin release for VyOS packages
+        with open(defaults.VYOS_PIN_FILE, 'w') as f:
-    apt_pin = f"""Package: *
+            f.write(apt_pin)
-Pin: release n={build_config['release_train']}
-Pin-Priority: 600
-"""
 
-    with open(defaults.VYOS_PIN_FILE, 'w') as f:
+        print("I: Configuring live-build")
-        f.write(apt_pin)
 
-    print("I: Configuring live-build")
+        if debug:
+            print("D: live-build configuration command")
+            print(lb_config_command)
 
-    if debug:
+        result = os.system(lb_config_command)
-        print("D: live-build configuration command")
+        if result > 0:
-        print(lb_config_command)
+            print("E: live-build config failed")
+            sys.exit(1)
 
-    result = os.system(lb_config_command)
+        ## In dry-run mode, exit at this point
-    if result > 0:
+        if build_config["dry_run"]:
-        print("E: live-build config failed")
+            print("I: dry-run, not starting image build")
-        sys.exit(1)
+            sys.exit(0)
 
-    ## In dry-run mode, exit at this point
+        ## Add local packages
-    if build_config["dry_run"]:
+        local_packages = glob.glob('../packages/*.deb')
-        print("I: dry-run, not starting image build")
+        if local_packages:
-        sys.exit(0)
+            for f in local_packages:
+                shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f)))
 
-    ## Add local packages
+        ## Build the image
-    local_packages = glob.glob('../packages/*.deb')
+        print("I: Starting image build")
-    if local_packages:
+        if debug:
-        for f in local_packages:
+            print("D: It's not like I'm building this specially for you or anything!")
-            shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f)))
+        res = os.system("lb build 2>&1")
+        if res > 0:
+            sys.exit(res)
 
-    ## Build the image
+        # Copy the image
-    print("I: Starting image build")
+        shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file)
-    if debug:
-        print("D: It's not like I'm building this specially for you or anything!")
-    res = os.system("lb build 2>&1")
-    if res > 0:
-        sys.exit(res)
 
-    # Copy the image
+    # Build additional flavors from the ISO,
-    shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]),
+    # if the flavor calls for them
-      "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"]))
+    if build_config["image_format"] != ["iso"]:
+        raw_image = raw_image.create_raw_image(build_config, iso_file, "tmp/")
+
+        other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"])
+        for f in other_formats:
+            target = f"{os.path.splitext(raw_image)[0]}.{f}"
+            print(f"I: building {f} file {target}")
+            os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}")
+
+        # Some flavors require special procedures that aren't covered by qemu-img
+        # (most notable, the VMware OVA that requires a custom tool to make and sign the image).
+        # Such procedures are executed as post-build hooks.
+        if has_nonempty_key(build_config, "post_build_hook"):
+            hook_path = build_config["post_build_hook"]
+            os.system(f"{hook_path} {raw_image}")

scripts/image-build/defaults.py

@@ -18,6 +18,15 @@
 
 import os
 
+# Default boot settings
+boot_settings: dict[str, str] = {
+    'timeout': '5',
+    'console_type': 'tty',
+    'console_num': '0',
+    'console_speed': '115200',
+    'bootmode': 'normal'
+}
+
 # Relative to the repository directory
 
 BUILD_DIR = 'build'

scripts/image-build/raw_image.py

@@ -0,0 +1,215 @@
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# File: raw_image.py
+# Purpose: Helper functions for building raw images.
+
+import os
+import sys
+import shutil
+import traceback
+
+import vyos.utils.process
+
+SQUASHFS_FILE = 'live/filesystem.squashfs'
+VERSION_FILE = 'version.json'
+
+
+def cmd(command):
+    res = vyos.utils.process.call(command, shell=True)
+    if res > 0:
+        raise OSError(f"Command '{command}' failed")
+
+def mkdir(path):
+    os.makedirs(path, exist_ok=True)
+
+
+class BuildContext:
+    def __init__(self, iso_path, work_dir, debug=False):
+        self.work_dir = work_dir
+        self.iso_path = iso_path
+        self.debug = debug
+        self.loop_device = None
+
+    def __enter__(self):
+        print(f"I: Setting up a raw image build directory in {self.work_dir}")
+
+        self.iso_dir = os.path.join(self.work_dir, "iso")
+        self.squash_dir = os.path.join(self.work_dir, "squash")
+        self.raw_dir = os.path.join(self.work_dir, "raw")
+        self.efi_dir = os.path.join(self.work_dir, "efi")
+
+        # Create mount point directories
+        mkdir(self.iso_dir)
+        mkdir(self.squash_dir)
+        mkdir(self.raw_dir)
+        mkdir(self.efi_dir)
+
+        # Mount the ISO image
+        cmd(f"""mount -t iso9660 -o ro,loop {self.iso_path} {self.iso_dir}""")
+
+        # Mount the SquashFS image
+        cmd(f"""mount -t squashfs -o ro,loop {self.iso_dir}/{SQUASHFS_FILE} {self.squash_dir}""")
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        print(f"I: Tearing down the raw image build environment in {self.work_dir}")
+        cmd(f"""umount {self.squash_dir}/dev/""")
+        cmd(f"""umount {self.squash_dir}/proc/""")
+        cmd(f"""umount {self.squash_dir}/sys/""")
+
+        cmd(f"umount {self.squash_dir}/boot/efi")
+        cmd(f"umount {self.squash_dir}/boot")
+
+        cmd(f"""umount {self.squash_dir}""")
+        cmd(f"""umount {self.iso_dir}""")
+        cmd(f"""umount {self.raw_dir}""")
+        cmd(f"""umount {self.efi_dir}""")
+
+        if self.loop_device:
+            cmd(f"""losetup -d {self.loop_device}""")
+
+def create_disk(path, size):
+    cmd(f"""qemu-img create -f raw "{path}" {size}G""")
+
+def read_version_data(iso_dir):
+    from json import load
+    with open(os.path.join(iso_dir, VERSION_FILE), 'r') as f:
+        data = load(f)
+    return data
+
+def setup_loop_device(con, raw_file):
+    from subprocess import Popen, PIPE, STDOUT
+    from re import match
+    command = f'losetup --show -f {raw_file}'
+    p = Popen(command, stderr=PIPE, stdout=PIPE, stdin=PIPE, shell=True)
+    (stdout, stderr) = p.communicate()
+
+    if p.returncode > 0:
+        raise OSError(f"Could not set up a loop device: {stderr.decode()}")
+
+    con.loop_device = stdout.decode().strip()
+    if con.debug:
+        print(f"I: Using loop device {con.loop_device}")
+
+def mount_image(con):
+    import vyos.system.disk
+
+    from subprocess import Popen, PIPE, STDOUT
+    from re import match
+
+    vyos.system.disk.filesystem_create(con.disk_details.partition['efi'], 'efi')
+    vyos.system.disk.filesystem_create(con.disk_details.partition['root'], 'ext4')
+
+    cmd(f"mount -t ext4 {con.disk_details.partition['root']} {con.raw_dir}")
+    cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}")
+
+def install_image(con, version):
+    from glob import glob
+
+    vyos_dir = os.path.join(con.raw_dir, f'boot/{version}/')
+    mkdir(vyos_dir)
+    mkdir(os.path.join(vyos_dir, 'work/work'))
+    mkdir(os.path.join(vyos_dir, 'rw'))
+
+    shutil.copy(f"{con.iso_dir}/{SQUASHFS_FILE}", f"{vyos_dir}/{version}.squashfs")
+
+    boot_files = glob(f'{con.squash_dir}/boot/*')
+    boot_files = [f for f in boot_files if os.path.isfile(f)]
+
+    for f in boot_files:
+        print(f"I: Copying file {f}")
+        shutil.copy(f, vyos_dir)
+
+    with open(f"{con.raw_dir}/persistence.conf", 'w') as f:
+        f.write("/ union\n")
+
+def setup_grub_configuration(build_config, root_dir) -> None:
+    """Install GRUB configurations
+
+    Args:
+        root_dir (str): a path to the root of target filesystem
+    """
+    from vyos.template import render
+    from vyos.system import grub
+
+    print('I: Installing GRUB configuration files')
+    grub_cfg_main = f'{root_dir}/{grub.GRUB_DIR_MAIN}/grub.cfg'
+    grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}'
+    grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}'
+    grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}'
+    grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}'
+
+    # create new files
+    render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {})
+    grub.common_write(root_dir)
+    grub.vars_write(grub_cfg_vars, build_config["boot_settings"])
+    grub.modules_write(grub_cfg_modules, [])
+    grub.write_cfg_ver(1, root_dir)
+    render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {})
+    render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {})
+
+def install_grub(con, version):
+    from re import match
+    from vyos.system import disk, grub
+
+    # Mount the required virtual filesystems
+    os.makedirs(f"{con.raw_dir}/boot/efi", exist_ok=True)
+    cmd(f"mount --bind /dev {con.squash_dir}/dev")
+    cmd(f"mount --bind /proc {con.squash_dir}/proc")
+    cmd(f"mount --bind /sys {con.squash_dir}/sys")
+
+    cmd(f"mount --bind {con.raw_dir}/boot {con.squash_dir}/boot")
+    cmd(f"mount --bind {con.efi_dir} {con.squash_dir}/boot/efi")
+
+    DIR_DST_ROOT = con.raw_dir
+
+    setup_grub_configuration(con.build_config, DIR_DST_ROOT)
+    # add information about version
+    grub.create_structure(DIR_DST_ROOT)
+    grub.version_add(version, DIR_DST_ROOT)
+    grub.set_default(version, DIR_DST_ROOT)
+    grub.set_console_type(con.build_config["boot_settings"]["console_type"], DIR_DST_ROOT)
+
+    print('I: Installing GRUB to the disk image')
+    grub.install(con.loop_device, f'/boot/', f'/boot/efi', chroot=con.squash_dir)
+
+    # sort inodes (to make GRUB read config files in alphabetical order)
+    grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS}')
+    grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS_VERS}')
+
+
+def create_raw_image(build_config, iso_file, work_dir):
+    from vyos.system.disk import parttable_create
+
+    if not os.path.exists(iso_file):
+        print(f"E: ISO file {iso_file} does not exist in the build directory")
+        sys.exit(1)
+
+    with BuildContext(iso_file, work_dir, debug=True) as con:
+        con.build_config = build_config
+        version_data = read_version_data(con.iso_dir)
+        version = version_data['version']
+        raw_file = f"vyos-{version}.raw"
+        print(f"I: Building raw file {raw_file}")
+        create_disk(raw_file, build_config["disk_size"])
+        setup_loop_device(con, raw_file)
+        disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024)
+        con.disk_details = disk_details
+        mount_image(con)
+        install_image(con, version)
+        install_grub(con, version)
+
+        return raw_file

vyos-1x

@@ -0,0 +1 @@
+Subproject commit c55754fd5fe69a44ea33830d60342b894768af58