Merge pull request #1 from vyos/master

Update fork

.github/vyos-linter.py

@@ -0,0 +1,177 @@
+import os
+import re
+import ipaddress
+import sys
+import ast
+
+IPV4SEG  = r'(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])'
+IPV4ADDR = r'\b(?:(?:' + IPV4SEG + r'\.){3,3}' + IPV4SEG + r')\b'
+IPV6SEG  = r'(?:(?:[0-9a-fA-F]){1,4})'
+IPV6GROUPS = (
+    r'(?:' + IPV6SEG + r':){7,7}' + IPV6SEG,                  # 1:2:3:4:5:6:7:8
+    r'(?:\s' + IPV6SEG + r':){1,7}:',                           # 1::                                 1:2:3:4:5:6:7::
+    r'(?:' + IPV6SEG + r':){1,6}:' + IPV6SEG,                 # 1::8               1:2:3:4:5:6::8   1:2:3:4:5:6::8
+    r'(?:' + IPV6SEG + r':){1,5}(?::' + IPV6SEG + r'){1,2}',  # 1::7:8             1:2:3:4:5::7:8   1:2:3:4:5::8
+    r'(?:' + IPV6SEG + r':){1,4}(?::' + IPV6SEG + r'){1,3}',  # 1::6:7:8           1:2:3:4::6:7:8   1:2:3:4::8
+    r'(?:' + IPV6SEG + r':){1,3}(?::' + IPV6SEG + r'){1,4}',  # 1::5:6:7:8         1:2:3::5:6:7:8   1:2:3::8
+    r'(?:' + IPV6SEG + r':){1,2}(?::' + IPV6SEG + r'){1,5}',  # 1::4:5:6:7:8       1:2::4:5:6:7:8   1:2::8
+    IPV6SEG + r':(?:(?::' + IPV6SEG + r'){1,6})',             # 1::3:4:5:6:7:8     1::3:4:5:6:7:8   1::8
+    r':(?:(?::' + IPV6SEG + r'){1,7}|:)',                     # ::2:3:4:5:6:7:8    ::2:3:4:5:6:7:8  ::8       ::
+    r'fe80:(?::' + IPV6SEG + r'){0,4}%[0-9a-zA-Z]{1,}',       # fe80::7:8%eth0     fe80::7:8%1  (link-local IPv6 addresses with zone index)
+    r'::(?:ffff(?::0{1,4}){0,1}:){0,1}[^\s:]' + IPV4ADDR,     # ::255.255.255.255  ::ffff:255.255.255.255  ::ffff:0:255.255.255.255 (IPv4-mapped IPv6 addresses and IPv4-translated addresses)
+    r'(?:' + IPV6SEG + r':){1,4}:[^\s:]' + IPV4ADDR,          # 2001:db8:3:4::192.0.2.33  64:ff9b::192.0.2.33 (IPv4-Embedded IPv6 Address)
+)
+IPV6ADDR = '|'.join(['(?:{})'.format(g) for g in IPV6GROUPS[::-1]])  # Reverse rows for greedy match
+
+MAC = r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})'
+
+NUMBER = r"([\s']\d+[\s'])"
+
+
+def lint_mac(cnt, line):
+    mac = re.search(MAC, line, re.I)
+    if mac is not None:
+        mac = mac.group()
+        u_mac = re.search(r'((00)[:-](53)([:-][0-9A-F]{2}){4})', mac, re.I)
+        m_mac = re.search(r'((90)[:-](10)([:-][0-9A-F]{2}){4})', mac, re.I)
+        if u_mac is None and m_mac is None:
+            return (f"Use MAC reserved for Documentation (RFC7042): {mac}", cnt, 'error')
+
+
+def lint_ipv4(cnt, line):
+    ip = re.search(IPV4ADDR, line, re.I)
+    if ip is not None:
+        ip = ipaddress.ip_address(ip.group().strip(' '))
+        # https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Address.is_private
+        if ip.is_private:
+            return None
+        if ip.is_multicast:
+            return None
+        if ip.is_global is False:
+            return None
+        return (f"Use IPv4 reserved for Documentation (RFC 5737) or private Space: {ip}", cnt, 'error')
+
+
+def lint_ipv6(cnt, line):
+    ip = re.search(IPV6ADDR, line, re.I)
+    if ip is not None:
+        ip = ipaddress.ip_address(ip.group().strip(' '))
+        if ip.is_private:
+            return None
+        if ip.is_multicast:
+            return None
+        if ip.is_global is False:
+            return None
+        return (f"Use IPv6 reserved for Documentation (RFC 3849) or private Space: {ip}", cnt, 'error')
+
+
+def lint_AS(cnt, line):
+    number = re.search(NUMBER, line, re.I)
+    if number:
+        pass
+        # find a way to detect AS numbers
+
+
+def lint_linelen(cnt, line):
+    line = line.rstrip()
+    if len(line) > 80:
+        return (f"Line too long: len={len(line)}", cnt, 'warning')
+
+def handle_file_action(filepath):
+    errors = []
+    try:
+        with open(filepath) as fp:
+            line = fp.readline()
+            cnt = 1
+            test_line_lenght = True
+            start_vyoslinter = True
+            indentation = 0
+            while line:
+                # search for ignore linter comments in lines
+                if ".. stop_vyoslinter" in line:
+                    start_vyoslinter = False
+                if ".. start_vyoslinter" in line:
+                    start_vyoslinter = True
+                if start_vyoslinter:
+                    # ignore every '.. code-block::' for line lenght
+                    # rst code-block have its own style in html the format in rst
+                    # and the build page must be the same
+                    if test_line_lenght is False:
+                        if len(line) > indentation:
+                            #print(f"'{line}'")
+                            #print(indentation)
+                            if line[indentation].isspace() is False:
+                                test_line_lenght = True
+
+                    if ".. code-block::" in line:
+                        test_line_lenght = False
+                        indentation = 0
+                        for i in line:
+                            if i.isspace():
+                                indentation = indentation + 1
+                            else:
+                                break
+                    
+                    err_mac = lint_mac(cnt, line.strip())
+                    # disable mac detection for the moment, too many false positives
+                    err_mac = None
+                    err_ip4 = lint_ipv4(cnt, line.strip())
+                    err_ip6 = lint_ipv6(cnt, line.strip())
+                    if test_line_lenght:
+                        err_len = lint_linelen(cnt, line)
+                    else:
+                        err_len = None
+                    if err_mac:
+                        errors.append(err_mac)
+                    if err_ip4:
+                        errors.append(err_ip4)
+                    if err_ip6:
+                        errors.append(err_ip6)
+                    if err_len:
+                        errors.append(err_len)
+                
+                line = fp.readline()
+                cnt += 1
+            
+            # ensure linter was not stop on top and forgot to tun on again
+            if start_vyoslinter == False:
+                errors.append((f"Don't forgett to turn linter back on", cnt, 'error'))
+    finally:
+        fp.close()
+
+    if len(errors) > 0:
+        '''
+        "::{$type} file={$filename},line={$line},col=$column::{$log}"
+        '''
+        print(f"File: {filepath}")
+        for error in errors:
+            print(f"::{error[2]} file={filepath},line={error[1]}::{error[0]}")
+        print('')
+        return False
+
+
+def main():
+    bool_error = True
+    print('start')
+    try:
+        files = ast.literal_eval(sys.argv[1])
+        for file in files:
+                if file[-4:] in [".rst", ".txt"] and "_build" not in file:
+                    if handle_file_action(file) is False:
+                        bool_error = False
+    except Exception as e:
+        for root, dirs, files in os.walk("docs"):
+            path = root.split(os.sep)
+            for file in files:
+                if file[-4:] in [".rst", ".txt"] and "_build" not in path:
+                    fpath = '/'.join(path)
+                    filepath = f"{fpath}/{file}"
+                    if handle_file_action(filepath) is False:
+                        bool_error = False
+
+    return bool_error
+
+
+if __name__ == "__main__":
+    if main() == False:
+        exit(1)

.github/workflows/main.yml

@@ -0,0 +1,32 @@
+name: Linting
+on:
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: File Changes
+      id: file_changes
+      uses: trilom/file-changes-action@v1.2.3
+
+    #- name: Vale
+    #  uses: errata-ai/vale-action@v1.3.0
+    #  with:
+    #    files: '${{ steps.file_changes.outputs.files_modified }}' 
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: run python based linter
+      run: python .github/vyos-linter.py '${{ steps.file_changes.outputs.files_modified }}'
+
+    env:
+      GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    
+    

.github/workflows/submodules.yml

@@ -0,0 +1,62 @@
+name: Update submodule vyos-1x
+on:
+  workflow_dispatch:
+  schedule:
+  # 06:00 UTC on Monday
+    - cron:  '0 6 * * 1'
+jobs:
+  updateVyOS-1x_master:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: ${{ github.repository }}
+      - name: update submodule
+        run: |
+          git submodule status
+          git submodule update --init --force
+          cd docs/_include/vyos-1x
+          git checkout current
+          git pull
+          git submodule status
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+          commit-message: "vyos-1x: update current branch"
+          committer: GitHub <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          title: "vyos-1x: update current branch"
+          body: |
+            Autoupdate vyos-1x submodule
+          branch: update-dependencies-master
+          delete-branch: true
+
+
+  updateVyOS-1x_equuleus:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: ${{ github.repository }}
+          ref: equuleus
+      - name: update submodule
+        run: |
+          git submodule status
+          git submodule update --init --force
+          cd docs/_include/vyos-1x
+          git checkout equuleus
+          git pull
+          git submodule status
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+          commit-message: "vyos-1x: update equuleus branch"
+          committer: GitHub <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          title: "vyos-1x: update equuleus branch"
+          body: |
+            Autoupdate vyos-1x submodule
+          branch: update-dependencies-equuleus
+          delete-branch: true

.gitignore

@@ -1,3 +1,6 @@
+# Sphinx
+_build/
+
 # python virtualenv
 venv/
 ENV/
@@ -12,7 +15,7 @@ ENV/
 
 # python cache files
 *.pyc
-__pychache__
+__pycache__
 
 # dotenv
 .env

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "docs/_include/vyos-1x"]
+	path = docs/_include/vyos-1x
+	url = https://github.com/vyos/vyos-1x
+	branch = current

.readthedocs.yml

@@ -0,0 +1,27 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF
+formats:
+  - pdf
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - requirements: requirements.txt
+
+submodules:
+  include: all

Pipfile

@@ -0,0 +1,16 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+sphinx-rtd-theme = "*"
+docutils = "*"
+lxml = "*"
+sphinx-notfound-page = "*"
+Sphinx = ">=1.4.3"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

README.md

@@ -1,17 +1,38 @@
-Starting with VyOS 1.2 (`crux`) documentation will be migrated from the old wiki
-to ReadTheDocs. Documentation can be accessed via the following URL:
+Starting with VyOS 1.2 (`crux`) our documentation is being migrated from the old wiki
+to ReadTheDocs. Documentation can be accessed via the following URL: https://docs.vyos.io
 
-* https://docs.vyos.io
+Our old WiKi can still be accessed from the
+[Wayback Machine](https://web.archive.org/web/20200225171529/https://wiki.vyos.net/wiki/Main_Page)
 
 # Build
 
 [![Documentation Status](https://readthedocs.org/projects/vyos/badge/?version=latest)](https://docs.vyos.io/en/latest/?badge=latest)
 
+# Versions
+
+Our version follows the very same branching scheme as the VyOS source modules
+itself. We maintain one documentation branch per VyOS release. The default
+branch that contains the most recent VyOS documentation is called `master`
+and matches the latest VyOS release which is 1.4 at the time.
+
+All new documentation enhancements go to the `master` branch. If those changes
+are beneficial for previous VyOS documentation versions they will be
+cherry-picked to the appropriate branch(es).
+
+Post-1.2.0 branches are named after constellations sorted by area from smallest to
+largest. There are 88 of them, here's the
+[complete list](https://en.wikipedia.org/wiki/IAU_designated_constellations_by_area).
+
+* 1.2.x: `crux` (Southern Cross)
+* 1.3.x: `equuleus` (Little Horse)
+* 1.4.x: `sagitta` (Arrow)
+* ...
+
 ## Native
 
-To build the manual run the following commands inside the `docs` folder:
+To build the manual, run the following commands inside the `docs` folder:
 
-* `make html` for a HTML manual
+* `make html` for an HTML manual
 * `make latexpdf` for a LaTeX rendered PDF
 
 Required Debian Packages:
@@ -22,28 +43,28 @@ Required Debian Packages:
 * `sphinx`
 
 ### sphinx
-Debian, requires some extra steps for 
+Debian requires some extra steps for
 installing `sphinx`, `sphinx-autobuild` and `sphinx-rtd-theme` packages:
 
-First ensure that phython2 & phython3 are installed and phython3 is the default:
+First ensure that Python 2 & Python 3 are installed and Python 3 is the default:
 ```bash
 python --version
 ```
 
-Alternatively, to make python3 the default, revise the following line to
-point to the relevant 3.x version of the binary on your system:
+Alternatively, to make Python the default, revise the following line to
+point at the relevant 3.x version of the binary on your system:
 
 ```bash
 sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 0
 ```
 
-Then follow these steps to install sphinx group of packages: 
+Then install the sphinx group of packages:
 ```bash
 sudo apt-get install python3-sphinx
 ```
 
-Although mostly everything uses phython3, But to install this specific 
-package, make sure that pip points to the python2 version of the package manager:
+Although almost everything uses Python 3, in order to install this specific
+package, make sure that pip points at the Python 2 version of the package manager:
 
 ```bash
 python --version
@@ -56,29 +77,29 @@ sudo pip install sphinx-rtd-theme
 ```
 
 
-Do the following to  build the html and start a webeserver:
+Do the following to build the HTML and start a webserver:
 * Run `make livehtml` inside the `docs` folder
 
 Then, to view the live output:
 * Browse to http://localhost:8000
-Note: The changes you save to the sources are represented in the live HTML outout 
+Note: The changes you save to the sources are represented in the live HTML output
 automatically (and almost instantly) without the need to rebuild or refresh manually.
 
 ## Docker
 
-Using our [Dockerfile](docker/Dockerfile) you create your own Docker container
+Using our [Dockerfile](docker/Dockerfile) you can create your own Docker container
 that is used to build a VyOS documentation.
 
 ## Setup
 
-You can either build the container on your own or directly fetch it prebuild
+You can either build the container on your own or directly fetch it prebuilt
 from Dockerhub. If you want to build it for yourself, use the following command.
 
 ```bash
 $ docker build -t vyos/vyos-documentation docker
 ```
 
-### Build documentation
+### Building documentation
 
 If the `vyos/vyos-documentation` container could not be found locally it will be
 automatically fetched from Dockerhub.
@@ -103,7 +124,7 @@ $ docker run --rm -it -v "$(pwd)":/vyos -w /vyos/docs \
   -e GOSU_UID=$(id -u) -e GOSU_GID=$(id -g) vyos/vyos-documentation vale .
 ```
 
-to test a specific file e.g. `clustering.rst`
+to test a specific file (e.g. `clustering.rst`)
 
 ```bash
 $ docker run --rm -it -v "$(pwd)":/vyos -w /vyos/docs -e GOSU_UID=$(id -u) \

ci/vyos-linter.py

@@ -1,117 +0,0 @@
-import os
-import re
-import ipaddress
-
-IPV4SEG  = r'(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])'
-IPV4ADDR = r'(?:(?:' + IPV4SEG + r'\.){3,3}' + IPV4SEG + r')'
-IPV6SEG  = r'(?:(?:[0-9a-fA-F]){1,4})'
-IPV6GROUPS = (
-    r'(?:' + IPV6SEG + r':){7,7}' + IPV6SEG,                  # 1:2:3:4:5:6:7:8
-    r'(?:\s' + IPV6SEG + r':){1,7}:',                           # 1::                                 1:2:3:4:5:6:7::
-    r'(?:' + IPV6SEG + r':){1,6}:' + IPV6SEG,                 # 1::8               1:2:3:4:5:6::8   1:2:3:4:5:6::8
-    r'(?:' + IPV6SEG + r':){1,5}(?::' + IPV6SEG + r'){1,2}',  # 1::7:8             1:2:3:4:5::7:8   1:2:3:4:5::8
-    r'(?:' + IPV6SEG + r':){1,4}(?::' + IPV6SEG + r'){1,3}',  # 1::6:7:8           1:2:3:4::6:7:8   1:2:3:4::8
-    r'(?:' + IPV6SEG + r':){1,3}(?::' + IPV6SEG + r'){1,4}',  # 1::5:6:7:8         1:2:3::5:6:7:8   1:2:3::8
-    r'(?:' + IPV6SEG + r':){1,2}(?::' + IPV6SEG + r'){1,5}',  # 1::4:5:6:7:8       1:2::4:5:6:7:8   1:2::8
-    IPV6SEG + r':(?:(?::' + IPV6SEG + r'){1,6})',             # 1::3:4:5:6:7:8     1::3:4:5:6:7:8   1::8
-    r':(?:(?::' + IPV6SEG + r'){1,7}|:)',                     # ::2:3:4:5:6:7:8    ::2:3:4:5:6:7:8  ::8       ::
-    r'fe80:(?::' + IPV6SEG + r'){0,4}%[0-9a-zA-Z]{1,}',       # fe80::7:8%eth0     fe80::7:8%1  (link-local IPv6 addresses with zone index)
-    r'::(?:ffff(?::0{1,4}){0,1}:){0,1}[^\s:]' + IPV4ADDR,     # ::255.255.255.255  ::ffff:255.255.255.255  ::ffff:0:255.255.255.255 (IPv4-mapped IPv6 addresses and IPv4-translated addresses)
-    r'(?:' + IPV6SEG + r':){1,4}:[^\s:]' + IPV4ADDR,          # 2001:db8:3:4::192.0.2.33  64:ff9b::192.0.2.33 (IPv4-Embedded IPv6 Address)
-)
-IPV6ADDR = '|'.join(['(?:{})'.format(g) for g in IPV6GROUPS[::-1]])  # Reverse rows for greedy match
-
-MAC = r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})'
-
-NUMBER = r"([\s']\d+[\s'])"
-
-
-def lint_mac(cnt, line):
-    mac = re.search(MAC, line, re.I)
-    if mac is not None:
-        mac = mac.group()
-        u_mac = re.search(r'((00)[:-](53)([:-][0-9A-F]{2}){4})', mac, re.I)
-        m_mac = re.search(r'((90)[:-](10)([:-][0-9A-F]{2}){4})', mac, re.I)
-        if u_mac is None and m_mac is None:
-            return f"MAC-Address Error Line {cnt}: {mac}"
-
-
-def lint_ipv4(cnt, line):
-    ip = re.search(IPV4ADDR, line, re.I)
-    if ip is not None:
-        ip = ipaddress.ip_address(ip.group().strip(' '))
-        # https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Address.is_private
-        if ip.is_private is False and ip.is_multicast is False:
-            return f"IPv4 Error Line {cnt}: {ip}"
-
-
-def lint_ipv6(cnt, line):
-    ip = re.search(IPV6ADDR, line, re.I)
-    if ip is not None:
-        ip = ipaddress.ip_address(ip.group().strip(' '))
-        # https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Address.is_private
-        if ip.is_private is False and ip.is_multicast is False:
-            return f"IPv6 Error Line {cnt}: {ip}"
-
-
-def lint_AS(cnt, line):
-    number = re.search(NUMBER, line, re.I)
-    if number:
-        pass
-        # find a way to detect AS numbers
-
-
-def lint_linelen(cnt, line):
-    if len(line) > 80:
-        return f"Line {cnt} too long: len={len(line)}"
-
-
-def handle_file(path, file):
-    errors = []
-    path = '/'.join(path)
-    filepath = f"{path}/{file}"
-    try:
-        with open(filepath) as fp:
-            line = fp.readline()
-            cnt = 1
-            while line:
-                err_mac = lint_mac(cnt, line.strip())
-                err_ip4 = lint_ipv4(cnt, line.strip())
-                err_ip6 = lint_ipv6(cnt, line.strip())
-                err_len = lint_linelen(cnt, line.strip())
-                if err_mac:
-                    errors.append(err_mac)
-                if err_ip4:
-                    errors.append(err_ip4)
-                if err_ip6:
-                    errors.append(err_ip6)
-                if err_len:
-                    errors.append(err_len)
-                line = fp.readline()
-                cnt += 1
-    finally:
-        fp.close()
-
-    if len(errors) > 0:
-        print(f"File: {filepath}")
-        for error in errors:
-            print(error)
-        print('')
-        return False
-
-
-def main():
-    bool_error = True
-    # TODO: path and/or files via cli arg
-    for root, dirs, files in os.walk("../docs"):
-        path = root.split(os.sep)
-        for file in files:
-            if file[-4:] == ".rst":
-                if handle_file(path, file) is False:
-                    bool_error = False
-    return bool_error
-
-
-if __name__ == "__main__":
-    if main() is False:
-        exit(1)

docker/Dockerfile

@@ -33,6 +33,7 @@ RUN pip3 install Sphinx
 RUN pip3 install sphinx-rtd-theme
 RUN pip3 install sphinx-autobuild
 RUN pip3 install sphinx-notfound-page
+RUN pip3 install lxml
 
 
 # Cleanup

docs/.gitignore

@@ -1 +0,0 @@
-_build/

docs/Makefile

@@ -8,9 +8,9 @@ SPHINXPROJ    = VyOS
 SOURCEDIR     = .
 BUILDDIR      = _build
 
-AUTOHOST      =0.0.0.0
-AUTOPORT      =8000
-AUTOOPTS      =--poll
+AUTOHOST      = 0.0.0.0
+AUTOPORT      = 8000
+AUTOOPTS      = --watch .
 
 # Put it first so that "make" without argument is like "make help".
 help:

docs/_ext/testcoverage.py

@@ -0,0 +1,382 @@
+'''
+generate json with all commands from xml for vyos documentation coverage
+
+'''
+
+
+import sys
+import os
+import json
+import re
+import logging
+
+from io import BytesIO
+from lxml import etree as ET
+import shutil
+
+default_constraint_err_msg = "Invalid value"
+validator_dir = ""
+
+
+input_data = [
+    {
+        "kind": "cfgcmd",
+        "input_dir": "_include/vyos-1x/interface-definitions/",
+        "schema_file": "_include/vyos-1x/schema/interface_definition.rng",
+        "files": []
+    },
+    {
+        "kind": "opcmd",
+        "input_dir": "_include/vyos-1x/op-mode-definitions/",
+        "schema_file": "_include/vyos-1x/schema/op-mode-definition.rng",
+        "files": []
+    }
+]
+
+node_data = {
+    'cfgcmd': {},
+    'opcmd': {},
+}
+
+def get_properties(p):
+    props = {}
+    props['valueless'] = False
+
+    try:
+        if p.find("valueless") is not None:
+            props['valueless'] = True
+    except:
+        pass
+
+    if p is None:
+        return props
+
+    # Get the help string
+    try:
+        props["help"] = p.find("help").text
+    except:
+        pass
+
+    # Get value help strings
+    try:
+        vhe = p.findall("valueHelp")
+        vh = []
+        for v in vhe:
+            vh.append( (v.find("format").text, v.find("description").text) )
+        props["val_help"] = vh
+    except:
+        props["val_help"] = []
+
+    # Get the constraint statements
+    error_msg = default_constraint_err_msg
+    # Get the error message if it's there
+    try:
+        error_msg = p.find("constraintErrorMessage").text
+    except:
+        pass
+    
+
+    vce = p.find("constraint")
+    vc = []
+    if vce is not None:
+        # The old backend doesn't support multiple validators in OR mode
+        # so we emulate it
+
+        regexes = []
+        regex_elements = vce.findall("regex")
+        if regex_elements is not None:
+            regexes = list(map(lambda e: e.text.strip(), regex_elements))
+        if "" in regexes:
+            print("Warning: empty regex, node will be accepting any value")
+
+        validator_elements = vce.findall("validator")
+        validators = []
+        if validator_elements is not None:
+            for v in validator_elements:
+                v_name = os.path.join(validator_dir, v.get("name"))
+
+                # XXX: lxml returns None for empty arguments
+                v_argument = None
+                try:
+                    v_argument = v.get("argument")
+                except:
+                    pass
+                if v_argument is None:
+                    v_argument = ""
+
+                validators.append("{0} {1}".format(v_name, v_argument))
+
+
+        regex_args = " ".join(map(lambda s: "--regex \\\'{0}\\\'".format(s), regexes))
+        validator_args = " ".join(map(lambda s: "--exec \\\"{0}\\\"".format(s), validators))
+        validator_script = '${vyos_libexec_dir}/validate-value.py'
+        validator_string = "exec \"{0} {1} {2} --value \\\'$VAR(@)\\\'\"; \"{3}\"".format(validator_script, regex_args, validator_args, error_msg)
+
+        props["constraint"] = validator_string
+
+    # Get the completion help strings
+    try:
+        che = p.findall("completionHelp")
+        ch = ""
+        for c in che:
+            scripts = c.findall("script")
+            paths = c.findall("path")
+            lists = c.findall("list")
+
+            # Current backend doesn't support multiple allowed: tags
+            # so we get to emulate it
+            comp_exprs = []
+            for i in lists:
+                comp_exprs.append("echo \"{0}\"".format(i.text))
+            for i in paths:
+                comp_exprs.append("/bin/cli-shell-api listNodes {0}".format(i.text))
+            for i in scripts:
+                comp_exprs.append("sh -c \"{0}\"".format(i.text))
+            comp_help = " && ".join(comp_exprs)
+            props["comp_help"] = comp_help
+    except:
+        props["comp_help"] = []
+
+    # Get priority
+    try:
+        props["priority"] = p.find("priority").text
+    except:
+        pass
+
+    # Get "multi"
+    if p.find("multi") is not None:
+        props["multi"] = True
+
+    # Get "valueless"
+    if p.find("valueless") is not None:
+        props["valueless"] = True
+
+    return props
+
+def process_node(n, f):
+
+    props_elem = n.find("properties")
+    children = n.find("children")
+    command = n.find("command")
+    children_nodes = []
+    owner = n.get("owner")
+    node_type = n.tag
+
+    name = n.get("name")
+    props = get_properties(props_elem)
+
+    if node_type != "node":
+        if "valueless" not in props.keys():
+            props["type"] = "txt"
+    if node_type == "tagNode":
+        props["tag"] = "True"
+    
+    if node_type == "node" and children is not None:
+        inner_nodes = children.iterfind("*")
+        index_child = 0
+        for inner_n in inner_nodes:
+            children_nodes.append(process_node(inner_n, f))
+            index_child = index_child + 1
+
+    if node_type == "tagNode" and children is not None:
+        inner_nodes = children.iterfind("*")
+        index_child = 0
+        for inner_n in inner_nodes:
+            children_nodes.append(process_node(inner_n, f))
+            index_child = index_child + 1
+    else:
+        # This is a leaf node
+        pass
+    
+    if command is not None:
+        test_command = True
+    else:
+        test_command = False
+    node = {
+        'name': name,
+        'type': node_type,
+        'children': children_nodes,
+        'props': props,
+        'command': test_command,
+        'filename': f
+    }
+    return node
+
+
+
+def create_commands(data, parent_list=[], level=0):
+    result = []
+    command = {
+        'name': [],
+        'help': None,
+        'tag_help': [],
+        'level': level,
+        'no_childs': False,
+        'filename': None
+    }
+    command['filename'] = data['filename']
+    command['name'].extend(parent_list)
+    command['name'].append(data['name'])
+
+    if data['type'] == 'tagNode':
+        command['name'].append("<" + data['name'] + ">")
+
+    if 'val_help' in data['props'].keys():
+        for val_help in data['props']['val_help']:
+            command['tag_help'].append(val_help)
+    
+    if len(data['children']) == 0:
+        command['no_childs'] = True
+    
+    if data['command']:
+        command['no_childs'] = True
+    
+    try:
+        help_text = data['props']['help']
+        command['help'] = re.sub(r"[\n\t]*", "", help_text)
+        
+    except:
+        command['help'] = ""
+    
+    command['valueless'] = data['props']['valueless']
+    
+    if 'children' in data.keys():
+        children_bool = True
+        for child in data['children']:
+            result.extend(create_commands(child, command['name'], level + 1))
+    
+    if command['no_childs']:
+        result.append(command)
+    
+
+
+    return result
+
+
+def include_file(line, input_dir):
+    string = ""
+    if "#include <include" in line.strip():
+        include_filename = line.strip().split('<')[1][:-1]
+        with open(input_dir + include_filename) as ifp:
+            iline = ifp.readline()
+            while iline:
+                string = string + include_file(iline.strip(), input_dir)
+                iline = ifp.readline()
+    else:
+        string = line
+    return string
+
+
+def get_working_commands():
+    for entry in input_data:
+        for (dirpath, dirnames, filenames) in os.walk(entry['input_dir']):
+            entry['files'].extend(filenames)
+            break
+
+        for f in entry['files']:
+
+            string = ""
+            with open(entry['input_dir'] + f) as fp:
+                line = fp.readline()
+                while line:                
+                    string = string + include_file(line.strip(), entry['input_dir'])
+                    line = fp.readline()
+
+            try:
+                xml = ET.parse(BytesIO(bytes(string, 'utf-8')))
+            except Exception as e:
+                print("Failed to load interface definition file {0}".format(f))
+                print(e)
+                sys.exit(1)
+
+            override_defaults(xml)
+            
+            try:
+                relaxng_xml = ET.parse(entry['schema_file'])
+                validator = ET.RelaxNG(relaxng_xml)
+
+                if not validator.validate(xml):
+                    print(validator.error_log)
+                    print("Interface definition file {0} does not match the schema!".format(f))
+                    sys.exit(1)
+            except Exception as e:
+                print("Failed to load the XML schema {0}".format(entry['schema_file']))
+                print(e)
+                sys.exit(1)
+
+            root = xml.getroot()
+            nodes = root.iterfind("*")
+            for n in nodes:
+                node_data[entry['kind']][f] = process_node(n, f)
+
+    # build config tree and sort
+
+    config_tree_new = {
+        'cfgcmd': {},
+        'opcmd': {},
+    }
+
+    for kind in node_data:
+        for entry in node_data[kind]:
+            node_0 = node_data[kind][entry]['name']
+            
+            if node_0 not in config_tree_new[kind].keys():
+                config_tree_new[kind][node_0] = {
+                    'name': node_0,
+                    'type': node_data[kind][entry]['type'],
+                    'props': node_data[kind][entry]['props'],
+                    'children': [],
+                    'command': node_data[kind][entry]['command'],
+                    'filename': node_data[kind][entry]['filename'],
+                }
+            config_tree_new[kind][node_0]['children'].extend(node_data[kind][entry]['children'])
+    
+    result = {
+        'cfgcmd': [],
+        'opcmd': [],
+    }
+    for kind in  config_tree_new:
+        for e in config_tree_new[kind]:
+            result[kind].extend(create_commands(config_tree_new[kind][e]))
+    
+    for cmd in result['cfgcmd']:
+        cmd['cmd'] = " ".join(cmd['name'])
+    for cmd in result['opcmd']:
+        cmd['cmd'] = " ".join(cmd['name'])
+    return result
+
+def override_defaults(xml):
+    root = xml.getroot()
+    defv = {}
+
+    xpath_str = f'//defaultValue'
+    xp = xml.xpath(xpath_str)
+
+    for element in xp:
+        ap = element.xpath('ancestor::*[@name]')
+        defv.setdefault((ap[-1].get("name"), str(ap[:-1])), []).append(element)
+
+    for k, v in defv.items():
+        if len(v) > 1:
+            override_element(v)
+
+def override_element(l: list):
+    if len(l) < 2:
+        return
+
+    # assemble list of leafNodes of overriding defaultValues, for later removal
+    parents = []
+    for el in l[1:]:
+        parents.append(el.getparent())
+
+    # replace element with final override
+    l[0].getparent().replace(l[0], l[-1])
+
+    # remove all but overridden element
+    for el in parents:
+        el.getparent().remove(el)
+
+if __name__ == "__main__":
+    res = get_working_commands()
+    print(json.dumps(res))
+    #print(res['cfgcmd'][0])

docs/_ext/vyos.py

@@ -1,21 +1,42 @@
-from docutils import nodes, utils
+import re
+import json
+import os
+from docutils import io, nodes, utils, statemachine
 from docutils.parsers.rst.roles import set_classes
-from docutils.parsers.rst import Directive
+from docutils.parsers.rst import Directive, directives, states
+
 from sphinx.util.docutils import SphinxDirective
 
+from testcoverage import get_working_commands
+
 
 def setup(app):
 
     app.add_config_value(
         'vyos_phabricator_url',
-        'https://phabricator.vyos.net/', ''
+        'https://phabricator.vyos.net/',
+        'html'
     )
+
+    app.add_config_value(
+        'vyos_working_commands',
+        get_working_commands(),
+        #{"cfgcmd": [], "opcmd": []},
+        'html'
+    )
+    app.add_config_value(
+        'vyos_coverage',
+        {
+            'cfgcmd': [0,len(app.config.vyos_working_commands['cfgcmd'])],
+            'opcmd': [0,len(app.config.vyos_working_commands['opcmd'])]
+        },
+        'html'
+    )
+
     app.add_role('vytask', vytask_role)
     app.add_role('cfgcmd', cmd_role)
     app.add_role('opcmd', cmd_role)
 
-    print(app.config.vyos_phabricator_url)
-
     app.add_node(
         inlinecmd,
         html=(inlinecmd.visit_span, inlinecmd.depart_span),
@@ -42,24 +63,29 @@ def setup(app):
         text=(CmdHeader.visit_div, CmdHeader.depart_div)
     )
     app.add_node(CfgcmdList)
+    app.add_node(CfgcmdListCoverage)
     app.add_directive('cfgcmdlist', CfgcmdlistDirective)
 
     app.add_node(OpcmdList)
+    app.add_node(OpcmdListCoverage)
     app.add_directive('opcmdlist', OpcmdlistDirective)
 
     app.add_directive('cfgcmd', CfgCmdDirective)
     app.add_directive('opcmd', OpCmdDirective)
+    app.add_directive('cmdinclude', CfgInclude)
     app.connect('doctree-resolved', process_cmd_nodes)
 
-
 class CfgcmdList(nodes.General, nodes.Element):
     pass
 
-
 class OpcmdList(nodes.General, nodes.Element):
     pass
 
-import json
+class CfgcmdListCoverage(nodes.General, nodes.Element):
+    pass
+
+class OpcmdListCoverage(nodes.General, nodes.Element):
+    pass
 
 class CmdHeader(nodes.General, nodes.Element):
 
@@ -148,16 +174,177 @@ class inlinecmd(nodes.inline):
         #self.literal_whitespace -= 1
 
 
-class CfgcmdlistDirective(Directive):
+class CfgInclude(SphinxDirective):
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {
+        'var0': str,
+        'var1': str,
+        'var2': str,
+        'var3': str,
+        'var4': str,
+        'var5': str,
+        'var6': str,
+        'var7': str,
+        'var8': str,
+        'var9': str
+    }
+    standard_include_path = os.path.join(os.path.dirname(states.__file__),
+                                         'include')
 
     def run(self):
-        return [CfgcmdList('')]
+        ### Copy from include directive docutils 
+        """Include a file as part of the content of this reST file."""
+        rel_filename, filename = self.env.relfn2path(self.arguments[0])
+        self.arguments[0] = filename
+        self.env.note_included(filename)
+        if not self.state.document.settings.file_insertion_enabled:
+            raise self.warning('"%s" directive disabled.' % self.name)
+        source = self.state_machine.input_lines.source(
+            self.lineno - self.state_machine.input_offset - 1)
+        source_dir = os.path.dirname(os.path.abspath(source))
+        path = directives.path(self.arguments[0])
+        if path.startswith('<') and path.endswith('>'):
+            path = os.path.join(self.standard_include_path, path[1:-1])
+        path = os.path.normpath(os.path.join(source_dir, path))
+        path = utils.relative_path(None, path)
+        path = nodes.reprunicode(path)
+        encoding = self.options.get(
+            'encoding', self.state.document.settings.input_encoding)
+        e_handler=self.state.document.settings.input_encoding_error_handler
+        tab_width = self.options.get(
+            'tab-width', self.state.document.settings.tab_width)
+        try:
+            self.state.document.settings.record_dependencies.add(path)
+            include_file = io.FileInput(source_path=path,
+                                        encoding=encoding,
+                                        error_handler=e_handler)
+        except UnicodeEncodeError:
+            raise self.severe(u'Problems with "%s" directive path:\n'
+                              'Cannot encode input file path "%s" '
+                              '(wrong locale?).' %
+                              (self.name, SafeString(path)))
+        except IOError as error:
+            raise self.severe(u'Problems with "%s" directive path:\n%s.' %
+                      (self.name, error))
+        startline = self.options.get('start-line', None)
+        endline = self.options.get('end-line', None)
+        try:
+            if startline or (endline is not None):
+                lines = include_file.readlines()
+                rawtext = ''.join(lines[startline:endline])
+            else:
+                rawtext = include_file.read()
+        except UnicodeError:
+            raise self.severe(u'Problem with "%s" directive:\n%s' %
+                              (self.name, ErrorString(error)))
+        # start-after/end-before: no restrictions on newlines in match-text,
+        # and no restrictions on matching inside lines vs. line boundaries
+        after_text = self.options.get('start-after', None)
+        if after_text:
+            # skip content in rawtext before *and incl.* a matching text
+            after_index = rawtext.find(after_text)
+            if after_index < 0:
+                raise self.severe('Problem with "start-after" option of "%s" '
+                                  'directive:\nText not found.' % self.name)
+            rawtext = rawtext[after_index + len(after_text):]
+        before_text = self.options.get('end-before', None)
+        if before_text:
+            # skip content in rawtext after *and incl.* a matching text
+            before_index = rawtext.find(before_text)
+            if before_index < 0:
+                raise self.severe('Problem with "end-before" option of "%s" '
+                                  'directive:\nText not found.' % self.name)
+            rawtext = rawtext[:before_index]
+
+        include_lines = statemachine.string2lines(rawtext, tab_width,
+                                                  convert_whitespace=True)
+        if 'literal' in self.options:
+            # Convert tabs to spaces, if `tab_width` is positive.
+            if tab_width >= 0:
+                text = rawtext.expandtabs(tab_width)
+            else:
+                text = rawtext
+            literal_block = nodes.literal_block(rawtext, source=path,
+                                    classes=self.options.get('class', []))
+            literal_block.line = 1
+            self.add_name(literal_block)
+            if 'number-lines' in self.options:
+                try:
+                    startline = int(self.options['number-lines'] or 1)
+                except ValueError:
+                    raise self.error(':number-lines: with non-integer '
+                                     'start value')
+                endline = startline + len(include_lines)
+                if text.endswith('\n'):
+                    text = text[:-1]
+                tokens = NumberLines([([], text)], startline, endline)
+                for classes, value in tokens:
+                    if classes:
+                        literal_block += nodes.inline(value, value,
+                                                      classes=classes)
+                    else:
+                        literal_block += nodes.Text(value, value)
+            else:
+                literal_block += nodes.Text(text, text)
+            return [literal_block]
+        if 'code' in self.options:
+            self.options['source'] = path
+            codeblock = CodeBlock(self.name,
+                                  [self.options.pop('code')], # arguments
+                                  self.options,
+                                  include_lines, # content
+                                  self.lineno,
+                                  self.content_offset,
+                                  self.block_text,
+                                  self.state,
+                                  self.state_machine)
+            return codeblock.run()
+        
+        new_include_lines = []
+        for line in include_lines:
+            for i in range(10):
+                value = self.options.get(f'var{i}','')
+                if value == '':
+                    line = re.sub('\s?{{\s?var' + str(i) + '\s?}}',value,line)
+                else:
+                    line = re.sub('{{\s?var' + str(i) + '\s?}}',value,line)
+            new_include_lines.append(line)
+        self.state_machine.insert_input(new_include_lines, path)
+        return []
+
+
+class CfgcmdlistDirective(Directive):
+    has_content = False
+    required_arguments = 0
+    option_spec = {
+        'show-coverage': directives.flag
+    }
+
+    def run(self):
+        cfglist = CfgcmdList()
+        cfglist['coverage'] = False
+        if 'show-coverage' in self.options:
+            cfglist['coverage'] = True
+        return [cfglist]
 
 
 class OpcmdlistDirective(Directive):
+    has_content = False
+    required_arguments = 0
+    option_spec = {
+        'show-coverage': directives.flag
+    }
 
     def run(self):
-        return [OpcmdList('')]
+        oplist = OpcmdList()
+        oplist['coverage'] = False
+        if 'show-coverage' in self.options:
+            oplist['coverage'] = True
+            
+        return [oplist]
+
 
 
 class CmdDirective(SphinxDirective):
@@ -165,7 +352,8 @@ class CmdDirective(SphinxDirective):
     has_content = True
     custom_class = ''
 
-    def run(self):
+    def run(self):        
+
         title_list = []
         content_list = []
         title_text = ''
@@ -243,7 +431,148 @@ class CfgCmdDirective(CmdDirective):
     custom_class = 'cfg'
 
 
-def process_cmd_node(app, cmd, fromdocname):
+def strip_cmd(cmd, debug=False):
+    if debug:
+        print("")
+        print(cmd)
+    cmd = re.sub('set','',cmd)
+    if debug:
+        print(cmd)
+    #while " | " in cmd:
+    cmd = re.sub('\s+\|\s+','',cmd)
+    if debug:
+        print(cmd)
+    cmd = re.sub('<\S*>','',cmd)
+    if debug:
+        print(cmd)
+    cmd = re.sub('\[\S\]','',cmd)
+    if debug:
+        print(cmd)
+    cmd = re.sub('\s+','',cmd)
+    if debug:
+        print(cmd)
+        print("")
+    return cmd
+
+def build_row(app, fromdocname, rowdata):   
+    row = nodes.row()
+    for cell in rowdata:
+        entry = nodes.entry()
+        row += entry
+        if isinstance(cell, list):
+            for item in cell:
+                if isinstance(item, dict):
+                    entry += process_cmd_node(app, item, fromdocname, '')
+                else:
+                    entry += nodes.paragraph(text=item)
+        elif isinstance(cell, bool):
+            if cell:
+                entry += nodes.paragraph(text="")
+                entry['classes'] = ['coverage-ok']
+            else:
+                entry += nodes.paragraph(text="")
+                entry['classes'] = ['coverage-fail']
+        else:
+            entry += nodes.paragraph(text=cell)
+    return row
+
+
+
+def process_coverage(app, fromdocname, doccmd, xmlcmd, cli_type):
+    coverage_list = {}
+    int_docs = 0
+    int_xml = 0
+    for cmd in doccmd:
+        coverage_item = {
+            'doccmd': None,
+            'xmlcmd': None,
+            'doccmd_item': None,
+            'xmlcmd_item': None,
+            'indocs': False,
+            'inxml': False,
+            'xmlfilename': None
+        }
+        coverage_item['doccmd'] = cmd['cmd']
+        coverage_item['doccmd_item'] = cmd
+        coverage_item['indocs'] = True
+        int_docs += 1
+
+        coverage_list[strip_cmd(cmd['cmd'])] = dict(coverage_item)
+
+    
+    #print(coverage_list.keys())
+    
+    for cmd in xmlcmd:
+        
+        strip = strip_cmd(cmd['cmd'])
+        if strip not in coverage_list.keys():
+            coverage_item = {
+                'doccmd': None,
+                'xmlcmd': None,
+                'doccmd_item': None,
+                'xmlcmd_item': None,
+                'indocs': False,
+                'inxml': False,
+                'xmlfilename': None
+            }
+            coverage_item['xmlcmd'] = cmd['cmd']
+            coverage_item['xmlcmd_item'] = cmd
+            coverage_item['inxml'] = True
+            coverage_item['xmlfilename'] = cmd['filename']
+            int_xml += 1
+            coverage_list[strip] = dict(coverage_item)
+        else:
+            coverage_list[strip]['xmlcmd'] = cmd['cmd']
+            coverage_list[strip]['xmlcmd_item'] = cmd
+            coverage_list[strip]['inxml'] = True
+            coverage_list[strip]['xmlfilename'] = cmd['filename']
+            int_xml += 1
+
+
+    
+
+    table = nodes.table()
+    tgroup = nodes.tgroup(cols=3)
+    table += tgroup
+
+    header = (f'{int_docs}/{len(coverage_list)} in Docs', f'{int_xml}/{len(coverage_list)} in XML', 'Command')
+    colwidths = (1, 1, 8)
+    table = nodes.table()
+    tgroup = nodes.tgroup(cols=len(header))
+    table += tgroup
+    for colwidth in colwidths:
+        tgroup += nodes.colspec(colwidth=colwidth)
+    thead = nodes.thead()
+    tgroup += thead
+    thead += build_row(app, fromdocname, header)
+    tbody = nodes.tbody()
+    tgroup += tbody
+    for entry in sorted(coverage_list):
+        body_text_list = []
+        if coverage_list[entry]['indocs']:
+            body_text_list.append(coverage_list[entry]['doccmd_item'])
+        else:
+            body_text_list.append('Not documented yet')
+
+        if coverage_list[entry]['inxml']:
+            body_text_list.append("------------------")
+            body_text_list.append(str(coverage_list[entry]['xmlfilename']) + ":")
+            body_text_list.append(coverage_list[entry]['xmlcmd'])
+        else:
+            body_text_list.append('Nothing found in XML Definitions')
+
+            
+        tbody += build_row(app, fromdocname, 
+            (
+                coverage_list[entry]['indocs'],
+                coverage_list[entry]['inxml'],
+                body_text_list
+            )
+        )
+
+    return table
+
+def process_cmd_node(app, cmd, fromdocname, cli_type):
     para = nodes.paragraph()
     newnode = nodes.reference('', '')
     innernode = cmd['cmdnode']
@@ -258,21 +587,45 @@ def process_cmd_node(app, cmd, fromdocname):
 
 
 def process_cmd_nodes(app, doctree, fromdocname):
-    env = app.builder.env
+    try:
+        env = app.builder.env
+        
+        for node in doctree.traverse(CfgcmdList):
+            content = []
+            if node.attributes['coverage']:
+                node.replace_self(
+                    process_coverage(
+                        app,
+                        fromdocname,
+                        env.vyos_cfgcmd,
+                        app.config.vyos_working_commands['cfgcmd'],
+                        'cfgcmd'
+                        )
+                    )
+            else:
+                for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']):
+                    content.append(process_cmd_node(app, cmd, fromdocname, 'cfgcmd'))                
+                node.replace_self(content)
+            
+        for node in doctree.traverse(OpcmdList):
+            content = []
+            if node.attributes['coverage']:
+                node.replace_self(
+                    process_coverage(
+                        app,
+                        fromdocname,
+                        env.vyos_opcmd,
+                        app.config.vyos_working_commands['opcmd'],
+                        'opcmd'
+                        )
+                    )
+            else:
+                for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']):
+                    content.append(process_cmd_node(app, cmd, fromdocname, 'opcmd'))
+                node.replace_self(content)
 
-    for node in doctree.traverse(CfgcmdList):
-        content = []
-
-        for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']):
-            content.append(process_cmd_node(app, cmd, fromdocname))
-        node.replace_self(content)
-
-    for node in doctree.traverse(OpcmdList):
-        content = []
-
-        for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']):
-            content.append(process_cmd_node(app, cmd, fromdocname))
-        node.replace_self(content)
+    except Exception as inst:
+        print(inst)
 
 
 def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
@@ -287,4 +640,4 @@ def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
 
 def cmd_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     node = nodes.literal(text, text)
-    return [node], []
+    return [node], []

docs/_include/common-references.txt

@@ -1,3 +1,9 @@
+.. stop_vyoslinter
+
 .. _`accel-ppp`: https://accel-ppp.org/
 .. _`Secure Socket Tunneling Protocol`: https://en.wikipedia.org/wiki/Secure_Socket_Tunneling_Protocol
 .. _Phabricator: https://phabricator.vyos.net/
+.. _802.1ad: https://en.wikipedia.org/wiki/IEEE_802.1ad
+.. _802.1q: https://en.wikipedia.org/wiki/IEEE_802.1Q
+
+.. start_vyoslinter

docs/_include/interface-address-with-dhcp.txt

@@ -0,0 +1,21 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} address <address | dhcp |
+   dhcpv6>
+
+  Configure interface `<interface>` with one or more interface addresses.
+
+  * **address** can be specified multiple times as IPv4 and/or IPv6
+    address, e.g. 192.0.2.1/24 and/or 2001:db8::1/64
+  * **dhcp** interface address is received by DHCP from a DHCP server
+    on this segment.
+  * **dhcpv6** interface address is received by DHCPv6 from a DHCPv6
+    server on this segment.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} address 192.0.2.1/24
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} address 2001:db8::1/64
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcp
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6

docs/_include/interface-address.txt

@@ -0,0 +1,14 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> address <address>
+
+  Configure interface `<interface>` with one or more interface
+  addresses.
+
+  * **address** can be specified multiple times as IPv4 and/or IPv6
+    address, e.g. 192.0.2.1/24 and/or 2001:db8::1/64
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} address 192.0.2.1/24
+    set interfaces {{ var0 }} {{ var1 }} address 2001:db8::1/64

docs/_include/interface-common-with-dhcp.txt

@@ -0,0 +1,21 @@
+.. cmdinclude:: /_include/interface-address-with-dhcp.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-common.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+**DHCP(v6)**
+
+.. cmdinclude:: /_include/interface-dhcp-options.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-dhcpv6-options.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-dhcpv6-prefix-delegation.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}

docs/_include/interface-common-without-dhcp.txt

@@ -0,0 +1,7 @@
+.. cmdinclude:: /_include/interface-address.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-common.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}

docs/_include/interface-common.txt

@@ -0,0 +1,35 @@
+.. cmdinclude:: /_include/interface-description.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-disable.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-disable-flow-control.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-disable-link-detect.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-mac.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-mtu.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-ip.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-ipv6.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}
+
+.. cmdinclude:: /_include/interface-vrf.txt
+  :var0: {{ var0 }}
+  :var1: {{ var1 }}

docs/_include/interface-description.txt

@@ -0,0 +1,11 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} description <description>
+
+  Set a human readable, descriptive alias for this connection. Alias is used by
+  e.g. the :opcmd:`show interfaces` command or SNMP based monitoring tools.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} description 'This is an awesome interface running on VyOS'

docs/_include/interface-dhcp-options.txt

@@ -0,0 +1,50 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcp-options client-id <description>
+
+  :rfc:`2131` states: The client MAY choose to explicitly provide the identifier
+  through the 'client identifier' option. If the client supplies a 'client
+  identifier', the client MUST use the same 'client identifier' in all
+  subsequent messages, and the server MUST use that identifier to identify the
+  client.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcp-options client-id 'foo-bar'
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcp-options host-name <hostname>
+
+  Instead of sending the real system hostname to the DHCP server, overwrite the
+  host-name with this given-value.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcp-options host-name 'VyOS'
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcp-options vendor-class-id <vendor-id>
+
+  The vendor-class-id option can be used to request a specific class of vendor
+  options from the server.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcp-options vendor-class-id 'VyOS'
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcp-options no-default-route
+
+  Only request an address from the DHCP server but do not request a default
+  gateway.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcp-options no-default-route

docs/_include/interface-dhcpv6-options.txt

@@ -0,0 +1,44 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options duid <duid>
+
+  The DHCP unique identifier (DUID) is used by a client to get an IP address
+  from a DHCPv6 server. It has a 2-byte DUID type field, and a variable-length
+  identifier field up to 128 bytes. Its actual length depends on its type. The
+  server compares the DUID with its database and delivers configuration data
+  (address, lease times, DNS servers, etc.) to the client.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} duid '0e:00:00:01:00:01:27:71:db:f0:00:50:56:bf:c5:6d'
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options parameters-only
+
+  This statement specifies dhcp6c to only exchange informational configuration
+  parameters with servers. A list of DNS server addresses is an example of such
+  parameters. This statement is useful when the client does not need stateful
+  configuration parameters such as IPv6 addresses or prefixes.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6-options parameters-only
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options rapid-commit
+
+  When rapid-commit is specified, dhcp6c will include a rapid-commit option in
+  solicit messages and wait for an immediate reply instead of advertisements.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6-options rapid-commit
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options temporary
+
+  Request only a temporary address and not form an IA_NA (Identity Association
+  for Non-temporary Addresses) partnership.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6-options temporary

docs/_include/interface-dhcpv6-prefix-delegation.txt

@@ -0,0 +1,62 @@
+**DHCPv6 Prefix Delegation (PD)**
+
+VyOS 1.3 (equuleus) supports DHCPv6-PD (:rfc:`3633`). DHCPv6 Prefix Delegation
+is supported by most ISPs who provide native IPv6 for consumers on fixed
+networks.
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options pd <id> length <length>
+
+  Some ISPs by default only delegate a /64 prefix. To request for a specific
+  prefix size use this option to request for a bigger delegation for this pd
+  `<id>`. This value is in the range from 32 - 64 so you could request up to a
+  /32 prefix (if your ISP allows this) down to a /64 delegation.
+
+  The default value corresponds to 64.
+
+  To request a /56 prefix from your ISP use:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6-options pd 0 length 56
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options pd <id> interface <delegatee>
+  address <address>
+
+  Specify the interface address used locally on the interfcae where the prefix
+  has been delegated to. ID must be a decimal integer.
+
+  It will be combined with the delegated prefix and the sla-id to form a
+  complete interface address. The default is to use the EUI-64 address of the
+  interface.
+
+  .. stop_vyoslinter
+
+  Example: Delegate a /64 prefix to interface eth8 which will use a local
+  address on this router of ``<prefix>::ffff``, as the address 65534 will
+  correspond to ``ffff`` in hexadecimal notation.
+
+  .. start_vyoslinter
+
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6-options pd 0 interface eth8 address 65534
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} dhcpv6-options pd <id> interface <delegatee> sla-id <id>
+
+  Specify the identifier value of the site-level aggregator (SLA) on the
+  interface. ID must be a decimal number greater then 0 which fits in the
+  length of SLA IDs (see below).
+
+  Example: If ID is 1 and the client is delegated an IPv6 prefix
+  2001:db8:ffff::/48, dhcp6c will combine the two values into a single IPv6
+  prefix, 2001:db8:ffff:1::/64, and will configure the prefix on the specified
+  interface.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} dhcpv6-options pd 0 interface eth8 sla-id 1
+

docs/_include/interface-disable-flow-control.txt

@@ -0,0 +1,23 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+   disable-flow-control
+
+  Ethernet flow control is a mechanism for temporarily stopping the transmission
+  of data on Ethernet family computer networks. The goal of this mechanism is to
+  ensure zero packet loss in the presence of network congestion.
+
+  The first flow control mechanism, the pause frame, was defined by the IEEE
+  802.3x standard.
+
+  A sending station (computer or network switch) may be transmitting data faster
+  than the other end of the link can accept it. Using flow control, the
+  receiving station can signal the sender requesting suspension of
+  transmissions until the receiver catches up.
+
+  Use this command to disable the generation of Ethernet flow control (pause
+  frames).
+
+   Example:
+
+   .. code-block:: none
+
+     set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} disable-flow-control

docs/_include/interface-disable-link-detect.txt

@@ -0,0 +1,13 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} disable-link-detect
+
+  Use this command to direct an interface to not detect any physical state
+  changes on a link, for example, when the cable is unplugged.
+
+  Default is to detects physical link state changes.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} disable-link-detect

docs/_include/interface-disable.txt

@@ -0,0 +1,11 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} disable
+
+  Disable given `<interface>`. It will be placed in administratively down
+  (``A/D``) state.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} disable

docs/_include/interface-eapol.txt

@@ -0,0 +1,37 @@
+:abbr:`EAP (Extensible Authentication Protocol)` over LAN (EAPoL) is a network
+port authentication protocol used in IEEE 802.1X (Port Based Network Access
+Control) developed to give a generic network sign-on to access network
+resources.
+
+EAPoL comes with an identify option. We automatically use the interface MAC
+address as identity parameter.
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} eapol ca-cert-file <file>
+
+  SSL :abbr:`CA (Certificate Authority)` x509 PEM file used afor authentication
+  of the remote side.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} eapol ca-cert-file /config/auth/ca.pem
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} eapol cert-file <file>
+
+  SSL/x509 public certificate file provided by the client to authenticate
+  against the 802.1x system.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} eapol cert-file /config/auth/public.pem
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} eapol key-file <file>
+
+  SSL/x509 private certificate file provided by the client to authenticate
+  against the 802.1x system.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} eapol key-file /config/auth/private.key

docs/_include/interface-ip.txt

@@ -0,0 +1,157 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip arp-cache-timeout
+
+  Once a neighbor has been found, the entry is considered to be valid for at
+  least for this specifc time. An entry's validity will be extended if it
+  receives positive feedback from higher level protocols.
+
+  This defaults to 30 seconds.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip arp-cache-timeout 180
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip disable-arp-filter
+
+  If set the kernel can respond to arp requests with addresses from other
+  interfaces. This may seem wrong but it usually makes sense, because it
+  increases the chance of successful communication. IP addresses are owned by
+  the complete host on Linux, not by particular interfaces. Only for more
+  complex setups like load-balancing, does this behaviour cause problems.
+
+  If not set (default) allows you to have multiple network interfaces on the
+  same subnet, and have the ARPs for each interface be answered based on whether
+  or not the kernel would route a packet from the ARP'd IP out that interface
+  (therefore you must use source based routing for this to work).
+
+  In other words it allows control of which cards (usually 1) will respond to an
+  arp request.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip disable-arp-filter
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip disable-forwarding
+
+  Configure interface-specific Host/Router behaviour. If set, the interface will
+  switch to host mode and IPv6 forwarding will be disabled on this interface.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip disable-forwarding
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip enable-arp-accept
+
+  Define behavior for gratuitous ARP frames who's IP is not already present in
+  the ARP table. If configured create new entries in the ARP table.
+
+  Both replies and requests type gratuitous arp will trigger the ARP table to be
+  updated, if this setting is on.
+
+  If the ARP table already contains the IP address of the gratuitous arp frame,
+  the arp table will be updated regardless if this setting is on or off.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip enable-arp-accept
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip enable-arp-announce
+
+  Define different restriction levels for announcing the local source IP address
+  from IP packets in ARP requests sent on interface.
+
+  Use any local address, configured on any interface if this is not set.
+
+  If configured, try to avoid local addresses that are not in the target's
+  subnet for this interface. This mode is useful when target hosts reachable via
+  this interface require the source IP address in ARP requests to be part of
+  their logical network configured on the receiving interface. When we generate
+  the request we will check all our subnets that include the target IP and will
+  preserve the source address if it is from such subnet. If there is no such
+  subnet we select source address according to the rules for level 2.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip enable-arp-announce
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip enable-arp-ignore
+
+  Define different modes for sending replies in response to received ARP
+  requests that resolve local target IP addresses:
+
+  If configured, reply only if the target IP address is local address configured
+  on the incoming interface.
+
+  If this option is unset (default), reply for any local target IP address,
+  configured on any interface.
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip enable-arp-ignore
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip enable-proxy-arp
+
+  Use this command to enable proxy Address Resolution Protocol (ARP) on this
+  interface. Proxy ARP allows an Ethernet interface to respond with its own
+  :abbr:`MAC (Media Access Control)` address to ARP requests for destination IP
+  addresses on subnets attached to other interfaces on the system. Subsequent
+  packets sent to those destination IP addresses are forwarded appropriately by
+  the system.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ip enable-proxy-arp
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip proxy-arp-pvlan
+
+  Private VLAN proxy arp. Basically allow proxy arp replies back to the same
+  interface (from which the ARP request/solicitation was received).
+
+  This is done to support (ethernet) switch features, like :rfc:`3069`, where
+  the individual ports are NOT allowed to communicate with each other, but they
+  are allowed to talk to the upstream router. As described in :rfc:`3069`, it is
+  possible to allow these hosts to communicate through the upstream router by
+  proxy_arp'ing.
+
+  .. note:: Don't need to be used together with proxy_arp.
+
+  This technology is known by different names:
+
+  - In :rfc:`3069` it is called VLAN Aggregation
+
+  - Cisco and Allied Telesyn call it Private VLAN
+
+  - Hewlett-Packard call it Source-Port filtering or port-isolation
+
+  - Ericsson call it MAC-Forced Forwarding (RFC Draft)
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ip source-validation <strict | loose | disable>
+
+  Enable policy for source validation by reversed path, as specified in
+  :rfc:`3704`. Current recommended practice in :rfc:`3704` is to enable strict
+  mode to prevent IP spoofing from DDos attacks. If using asymmetric routing
+  or other complicated routing, then loose mode is recommended.
+
+  - strict: Each incoming packet is tested against the FIB and if the interface
+    is not the best reverse path the packet check will fail. By default failed
+    packets are discarded.
+
+  - loose: Each incoming packet's source address is also tested against the FIB
+    and if the source address is not reachable via any interface the packet
+    check will fail.
+
+  - disable: No source validation

docs/_include/interface-ipv6.txt

@@ -0,0 +1,55 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ipv6 address autoconf
+
+  :abbr:`SLAAC (Stateless Address Autoconfiguration)` :rfc:`4862`. IPv6 hosts
+  can configure themselves automatically when connected to an IPv6 network using
+  the Neighbor Discovery Protocol via :abbr:`ICMPv6 (Internet Control Message
+  Protocol version 6)` router discovery messages. When first connected to a
+  network, a host sends a link-local router solicitation multicast request for
+  its configuration parameters; routers respond to such a request with a router
+  advertisement packet that contains Internet Layer configuration parameters.
+
+  .. note:: This method automatically disables IPv6 traffic forwarding on the
+    interface in question.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ipv6 address autoconf
+
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ipv6 address eui64 <prefix>
+
+  :abbr:`EUI-64 (64-Bit Extended Unique Identifier)` as specified in
+  :rfc:`4291` allows a host to assign iteslf a unique 64-Bit IPv6 address.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ipv6 address eui64 2001:db8:beef::/64
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ipv6 address no-default-link-local
+
+  Do not assign a link-local IPv6 address to this interface.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ipv6 address no-default-link-local
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} ipv6 disable-forwarding
+
+  Configure interface-specific Host/Router behaviour. If set, the interface will
+  switch to host mode and IPv6 forwarding will be disabled on this interface.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} ipv6 disable-forwarding

docs/_include/interface-mac.txt

@@ -0,0 +1,11 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} mac <xx:xx:xx:xx:xx:xx>
+
+  Configure user defined :abbr:`MAC (Media Access Control)` address on given
+  `<interface>`.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} mac '00:01:02:03:04:05'

docs/_include/interface-mirror.txt

@@ -0,0 +1,34 @@
+SPAN port mirroring can copy the inbound/outbound traffic of the interface to
+the specified interface, usually the interface can be connected to some special
+equipment, such as behavior control system, intrusion detection system and
+traffic collector, and can copy all related traffic from this port
+
+VyOS uses the `mirror` option to configure port mirroring. The configuration
+is divided into 2 different directions. Destination ports should be configured
+for different traffic directions.
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> mirror
+   ingress <monitor-interface>
+
+   Configure port mirroring for `interface` inbound traffic and copy the
+   traffic to `monitor-interface`
+   
+   Example: Mirror the inbound traffic of `{{ var1 }}` port to `{{ var2 }}`
+   
+   .. code-block:: none
+   
+     set interfaces {{ var0 }} {{ var1 }} mirror ingress {{ var2 }} 
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> mirror egress
+   <monitor-interface>
+
+   Configure port mirroring for `interface` outbound traffic and copy the
+   traffic to `monitor-interface`
+   
+   Example: Mirror the outbound traffic of `{{ var1 }}` port to `{{ var2 }}`
+   
+   .. code-block:: none
+   
+     set interfaces {{ var0 }} {{ var1 }} mirror egress {{ var2 }} 
+
+   

docs/_include/interface-mtu.txt

@@ -0,0 +1,11 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} mtu <mtu>
+
+  Configure :abbr:`MTU (Maximum Transmission Unit)` on given `<interface>`. It
+  is the size (in bytes) of the largest ethernet frame sent on this link.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} mtu 9000

docs/_include/interface-vlan-8021ad.txt

@@ -0,0 +1,153 @@
+.. include:: /_include/need_improvement.txt
+
+IEEE 802.1ad_ was an Ethernet networking standard informally known as QinQ as
+an amendment to IEEE standard 802.1q VLAN interfaces as described above.
+802.1ad was incorporated into the base 802.1q_ standard in 2011. The technique
+is also known as provider bridging, Stacked VLANs, or simply QinQ or Q-in-Q.
+"Q-in-Q" can for supported devices apply to C-tag stacking on C-tag (Ethernet
+Type = 0x8100).
+
+The original 802.1q_ specification allows a single Virtual Local Area Network
+(VLAN) header to be inserted into an Ethernet frame. QinQ allows multiple
+VLAN tags to be inserted into a single frame, an essential capability for
+implementing Metro Ethernet network topologies. Just as QinQ extends 802.1Q,
+QinQ itself is extended by other Metro Ethernet protocols.
+
+In a multiple VLAN header context, out of convenience the term "VLAN tag" or
+just "tag" for short is often used in place of "802.1q_ VLAN header". QinQ
+allows multiple VLAN tags in an Ethernet frame; together these tags constitute
+a tag stack. When used in the context of an Ethernet frame, a QinQ frame is a
+frame that has 2 VLAN 802.1q_ headers (double-tagged).
+
+In VyOS the terms ``vif-s`` and ``vif-c`` stand for the ethertype tags that
+are used.
+
+The inner tag is the tag which is closest to the payload portion of the frame.
+It is officially called C-TAG (customer tag, with ethertype 0x8100). The outer
+tag is the one closer/closest to the Ethernet header, its name is S-TAG
+(service tag with Ethernet Type = 0x88a8).
+
+
+.. cmdinclude:: /_include/interface-address-with-dhcp.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-description.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-disable.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-disable-link-detect.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-mac.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-mtu.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-ip.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-ipv6.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-vrf.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+**DHCP(v6)**
+
+.. cmdinclude:: /_include/interface-dhcp-options.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-dhcpv6-options.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. cmdinclude:: /_include/interface-dhcpv6-prefix-delegation.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif-s
+   :var3: <vlan-id>
+   :var4: 1000
+   :var5: vif-c
+   :var6: <vlan-id>
+   :var7: 20
+
+.. include:: /_include/common-references.txt

docs/_include/interface-vlan-8021q.txt

@@ -0,0 +1,120 @@
+IEEE 802.1q_, often referred to as Dot1q, is the networking standard that
+supports virtual LANs (VLANs) on an IEEE 802.3 Ethernet network. The standard
+defines a system of VLAN tagging for Ethernet frames and the accompanying
+procedures to be used by bridges and switches in handling such frames.
+The standard also contains provisions for a quality-of-service prioritization
+scheme commonly known as IEEE 802.1p and defines the
+Generic Attribute Registration Protocol.
+
+Portions of the network which are VLAN-aware (i.e., IEEE 802.1q_ conformant) can
+include VLAN tags. When a frame enters the VLAN-aware portion of the network, a
+tag is added to represent the VLAN membership. Each frame must be
+distinguishable as being within exactly one VLAN. A frame in the VLAN-aware
+portion of the network that does not contain a VLAN tag is assumed to be
+flowing on the native VLAN.
+
+The standard was developed by IEEE 802.1, a working group of the IEEE 802
+standards committee, and continues to be actively revised. One of the notable
+revisions is 802.1Q-2014 which incorporated IEEE 802.1aq
+(Shortest Path Bridging) and much of the IEEE 802.1d standard.
+
+802.1q VLAN interfaces are represented as virtual sub-interfaces in VyOS. The
+term used for this is ``vif``.
+
+.. cfgcmd:: set interfaces {{ var0 }} <interface> vif <vlan-id>
+
+  Create a new VLAN interface on interface `<interface>` using the VLAN number
+  provided via `<vlan-id>`.
+
+  You can create multiple VLAN interfaces on a physical interface. The VLAN ID
+  range is from 0 to 4094.
+
+  .. note:: Only 802.1Q-tagged packets are accepted on Ethernet vifs.
+
+.. cmdinclude:: /_include/interface-address-with-dhcp.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-description.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-disable.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-disable-link-detect.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-mac.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-mtu.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-ip.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-ipv6.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-vrf.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+**DHCP(v6)**
+
+.. cmdinclude:: /_include/interface-dhcp-options.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-dhcpv6-options.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. cmdinclude:: /_include/interface-dhcpv6-prefix-delegation.txt
+   :var0: {{ var0 }}
+   :var1: {{ var1 }}
+   :var2: vif
+   :var3: <vlan-id>
+   :var4: 10
+
+.. include:: /_include/common-references.txt

docs/_include/interface-vrf.txt

@@ -0,0 +1,13 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> {{ var2 }} {{ var3 }}
+  {{ var5 }} {{ var6 }} vrf <vrf>
+
+  Place interface in given VRF instance.
+
+  .. seealso:: There is an entire chapter about how to configure a :ref:`vrf`,
+    please check this for additional information.
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} {{ var2 }} {{ var4 }} {{ var5 }} {{ var7 }} vrf red

docs/_include/interface-xdp.txt

@@ -0,0 +1,27 @@
+.. cfgcmd:: set interfaces {{ var0 }} <interface> xdp
+
+  Enable support for Linux :abbr:`XDP (eXpress Data Path)` on recent 1.3 rolling
+  releases. You must enable it for every interface which should participate in
+  the XDP forwarding.
+
+  XDP is an eBPF based high performance data path merged in the Linux kernel
+  since version 4.8. The idea behind XDP is to add an early hook in the RX path
+  of the kernel, and let a user supplied eBPF program decide the fate of the
+  packet. The hook is placed in the NIC driver just after the interrupt
+  processing, and before any memory allocation needed by the network stack
+  itself, because memory allocation can be an expensive operation.
+
+  .. warning:: This is highly experimental!
+
+  .. note:: Enabling this feature will break any form of NAT or Firewalling on
+    this interface, as XDP is handled way earlier in the driver then iptables/
+    nftables.
+
+  Enabling this feature will only load the XDP router code as described here:
+  https://blog.apnic.net/2020/04/30/how-to-build-an-xdp-based-bgp-peering-router/
+
+  Example:
+
+  .. code-block:: none
+
+    set interfaces {{ var0 }} {{ var1 }} xdp

docs/_include/need_improvement.txt

@@ -8,8 +8,9 @@
    <p class="admonition-title">Call for Contributions</p>
 
 
-This page needs improvements, examples and explanations.
-Please take a look at the Contributing Guide for :ref:`documentation`.
+This section needs improvements, examples and explanations.
+
+Please take a look at the Contributing Guide for our :ref:`documentation`.
 
 .. raw:: html
 

docs/_include/vyos-1x

@@ -0,0 +1 @@
+Subproject commit 0dd41096f14771ffa476f52793308bffac51b59a

docs/_static/css/custom.css

@@ -10,8 +10,45 @@ span.cfgcmd {
     font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace;
 }
 
-.opcmd-heading,
+span.cfgcmd:before {
+  content: "#";
+  margin-right: 0px;
+}
+
+td p a.cmdlink span.cfgcmd:before,
+td p a.cmdlink span.opcmd:before  {
+  content: "";
+}
+
+td p a.cmdlink,
+td p a.cmdlink {
+  margin-left: 0px;
+}
+
+tr td p {
+  margin-bottom:0px
+ }
+
+span.opcmd:before {
+  content: "$";
+  margin-right: 0px;
+}
+
 .cfgcmd-heading {
+    display: inline-block;
+    margin: 6px 0;
+    font-size: 90%;
+    line-height: normal;
+    background: #f0d481;
+    color: #2980B9;
+    border-top: solid 3px #6ab0de;
+    border-top-width: 3px;
+    border-top-style: solid;
+    border-top-color: #FF9302;
+    padding: 6px;
+}
+
+.opcmd-heading {
     display: inline-block;
     margin: 6px 0;
     font-size: 90%;
@@ -34,7 +71,7 @@ span.cfgcmd {
 
 
 .cfgcmd-heading .cmdlink:after,
-.opcmd-heading .cmdlink:after {
+.opcmd-heading .cmdlink:after{
     content: "";
     font-family: FontAwesome
 }
@@ -97,21 +134,44 @@ a.cmdlink span:hover{
 }
 
 .wy-side-nav-search {
-  background-color : #FF0000 !important;
+  background-color : #ffffff !important;
 }
 
 .wy-side-nav-search img {
-  background-color : #FF0000 !important;
+  background-color : #ffffff !important;
 }
 
 .wy-side-nav-search > div.version {
-  color : rgba(255, 255, 255, 0.7) !important;
+  color : #000000 !important;
+}
+
+.wy-side-nav-search>a,
+.wy-side-nav-search .wy-dropdown>a {
+ color:#000000;
+ font-size:100%;
+ font-weight:bold;
+ display:inline-block;
+ padding:4px 6px;
+ margin-bottom:.809em
 }
 
 .wy-nav-top {
-  background-color : #FF0000 !important;
+  background-color : #ffffff !important;
 }
 
 .wy-nav-top img {
-  background-color : #FF0000 !important;
+  background-color : #000000 !important;
 }
+
+.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td.coverage-ok,
+.rst-content table.docutils td.coverage-ok {
+  background-color: green;
+  color: black;
+}
+
+
+.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td.coverage-fail,
+.rst-content table.docutils td.coverage-fail {
+  background-color: red;
+  color: black;
+}

docs/appendix/examples/dmvpn.rst

@@ -1,109 +0,0 @@
-.. _examples-dmvpn:
-
-#########
-DMVPN Hub
-#########
-
-General infomration can be found in the :ref:`vpn-dmvpn` chapter.
-
-Configuration
-=============
-
-VyOS Hub
---------
-
-.. code-block:: none
-
-  set interfaces tunnel tun100 address '172.16.253.134/29'
-  set interfaces tunnel tun100 encapsulation 'gre'
-  set interfaces tunnel tun100 local-ip '203.0.113.44'
-  set interfaces tunnel tun100 multicast 'enable'
-  set interfaces tunnel tun100 parameters ip key '1'
-
-  set protocols nhrp tunnel tun100 cisco-authentication <secret>
-  set protocols nhrp tunnel tun100 holding-time '300'
-  set protocols nhrp tunnel tun100 multicast 'dynamic'
-  set protocols nhrp tunnel tun100 redirect
-  set protocols nhrp tunnel tun100 shortcut
-
-  set vpn ipsec esp-group ESP-HUB compression 'disable'
-  set vpn ipsec esp-group ESP-HUB lifetime '1800'
-  set vpn ipsec esp-group ESP-HUB mode 'tunnel'
-  set vpn ipsec esp-group ESP-HUB pfs 'dh-group2'
-  set vpn ipsec esp-group ESP-HUB proposal 1 encryption 'aes256'
-  set vpn ipsec esp-group ESP-HUB proposal 1 hash 'sha1'
-  set vpn ipsec esp-group ESP-HUB proposal 2 encryption '3des'
-  set vpn ipsec esp-group ESP-HUB proposal 2 hash 'md5'
-  set vpn ipsec ike-group IKE-HUB ikev2-reauth 'no'
-  set vpn ipsec ike-group IKE-HUB key-exchange 'ikev1'
-  set vpn ipsec ike-group IKE-HUB lifetime '3600'
-  set vpn ipsec ike-group IKE-HUB proposal 1 dh-group '2'
-  set vpn ipsec ike-group IKE-HUB proposal 1 encryption 'aes256'
-  set vpn ipsec ike-group IKE-HUB proposal 1 hash 'sha1'
-  set vpn ipsec ike-group IKE-HUB proposal 2 dh-group '2'
-  set vpn ipsec ike-group IKE-HUB proposal 2 encryption 'aes128'
-  set vpn ipsec ike-group IKE-HUB proposal 2 hash 'sha1'
-  set vpn ipsec ipsec-interfaces interface 'eth0'
-
-  set vpn ipsec profile NHRPVPN authentication mode 'pre-shared-secret'
-  set vpn ipsec profile NHRPVPN authentication pre-shared-secret <secret>
-  set vpn ipsec profile NHRPVPN bind tunnel 'tun100'
-  set vpn ipsec profile NHRPVPN esp-group 'ESP-HUB'
-  set vpn ipsec profile NHRPVPN ike-group 'IKE-HUB'
-
-Cisco IOS Spoke
----------------
-
-This example is verified with a Cisco 2811 platform running IOS 15.1(4)M9 and
-VyOS 1.1.7 (helium) up to VyOS 1.2 (Crux).
-
-.. code-block:: none
-
-  Cisco IOS Software, 2800 Software (C2800NM-ADVENTERPRISEK9-M), Version 15.1(4)M9, RELEASE SOFTWARE (fc3)
-  Technical Support: http://www.cisco.com/techsupport
-  Copyright (c) 1986-2014 by Cisco Systems, Inc.
-  Compiled Fri 12-Sep-14 10:45 by prod_rel_team
-
-  ROM: System Bootstrap, Version 12.3(8r)T7, RELEASE SOFTWARE (fc1)
-
-Use this configuration on your Cisco device:
-
-.. code-block:: none
-
-  crypto pki token default removal timeout 0
-  crypto keyring DMVPN
-    pre-shared-key address 198.51.100.2 key <secretkey>
-  !
-  crypto isakmp policy 10
-   encr aes 256
-   authentication pre-share
-   group 2
-  !
-  crypto isakmp invalid-spi-recovery
-  crypto isakmp keepalive 30 30 periodic
-  crypto isakmp profile DMVPN
-     keyring DMVPN
-     match identity address 203.0.113.44 255.255.255.255
-  !
-  crypto ipsec transform-set DMVPN-AES256 esp-aes 256 esp-sha-hmac
-   mode transport
-  !
-  crypto ipsec profile DMVPN
-   set security-association idle-time 720
-   set transform-set DMVPN-AES256
-   set isakmp-profile DMVPN
-  !
-  interface Tunnel10
-   description Tunnel to DMVPN HUB
-   ip address 172.16.253.129 255.255.255.248
-   no ip redirects
-   ip nhrp authentication <nhrp secret key>
-   ip nhrp map multicast 203.0.113.44
-   ip nhrp map 172.16.253.134 203.0.113.44
-   ip nhrp network-id 1
-   ip nhrp holdtime 600
-   ip nhrp nhs 172.16.253.134
-   ip nhrp registration timeout 75
-   tunnel source Dialer1
-   tunnel mode gre multipoint
-   tunnel key 1

docs/appendix/release-notes.rst

@@ -1,305 +0,0 @@
-.. _release-notes:
-
-#############
-Release Notes
-#############
-
-1.2 (Crux)
-==========
-
-1.2.5
------
-
-1.2.5 is a maintenance release made in April 2020.
-
-Resolved issues
-^^^^^^^^^^^^^^^
-
-* :vytask:`1020` OSPF Stops distributing default route after a while
-* :vytask:`1228` pppoe default-route force option not working (Rel 1.2.0-rc11)
-* :vytask:`1301` bgp peer-groups don't work when "no-ipv4-unicast" is enabled.
-* :vytask:`1341` Adding rate-limiter for pppoe server users
-* :vytask:`1376` Incorrect DHCP lease counting
-* :vytask:`1392` Large firewall rulesets cause the system to lose configuration and crash at startup
-* :vytask:`1416` 2 dhcp server run in failover mode can't sync hostname with each other
-* :vytask:`1452` accel-pppoe - add vendor option to shaper
-* :vytask:`1490` BGP configuration (is lost|not applied) when updating 1.1.8 -> 1.2.1
-* :vytask:`1780` Adding ipsec ike closeaction
-* :vytask:`1803` Unbind NTP while it's not requested...
-* :vytask:`1821` "authentication mode radius" has no effect for PPPoE server
-* :vytask:`1827` Increase default gc_thresh
-* :vytask:`1828` Missing completion helper for "set system syslog host 192.0.2.1 facility all protocol"
-* :vytask:`1832` radvd adding feature DNSSL branch.example.com example.com to existing package
-* :vytask:`1837` PPPoE unrecognized option 'replacedefaultroute'
-* :vytask:`1851` wireguard - changing the pubkey on an existing peer seems to destroy the running config.
-* :vytask:`1858` l2tp: Delete depricated outside-nexthop and add gateway-address
-* :vytask:`1864` Lower IPSec DPD timeout lower limit from 10s -> 2s
-* :vytask:`1879` Extend Dynamic DNS XML definition value help strings and validators
-* :vytask:`1881` Execute permissions are removed from custom SNMP scripts at commit time
-* :vytask:`1884` Keeping VRRP transition-script native behaviour and adding stop-script
-* :vytask:`1891` Router announcements broken on boot
-* :vytask:`1900` Enable SNMP for VRRP.
-* :vytask:`1902` Add redistribute non main table in bgp
-* :vytask:`1909` Incorrect behaviour of static routes with overlapping networks
-* :vytask:`1913` "system ipv6 blacklist" command has no effect
-* :vytask:`1914` IPv6 multipath hash policy does not apply
-* :vytask:`1917` Update WireGuard to Debian release 0.0.20191219-1
-* :vytask:`1934` Change default hostname when deploy from OVA without params.
-* :vytask:`1935` NIC identification and usage problem in Hyper-V environments
-* :vytask:`1936` pppoe-server CLI control features
-* :vytask:`1964` SNMP Script-extensions allows names with spaces, but commit fails
-* :vytask:`1967` BGP parameter "enforce-first-as" does not work anymore
-* :vytask:`1970` Correct adding interfaces on boot
-* :vytask:`1971` Missing modules in initrd.img for PXE boot
-* :vytask:`1998` Update FRR to 7.3
-* :vytask:`2001` Error when router reboot
-* :vytask:`2032` Monitor bandwidth bits
-* :vytask:`2059` Set source-validation on bond vif don't work
-* :vytask:`2066` PPPoE interface can be created multiple times - last wins
-* :vytask:`2069` PPPoE-client does not works with service-name option
-* :vytask:`2077` ISO build from crux branch is failing
-* :vytask:`2079` Update Linux Kernel to v4.19.106
-* :vytask:`2087` Add maxfail 0 option to pppoe configuration.
-* :vytask:`2100` BGP route adverisement wih checks rib
-* :vytask:`2120` "reset vpn ipsec-peer" doesn't work with named peers
-* :vytask:`2197` Cant add vif-s interface into a bridge
-* :vytask:`2228` WireGuard does not allow ports < 1024 to be used
-* :vytask:`2252` HTTP API add system image can return '504 Gateway Time-out'
-* :vytask:`2272` Set system flow-accounting disable-imt has syntax error
-* :vytask:`2276` PPPoE server vulnerability
- 
-
-1.2.4
------
-
-1.2.4 is a maintenance release made in December 2019.
-
-Resolved issues
-^^^^^^^^^^^^^^^
-
-* :vytask:`T258` Can not configure wan load-balancing on vyos-1.2
-* :vytask:`T818` SNMP v3 - remove required engineid from user node
-* :vytask:`T1030` Upgrade ddclient from 3.8.2 to 3.9.0 (support Cloudflare API v4)
-* :vytask:`T1183` BFD Support via FRR
-* :vytask:`T1299` Allow SNMPd to be extended with custom scripts
-* :vytask:`T1351` accel-pppoe adding CIDR based IP pool option
-* :vytask:`T1391` In route-map set community additive
-* :vytask:`T1394` syslog systemd and host_name.py race condition
-* :vytask:`T1401` Copying files with the FTP protocol fails if the password contains special characters
-* :vytask:`T1421` OpenVPN client push-route stopped working, needs added quotes to fix
-* :vytask:`T1430` Add options for custom DHCP client-id and hostname
-* :vytask:`T1447` Python subprocess called without import in host_name.py
-* :vytask:`T1470` improve output of "show dhcpv6 server leases"
-* :vytask:`T1485` Enable 'AdvIntervalOpt' option in for radvd.conf
-* :vytask:`T1496` Separate rolling release and LTS kernel builds
-* :vytask:`T1560` "set load-balancing wan rule 0" causes segfault and prevents load balancing from starting
-* :vytask:`T1568` strip-private command improvement for additional masking of IPv6 and MAC address
-* :vytask:`T1578` completion offers "show table", but show table does not exist
-* :vytask:`T1593` Support ip6gre
-* :vytask:`T1597` /usr/sbin/rsyslogd after deleting "system syslog"
-* :vytask:`T1638` vyos-hostsd not setting system domain name
-* :vytask:`T1678` hostfile-update missing line feed
-* :vytask:`T1694` NTPd: Do not listen on all interfaces by default
-* :vytask:`T1701` Delete domain-name and domain-search won't work
-* :vytask:`T1705` High CPU usage by bgpd when snmp is active
-* :vytask:`T1707` DHCP static mapping and exclude address not working
-* :vytask:`T1708` Update Rolling Release Kernel to 4.19.76
-* :vytask:`T1709` Update WireGuard to 0.0.20190913
-* :vytask:`T1716` Update Intel NIC drivers to recent versions
-* :vytask:`T1726` Update Linux Firmware binaries to a more recent version 2019-03-14 -> 2019-10-07
-* :vytask:`T1728` Update Linux Kernel to 4.19.79
-* :vytask:`T1737` SNMP tab completion missing
-* :vytask:`T1738` Copy SNMP configuration from node to node raises exception
-* :vytask:`T1740` Broken OSPFv2 virtual-link authentication
-* :vytask:`T1742` NHRP unable to commit.
-* :vytask:`T1745` dhcp-server commit fails with "DHCP range stop address x must be greater or equal to the range start address y!" when static mapping has same IP as range stop
-* :vytask:`T1749` numeric validator doesn't support multiple ranges
-* :vytask:`T1769` Remove complex SNMPv3 Transport Security Model (TSM)
-* :vytask:`T1772` <regex> constraints in XML are partially broken
-* :vytask:`T1778` Kilobits/Megabits difference in configuration Vyos/FRR
-* :vytask:`T1780` Adding ipsec ike closeaction
-* :vytask:`T1786` disable-dhcp-nameservers is missed in current host_name.py implementation
-* :vytask:`T1788` Intel QAT (QuickAssist Technology ) implementation
-* :vytask:`T1792` Update WireGuard to Debian release 0.0.20191012-1
-* :vytask:`T1800` Update Linux Kernel to v4.19.84
-* :vytask:`T1809` Wireless: SSID scan does not work in AP mode
-* :vytask:`T1811` Upgrade from 1.1.8: Config file migration failed: module=l2tp
-* :vytask:`T1812` DHCP: hostnames of clients not resolving after update v1.2.3 -> 1.2-rolling
-* :vytask:`T1819` Reboot kills SNMPv3 configuration
-* :vytask:`T1822` Priority inversion wireless interface dhcpv6
-* :vytask:`T1825` Improve DHCP configuration error message
-* :vytask:`T1836` import-conf-mode-commands in vyos-1x/scripts fails to create an xml
-* :vytask:`T1839` LLDP shows "VyOS unknown" instead of "VyOS"
-* :vytask:`T1841` PPP ipv6-up.d direcotry missing
-* :vytask:`T1893` igmp-proxy: Do not allow adding unknown interface
-* :vytask:`T1903` Implementation udev predefined interface naming
-* :vytask:`T1904` update eth1 and eth2 link files for the vep4600
-
-
-1.2.3
------
-
-1.2.3 is a maintenance and feature backport release made in September 2019.
-
-New features
-^^^^^^^^^^^^
-
-* HTTP API
-* :vytask:`T1524` "set service dns forwarding allow-from <IPv4 net|IPv6 net>"
-  option for limiting queries to specific client networks
-* :vytask:`T1503` Functions for checking if a commit is in progress
-* :vytask:`T1543` "set system contig-mangement commit-archive source-address"
-  option
-* :vytask:`T1554` Intel NIC drivers now support receive side scaling and
-  multiqueue
-
-Resolved issues
-^^^^^^^^^^^^^^^
-
-* :vytask:`T1209` OSPF max-metric values over 100 no longer causes commit
-  errors
-* :vytask:`T1333` Fixes issue with DNS forwarding not performing recursive
-  lookups on domain specific forwarders
-* :vytask:`T1362` Special characters in VRRP passwords are handled correctly
-* :vytask:`T1377` BGP weight is applied properly
-* :vytask:`T1420` Fixed permission for log files
-* :vytask:`T1425` Wireguard interfaces now support /31 addresses
-* :vytask:`T1428` Wireguard correctly handles firewall marks
-* :vytask:`T1439` DHCPv6 static mappings now work correctly
-* :vytask:`T1450` Flood ping commands now works correctly
-* :vytask:`T1460` Op mode "show firewall" commands now support counters longer
-  than 8 digits (T1460)
-* :vytask:`T1465` Fixed priority inversion in VTI commands
-* :vytask:`T1468` Fixed remote-as check in the BGP route-reflector-client option
-* :vytask:`T1472` It's now possible to re-create VRRP groups with RFC
-  compatibility mode enabled
-* :vytask:`T1527` Fixed a typo in DHCPv6 server help strings
-* :vytask:`T1529` Unnumbered BGP peers now support VLAN interfaces
-* :vytask:`T1530` Fixed "set system syslog global archive file" command
-* :vytask:`T1531` Multiple fixes in cluster configuration scripts
-* :vytask:`T1537` Fixed missing help text for "service dns"
-* :vytask:`T1541` Fixed input validation in DHCPv6 relay options
-* :vytask:`T1551` It's now possible to create a QinQ interface and a firewall
-  assigned to it in one commit
-* :vytask:`T1559` URL filtering now uses correct rule database path and works
-  again
-* :vytask:`T1579` "show log vpn ipsec" command works again
-* :vytask:`T1576` "show arp interface <intf>" command works again
-* :vytask:`T1605` Fixed regression in L2TP/IPsec server
-* :vytask:`T1613` Netflow/sFlow captures IPv6 traffic correctly
-* :vytask:`T1616` "renew dhcpv6" command now works from op mode
-* :vytask:`T1642` BGP remove-private-as option iBGP vs eBGP check works
-  correctly now
-* :vytask:`T1540`, :vytask:`T1360`, :vytask:`T1264`, :vytask:`T1623` Multiple
-  improvements in name servers and hosts configuration handling
-
-Internals
-^^^^^^^^^
-
-``/etc/resolv.conf`` and ``/etc/hosts`` files are now managed by the
-*vyos-hostsd* service that listens on a ZMQ socket for update messages.
-
-1.2.2
------
-
-1.2.2 is a maintenance release made in July 2019.
-
-New features
-^^^^^^^^^^^^
-
-* Options for per-interface MSS clamping.
-* BGP extended next-hop capability
-* Relaxed BGP multipath option
-* Internal and external options for "remote-as" (accept any AS as long as it's
-  the same to this router or different, respectively)
-* "Unnumbered" (interface-based) BGP peers
-* BGP no-prepend option
-* Additive BGP community option
-* OSPFv3 network type option
-* Custom arguments for VRRP scripts
-* A script for querying values from config files
-
-Resolved issues
-^^^^^^^^^^^^^^^
-
-* Linux kernel 4.19.54, including a fix for the TCP SACK vulnerability
-* :vytask:`T1371` VRRP health-check scripts now can use arguments
-* :vytask:`T1497` DNS server addresses coming from a DHCP server are now
-  correctly propagated to resolv.conf
-* :vytask:`T1469` Domain-specific name servers in DNS forwarding are now used
-  for recursive queries
-* :vytask:`T1433` ``run show dhcpv6 server leases`` now display leases correctly
-* :vytask:`T1461` Deleting ``firewall options`` node no longer causes errors
-* :vytask:`T1458` Correct hostname is sent to remote syslog again
-* :vytask:`T1438` Board serial number from DMI is correctly displayed in
-  ``show version``
-* :vytask:`T1358`, :vytask:`T1355`, :vytask:`T1294` Multiple corrections in
-  remote syslog config
-* :vytask:`T1255` Fixed missing newline in ``/etc/hosts``
-* :vytask:`T1174` ``system domain-name`` is correctly included in
-  ``/etc/resolv.conf``
-* :vytask:`T1465` Fixed priority inversion in ``interfaces vti vtiX ip``
-  settings
-* :vytask:`T1446` Fixed errors when installing with RAID1 on UEFI machines
-* :vytask:`T1387` Fixed an error on disabling an interfaces that has no address
-* :vytask:`T1367` Fixed deleting VLAN interface with non-default MTU
-* :vytask:`T1505` vyos.config ``return_effective_values()`` function now
-  correctly returns a list rather than a string
-
-1.2.1
------
-
-VyOS 1.2.1 is a maintenance release made in April 2019.
-
-Resolved issues
-^^^^^^^^^^^^^^^
-
-* Package updates: kernel 4.19.32, open-vm-tools 10.3, latest Intel NIC drivers
-* :vytask:`T1326` The kernel now includes drivers for various USB serial
-  adapters, which allows people to add a serial console to a machine without
-  onboard RS232, or connect to something else from the router
-* The collection of network card firmware is now much more extensive
-* :vytask:`T1271` VRRP now correctly uses a virtual rather than physical MAC
-  addresses in the RFC-compliant mode
-* :vytask:`T1330` DHCP WPAD URL option works correctly again
-* :vytask:`T1312` Many to many NAT rules now can use source/destination and
-  translation networks of non-matching size. If 1:1 network bits translation is
-  desired, it's now users responsibility to check if prefix length matches.
-* :vytask:`T1290` IPv6 network prefix translation is fixed
-* :vytask:`T1308` Non-alphanumeric characters such as ``>`` can now be safely
-  used in PPPoE passwords
-* :vytask:`T1305` ``show | commands`` no longer fails when a config section ends
-  with a leaf node such as ``timezone`` in ``show system | commands``
-* :vytask:`T1235` ``show | commands`` correctly works in config mode now
-* :vytask:`T1298` VTI is now compatible with the DHCP-interface IPsec option
-* :vytask:`T1277` ``show dhcp server statistics`` command was broken in latest
-  Crux
-* :vytask:`T1261` An issue with TFTP server refusing to listen on addresses
-  other than loopback was fixed
-* :vytask:`T1224` Template issue that might cause UDP broadcast relay fail to
-  start is fixed
-* :vytask:`T1067` VXLAN value validation is improved
-* :vytask:`T1211` Blank hostnames in DHCP updates no longer can crash DNS
-  forwarding
-* :vytask:`T1322` Correct configuration is now generated for DHCPv6 relays with
-  more than one upstream interface
-* :vytask:`T1234` ``relay-agents-packets`` option works correctly now
-* :vytask:`T1231` Dynamic DNS data is now cleaned on configuration change
-* :vytask:`T1282` Remote Syslog can now use a fully qualified domain name
-* :vytask:`T1279` ACPI power off works again
-* :vytask:`T1247` Negation in WAN load balancing rules works again
-* :vytask:`T1218` FRR staticd now starts on boot correctly
-* :vytask:`T1296` The installer now correctly detects SD card devices
-* :vytask:`T1225` Wireguard peers can be disabled now
-* :vytask:`T1217` The issue with Wireguard interfaces impossible to delete
-  is fixed
-* :vytask:`T1160` Unintended IPv6 access is fixed in SNMP configuration
-* :vytask:`T1060` It's now possible to exclude hosts from the transparent
-  web proxy
-* :vytask:`T484` An issue with rules impossible to delete from the zone-based
-  firewall is fixed
-
-Earlier releases
-================
-
-Release notes for legacy versions (1.1.x, 1.0.x) can be found in the `archived wiki <https://web.archive.org/web/20200212180711/https://wiki.vyos.net/wiki/Category:Release_notes>`_.

docs/appendix/vyos-on-clouds.rst

@@ -1,173 +0,0 @@
-.. _vyos-on-clouds:
-
-Running on Clouds
-#################
-
-Amazon AWS
-**********
-
-Deploy VM
----------
-
-Deploy VyOS on Amazon :abbr:`AWS (Amazon Web Services)`
-
-1. Click to ``Instances`` and ``Launch Instance``
-
-.. figure:: /_static/images/cloud-aws-01.png
-
-2. On the marketplace search "VyOS"
-
-.. figure:: /_static/images/cloud-aws-02.png
-
-3. Choose the instance type. Minimum recommendation start from ``m3.medium``
-
-.. figure:: /_static/images/cloud-aws-03.png
-
-4. Configure instance for your requirements. Select number of instances / network / subnet
-
-.. figure:: /_static/images/cloud-aws-04.png
-
-5. Additional storage. You can remove additional storage ``/dev/sdb``. First root device will be ``/dev/xvda``. You can skeep this step.
-
-.. figure:: /_static/images/cloud-aws-05.png
-
-6. Configure Security Group. It's recommended that you configure ssh access only from certain address sources. Or permit any (by default).
-
-.. figure:: /_static/images/cloud-aws-06.png
-
-7. Select SSH key pair and click ``Launch Instances``
-
-.. figure:: /_static/images/cloud-aws-07.png
-
-8. Find out your public IP address.
-
-.. figure:: /_static/images/cloud-aws-08.png
-
-9. Connect to the instance by SSH key.
-
-  .. code-block:: none
-
-    ssh -i ~/.ssh/amazon.pem vyos@203.0.113.3
-    vyos@ip-192-0-2-10:~$
-
-
-
-
-References
-----------
-https://console.aws.amazon.com/
-
-Azure
-*****
-
-Deploy VM
----------
-
-Deploy VyOS on Azure.
-
-1. Go to the Azure services and Click to **Add new Virtual machine**
-
-2. Choose vm name, resource group, region and click **Browse all public and private images**
-
-.. figure:: /_static/images/cloud-azure-01.png
-
-3. On the marketplace search ``VyOS``
-
-.. figure:: /_static/images/cloud-azure-02.png
-
-4. Generate new SSH key pair or use existing.
-
-.. figure:: /_static/images/cloud-azure-03.png
-
-5. Define network, subnet, Public IP. Or it will be created by default.
-
-.. figure:: /_static/images/cloud-azure-04.png
-
-6. Click ``Review + create``. After fiew second your deployment will be complete
-
-.. figure:: /_static/images/cloud-azure-05.png
-
-7. Click to your new vm and find out your Public IP address.
-
-.. figure:: /_static/images/cloud-azure-06.png
-
-8. Connect to the instance by SSH key.
-
-  .. code-block:: none
-
-    ssh -i ~/.ssh/vyos_azure vyos@203.0.113.3
-    vyos@vyos-doc-r1:~$
-
-Add interface
--------------
-
-If instance was deployed with one **eth0** ``WAN`` interface and want to add new one.
-To add new interface an example **eth1** ``LAN`` you need shutdown the instance. Attach the interface in the Azure portal and then start the instance.
-
-.. NOTE:: Azure does not allow you attach interface when the instance in the **Running** state.
-
-References
-----------
-https://azure.microsoft.com
-
-Google Cloud Platform
-*********************
-
-Deploy VM
----------
-
-To deploy VyOS on GCP (Google Cloud Platform)
-
-1. Generate SSH key pair type **ssh-rsa** from the host that will connect to VyOS.
-
-  Example:
-
-  .. code-block:: none
-
-    ssh-keygen -t rsa -f ~/.ssh/vyos_gcp -C "vyos@mypc"
-
-
-.. NOTE:: In name "vyos@mypc" The first value must be "**vyos**". Because default user is vyos and google api uses this option.
-
-
-2. Open GCP console and navigate to the menu **Metadata**. Choose **SSH Keys** and click ``edit``.
-
-.. figure:: /_static/images/cloud-gcp-01.png
-
-
-Click **Add item** and paste your public ssh key. Click ``Save``.
-
-.. figure:: /_static/images/cloud-gcp-02.png
-
-
-2. On marketplace search "VyOS"
-
-3. Change Deployment name/Zone/Machine type and click ``Deploy``
-
-.. figure:: /_static/images/cloud-gcp-03.png
-
-4. After fiew seconds click to ``instance``
-
-.. figure:: /_static/images/cloud-gcp-04.png
-
-5. Find out your external IP address
-
-.. figure:: /_static/images/cloud-gcp-05.png
-
-6. Connect to the instance. SSH key was generated in the first step.
-
-  .. code-block:: none
-
-    ssh -i ~/.ssh/vyos_gcp vyos@203.0.113.3
-    vyos@vyos-r1-vm:~$
-
-References
-----------
-https://console.cloud.google.com/
-
-Oracle
-*****************
-
-References
-----------
-https://www.oracle.com/cloud/

docs/appendix/vyos-on-vmware.rst

@@ -1,32 +0,0 @@
-.. _vyosonvmware:
-
-Running on VMware ESXi
-######################
-
-ESXi 5.5 or later
-*****************
-
-.ova files are available for supporting users, and a VyOS can also be stood up using a generic Linux instance, and attaching the bootable ISO file and installing from the ISO 
-using the normal process around `install image`.
-
-.. NOTE:: There have been previous documented issues with GRE/IPSEC tunneling using the E1000 adapter on the VyOS guest, and use of the VMXNET3 has been advised.
-
-Memory Contention Considerations
---------------------------------
-When the underlying ESXi host is approaching ~92% memory utilisation it will start the balloon process in s a 'soft' state to start reclaiming memory from guest operating systems. 
-This causes an artificial pressure using the vmmemctl driver on memory usage on the virtual guest. As VyOS by default does not have a swap file, this vmmemctl pressure is unable to
-force processes to move in memory data to the paging file, and blindly consumes memory forcing the virtual guest into a low memory state with no way to escape. The balloon can expand to 65% of 
-guest allocated memory, so a VyOS guest running >35% of memory usage, can encounter an out of memory situation, and trigger the kernel oom_kill process. At this point a weighted 
-lottery favouring memory hungry processes will be run with the unlucky winner being terminated by the kernel.
-
-It is advised that VyOS routers are configured in a resource group with adequate memory reservations so that ballooning is not inflicted on virtual VyOS guests.
-
-
-
-
-
-References
-----------
-
-https://muralidba.blogspot.com/2018/03/how-does-linux-out-of-memory-oom-killer.html
-

docs/automation/index.rst

@@ -0,0 +1,15 @@
+###############
+VyOS Automation
+###############
+
+
+  * Ansible
+  * Saltstack
+  * HTTP-API
+  * startup scripts
+
+
+.. toctree::
+   :maxdepth: 1
+
+   command-scripting

docs/changelog/1.2.1.rst

@@ -0,0 +1,52 @@
+1.2.1
+=====
+
+VyOS 1.2.1 is a maintenance release made in April 2019.
+
+Resolved issues
+---------------
+
+* Package updates: kernel 4.19.32, open-vm-tools 10.3, latest Intel NIC drivers
+* :vytask:`T1326` The kernel now includes drivers for various USB serial
+  adapters, which allows people to add a serial console to a machine without
+  onboard RS232, or connect to something else from the router
+* The collection of network card firmware is now much more extensive
+* :vytask:`T1271` VRRP now correctly uses a virtual rather than physical MAC
+  addresses in the RFC-compliant mode
+* :vytask:`T1330` DHCP WPAD URL option works correctly again
+* :vytask:`T1312` Many to many NAT rules now can use source/destination and
+  translation networks of non-matching size. If 1:1 network bits translation is
+  desired, it's now users responsibility to check if prefix length matches.
+* :vytask:`T1290` IPv6 network prefix translation is fixed
+* :vytask:`T1308` Non-alphanumeric characters such as ``>`` can now be safely
+  used in PPPoE passwords
+* :vytask:`T1305` ``show | commands`` no longer fails when a config section ends
+  with a leaf node such as ``timezone`` in ``show system | commands``
+* :vytask:`T1235` ``show | commands`` correctly works in config mode now
+* :vytask:`T1298` VTI is now compatible with the DHCP-interface IPsec option
+* :vytask:`T1277` ``show dhcp server statistics`` command was broken in latest
+  Crux
+* :vytask:`T1261` An issue with TFTP server refusing to listen on addresses
+  other than loopback was fixed
+* :vytask:`T1224` Template issue that might cause UDP broadcast relay fail to
+  start is fixed
+* :vytask:`T1067` VXLAN value validation is improved
+* :vytask:`T1211` Blank hostnames in DHCP updates no longer can crash DNS
+  forwarding
+* :vytask:`T1322` Correct configuration is now generated for DHCPv6 relays with
+  more than one upstream interface
+* :vytask:`T1234` ``relay-agents-packets`` option works correctly now
+* :vytask:`T1231` Dynamic DNS data is now cleaned on configuration change
+* :vytask:`T1282` Remote Syslog can now use a fully qualified domain name
+* :vytask:`T1279` ACPI power off works again
+* :vytask:`T1247` Negation in WAN load balancing rules works again
+* :vytask:`T1218` FRR staticd now starts on boot correctly
+* :vytask:`T1296` The installer now correctly detects SD card devices
+* :vytask:`T1225` Wireguard peers can be disabled now
+* :vytask:`T1217` The issue with Wireguard interfaces impossible to delete
+  is fixed
+* :vytask:`T1160` Unintended IPv6 access is fixed in SNMP configuration
+* :vytask:`T1060` It's now possible to exclude hosts from the transparent
+  web proxy
+* :vytask:`T484` An issue with rules impossible to delete from the zone-based
+  firewall is fixed

docs/changelog/1.2.2.rst

@@ -0,0 +1,46 @@
+1.2.2
+=====
+
+1.2.2 is a maintenance release made in July 2019.
+
+New features
+------------
+
+* Options for per-interface MSS clamping.
+* BGP extended next-hop capability
+* Relaxed BGP multipath option
+* Internal and external options for "remote-as" (accept any AS as long as it's
+  the same to this router or different, respectively)
+* "Unnumbered" (interface-based) BGP peers
+* BGP no-prepend option
+* Additive BGP community option
+* OSPFv3 network type option
+* Custom arguments for VRRP scripts
+* A script for querying values from config files
+
+Resolved issues
+---------------
+
+* Linux kernel 4.19.54, including a fix for the TCP SACK vulnerability
+* :vytask:`T1371` VRRP health-check scripts now can use arguments
+* :vytask:`T1497` DNS server addresses coming from a DHCP server are now
+  correctly propagated to resolv.conf
+* :vytask:`T1469` Domain-specific name servers in DNS forwarding are now used
+  for recursive queries
+* :vytask:`T1433` ``run show dhcpv6 server leases`` now display leases correctly
+* :vytask:`T1461` Deleting ``firewall options`` node no longer causes errors
+* :vytask:`T1458` Correct hostname is sent to remote syslog again
+* :vytask:`T1438` Board serial number from DMI is correctly displayed in
+  ``show version``
+* :vytask:`T1358`, :vytask:`T1355`, :vytask:`T1294` Multiple corrections in
+  remote syslog config
+* :vytask:`T1255` Fixed missing newline in ``/etc/hosts``
+* :vytask:`T1174` ``system domain-name`` is correctly included in
+  ``/etc/resolv.conf``
+* :vytask:`T1465` Fixed priority inversion in ``interfaces vti vtiX ip``
+  settings
+* :vytask:`T1446` Fixed errors when installing with RAID1 on UEFI machines
+* :vytask:`T1387` Fixed an error on disabling an interfaces that has no address
+* :vytask:`T1367` Fixed deleting VLAN interface with non-default MTU
+* :vytask:`T1505` vyos.config ``return_effective_values()`` function now
+  correctly returns a list rather than a string

docs/changelog/1.2.3.rst

@@ -0,0 +1,62 @@
+1.2.3
+=====
+
+1.2.3 is a maintenance and feature backport release made in September 2019.
+
+New features
+------------
+
+* HTTP API
+* :vytask:`T1524` "set service dns forwarding allow-from <IPv4 net|IPv6 net>"
+  option for limiting queries to specific client networks
+* :vytask:`T1503` Functions for checking if a commit is in progress
+* :vytask:`T1543` "set system contig-mangement commit-archive source-address"
+  option
+* :vytask:`T1554` Intel NIC drivers now support receive side scaling and
+  multiqueue
+
+Resolved issues
+---------------
+
+* :vytask:`T1209` OSPF max-metric values over 100 no longer causes commit
+  errors
+* :vytask:`T1333` Fixes issue with DNS forwarding not performing recursive
+  lookups on domain specific forwarders
+* :vytask:`T1362` Special characters in VRRP passwords are handled correctly
+* :vytask:`T1377` BGP weight is applied properly
+* :vytask:`T1420` Fixed permission for log files
+* :vytask:`T1425` Wireguard interfaces now support /31 addresses
+* :vytask:`T1428` Wireguard correctly handles firewall marks
+* :vytask:`T1439` DHCPv6 static mappings now work correctly
+* :vytask:`T1450` Flood ping commands now works correctly
+* :vytask:`T1460` Op mode "show firewall" commands now support counters longer
+  than 8 digits (T1460)
+* :vytask:`T1465` Fixed priority inversion in VTI commands
+* :vytask:`T1468` Fixed remote-as check in the BGP route-reflector-client option
+* :vytask:`T1472` It's now possible to re-create VRRP groups with RFC
+  compatibility mode enabled
+* :vytask:`T1527` Fixed a typo in DHCPv6 server help strings
+* :vytask:`T1529` Unnumbered BGP peers now support VLAN interfaces
+* :vytask:`T1530` Fixed "set system syslog global archive file" command
+* :vytask:`T1531` Multiple fixes in cluster configuration scripts
+* :vytask:`T1537` Fixed missing help text for "service dns"
+* :vytask:`T1541` Fixed input validation in DHCPv6 relay options
+* :vytask:`T1551` It's now possible to create a QinQ interface and a firewall
+  assigned to it in one commit
+* :vytask:`T1559` URL filtering now uses correct rule database path and works
+  again
+* :vytask:`T1579` "show log vpn ipsec" command works again
+* :vytask:`T1576` "show arp interface <intf>" command works again
+* :vytask:`T1605` Fixed regression in L2TP/IPsec server
+* :vytask:`T1613` Netflow/sFlow captures IPv6 traffic correctly
+* :vytask:`T1616` "renew dhcpv6" command now works from op mode
+* :vytask:`T1642` BGP remove-private-as option iBGP vs eBGP check works
+  correctly now
+* :vytask:`T1540`, :vytask:`T1360`, :vytask:`T1264`, :vytask:`T1623` Multiple
+  improvements in name servers and hosts configuration handling
+
+Internals
+---------
+
+``/etc/resolv.conf`` and ``/etc/hosts`` files are now managed by the
+*vyos-hostsd* service that listens on a ZMQ socket for update messages.

docs/changelog/1.2.4.rst

@@ -0,0 +1,77 @@
+1.2.4
+=====
+
+1.2.4 is a maintenance release made in December 2019.
+
+Resolved issues
+---------------
+
+* :vytask:`T258` Can not configure wan load-balancing on vyos-1.2
+* :vytask:`T818` SNMP v3 - remove required engineid from user node
+* :vytask:`T1030` Upgrade ddclient from 3.8.2 to 3.9.
+  (support Cloudflare API v4)
+* :vytask:`T1183` BFD Support via FRR
+* :vytask:`T1299` Allow SNMPd to be extended with custom scripts
+* :vytask:`T1351` accel-pppoe adding CIDR based IP pool option
+* :vytask:`T1391` In route-map set community additive
+* :vytask:`T1394` syslog systemd and host_name.py race condition
+* :vytask:`T1401` Copying files with the FTP protocol fails if the passwor
+  contains special characters
+* :vytask:`T1421` OpenVPN client push-route stopped working, needs added quotes
+  to fix
+* :vytask:`T1430` Add options for custom DHCP client-id and hostname
+* :vytask:`T1447` Python subprocess called without import in host_name.py
+* :vytask:`T1470` improve output of "show dhcpv6 server leases"
+* :vytask:`T1485` Enable 'AdvIntervalOpt' option in for radvd.conf
+* :vytask:`T1496` Separate rolling release and LTS kernel builds
+* :vytask:`T1560` "set load-balancing wan rule 0" causes segfault and prevent
+  load balancing from starting
+* :vytask:`T1568` strip-private command improvement for additional masking o
+  IPv6 and MAC address
+* :vytask:`T1578` completion offers "show table", but show table does not exist
+* :vytask:`T1593` Support ip6gre
+* :vytask:`T1597` /usr/sbin/rsyslogd after deleting "system syslog"
+* :vytask:`T1638` vyos-hostsd not setting system domain name
+* :vytask:`T1678` hostfile-update missing line feed
+* :vytask:`T1694` NTPd: Do not listen on all interfaces by default
+* :vytask:`T1701` Delete domain-name and domain-search won't work
+* :vytask:`T1705` High CPU usage by bgpd when snmp is active
+* :vytask:`T1707` DHCP static mapping and exclude address not working
+* :vytask:`T1708` Update Rolling Release Kernel to 4.19.76
+* :vytask:`T1709` Update WireGuard to 0.0.20190913
+* :vytask:`T1716` Update Intel NIC drivers to recent versions
+* :vytask:`T1726` Update Linux Firmware binaries to a more recen
+  version 2019-03-14 -> 2019-10-07
+* :vytask:`T1728` Update Linux Kernel to 4.19.79
+* :vytask:`T1737` SNMP tab completion missing
+* :vytask:`T1738` Copy SNMP configuration from node to node raises exception
+* :vytask:`T1740` Broken OSPFv2 virtual-link authentication
+* :vytask:`T1742` NHRP unable to commit.
+* :vytask:`T1745` dhcp-server commit fails with "DHCP range stop address 
+  must be greater or equal to the range start address y!" when static mapping
+  has same IP as range stop
+* :vytask:`T1749` numeric validator doesn't support multiple ranges
+* :vytask:`T1769` Remove complex SNMPv3 Transport Security Model (TSM)
+* :vytask:`T1772` <regex> constraints in XML are partially broken
+* :vytask:`T1778` Kilobits/Megabits difference in configuration Vyos/FRR
+* :vytask:`T1780` Adding ipsec ike closeaction
+* :vytask:`T1786` disable-dhcp-nameservers is missed in current host_name.p
+  implementation
+* :vytask:`T1788` Intel QAT (QuickAssist Technology ) implementation
+* :vytask:`T1792` Update WireGuard to Debian release 0.0.20191012-1
+* :vytask:`T1800` Update Linux Kernel to v4.19.84
+* :vytask:`T1809` Wireless: SSID scan does not work in AP mode
+* :vytask:`T1811` Upgrade from 1.1.8: Config file migratio
+  failed: module=l2tp
+* :vytask:`T1812` DHCP: hostnames of clients not resolving afte
+  update v1.2.3 -> 1.2-rolling
+* :vytask:`T1819` Reboot kills SNMPv3 configuration
+* :vytask:`T1822` Priority inversion wireless interface dhcpv6
+* :vytask:`T1825` Improve DHCP configuration error message
+* :vytask:`T1836` import-conf-mode-commands in vyos-1x/scripts fails
+  to create an xml
+* :vytask:`T1839` LLDP shows "VyOS unknown" instead of "VyOS"
+* :vytask:`T1841` PPP ipv6-up.d direcotry missing
+* :vytask:`T1893` igmp-proxy: Do not allow adding unknown interface
+* :vytask:`T1903` Implementation udev predefined interface naming
+* :vytask:`T1904` update eth1 and eth2 link files for the vep4600

docs/changelog/1.2.5.rst

@@ -0,0 +1,70 @@
+1.2.5
+=====
+
+1.2.5 is a maintenance release made in April 2020.
+
+Resolved issues
+---------------
+
+* :vytask:`T1020` OSPF Stops distributing default route after a while
+* :vytask:`T1228` pppoe default-route force option not working (Rel 1.2.0-rc11)
+* :vytask:`T1301` bgp peer-groups don't work when "no-ipv4-unicast" is enabled.
+* :vytask:`T1341` Adding rate-limiter for pppoe server users
+* :vytask:`T1376` Incorrect DHCP lease counting
+* :vytask:`T1392` Large firewall rulesets cause the system to lose configuration
+  and crash at startup
+* :vytask:`T1416` 2 dhcp server run in failover mode can't sync hostname with
+  each other
+* :vytask:`T1452` accel-pppoe - add vendor option to shaper
+* :vytask:`T1490` BGP configuration (is lost|not applied) when updating
+  1.1.8 -> 1.2.1
+* :vytask:`T1780` Adding ipsec ike closeaction
+* :vytask:`T1803` Unbind NTP while it's not requested...
+* :vytask:`T1821` "authentication mode radius" has no effect for PPPoE server
+* :vytask:`T1827` Increase default gc_thresh
+* :vytask:`T1828` Missing completion helper for "set system syslog host
+  192.0.2.1 facility all protocol"
+* :vytask:`T1832` radvd adding feature DNSSL branch.example.com example.com to
+  existing package
+* :vytask:`T1837` PPPoE unrecognized option 'replacedefaultroute'
+* :vytask:`T1851` wireguard - changing the pubkey on an existing peer seems to
+  destroy the running config.
+* :vytask:`T1858` l2tp: Delete depricated outside-nexthop and add gateway-address
+* :vytask:`T1864` Lower IPSec DPD timeout lower limit from 10s -> 2s
+* :vytask:`T1879` Extend Dynamic DNS XML definition value help strings and
+  validators
+* :vytask:`T1881` Execute permissions are removed from custom SNMP scripts at
+  commit time
+* :vytask:`T1884` Keeping VRRP transition-script native behaviour and adding
+  stop-script
+* :vytask:`T1891` Router announcements broken on boot
+* :vytask:`T1900` Enable SNMP for VRRP.
+* :vytask:`T1902` Add redistribute non main table in bgp
+* :vytask:`T1909` Incorrect behaviour of static routes with overlapping networks
+* :vytask:`T1913` "system ipv6 blacklist" command has no effect
+* :vytask:`T1914` IPv6 multipath hash policy does not apply
+* :vytask:`T1917` Update WireGuard to Debian release 0.0.20191219-1
+* :vytask:`T1934` Change default hostname when deploy from OVA without params.
+* :vytask:`T1935` NIC identification and usage problem in Hyper-V environments
+* :vytask:`T1936` pppoe-server CLI control features
+* :vytask:`T1964` SNMP Script-extensions allows names with spaces, but commit
+  fails
+* :vytask:`T1967` BGP parameter "enforce-first-as" does not work anymore
+* :vytask:`T1970` Correct adding interfaces on boot
+* :vytask:`T1971` Missing modules in initrd.img for PXE boot
+* :vytask:`T1998` Update FRR to 7.3
+* :vytask:`T2001` Error when router reboot
+* :vytask:`T2032` Monitor bandwidth bits
+* :vytask:`T2059` Set source-validation on bond vif don't work
+* :vytask:`T2066` PPPoE interface can be created multiple times - last wins
+* :vytask:`T2069` PPPoE-client does not works with service-name option
+* :vytask:`T2077` ISO build from crux branch is failing
+* :vytask:`T2079` Update Linux Kernel to v4.19.106
+* :vytask:`T2087` Add maxfail 0 option to pppoe configuration.
+* :vytask:`T2100` BGP route adverisement wih checks rib
+* :vytask:`T2120` "reset vpn ipsec-peer" doesn't work with named peers
+* :vytask:`T2197` Cant add vif-s interface into a bridge
+* :vytask:`T2228` WireGuard does not allow ports < 1024 to be used
+* :vytask:`T2252` HTTP API add system image can return '504 Gateway Time-out'
+* :vytask:`T2272` Set system flow-accounting disable-imt has syntax error
+* :vytask:`T2276` PPPoE server vulnerability

docs/changelog/1.2.6.rst

@@ -0,0 +1,106 @@
+1.2.6-S1
+========
+
+1.2.6-S1 is a security release release made in September 2020.
+
+Resolved issues
+---------------
+
+VyOS 1.2.6 release was found to be suspectible to CVE-2020-10995. It's a low-
+impact vulnerability in the PowerDNS recursor that allows an attacker to cause
+performance degradation via a specially crafted authoritative DNS server reply.
+
+* :vytask:`T2899` remote syslog server migration error on update
+
+1.2.6
+=====
+
+1.2.6 is a maintenance release made in September 2020.
+
+Resolved issues
+---------------
+
+* :vytask:`T103` DHCP server prepends shared network name to hostnames
+* :vytask:`T125` Missing PPPoE interfaces in l2tp configuration
+* :vytask:`T1194` cronjob is being setup even if not saved
+* :vytask:`T1205` module pcspkr missing
+* :vytask:`T1219` Redundant active-active configuration, asymmetric routing and
+  conntrack-sync cache
+* :vytask:`T1220` Show transceiver information from plugin modules, e.g SFP+,
+  QSFP
+* :vytask:`T1221` BGP - Default route injection is not processed by the specific
+  route-map
+* :vytask:`T1241` Remove of policy route throws CLI error
+* :vytask:`T1291` Under certain conditions the VTI will stay forever down
+* :vytask:`T1463` Missing command `show ip bgp scan` appears in command
+  completion
+* :vytask:`T1575` `show snmp mib ifmib` crashes with IndexError
+* :vytask:`T1699` Default net.ipv6.route.max_size 32768 is too low
+* :vytask:`T1729` PIM (Protocol Independent Multicast) implementation
+* :vytask:`T1901` Semicolon in values is interpreted as a part of the shell
+  command by validators
+* :vytask:`T1934` Change default hostname when deploy from OVA without params.
+* :vytask:`T1938` syslog doesn't start automatically
+* :vytask:`T1949` Multihop IPv6 BFD is unconfigurable
+* :vytask:`T1953` DDNS service name validation rejects valid service names
+* :vytask:`T1956` PPPoE server: support PADO-delay
+* :vytask:`T1973` Allow route-map to match on BGP local preference value
+* :vytask:`T1974` Allow route-map to set administrative distance
+* :vytask:`T1982` Increase rotation for atop.acct
+* :vytask:`T1983` Expose route-map when BGP routes are programmed in to FIB
+* :vytask:`T1985` pppoe: Enable ipv6 modules without configured ipv6 pools
+* :vytask:`T2000` strongSwan does not install routes to table 220 in certain
+  cases
+* :vytask:`T2021` OSPFv3 doesn't support decimal area syntax
+* :vytask:`T2062` Wrong dhcp-server static route subnet bytes
+* :vytask:`T2091` swanctl.conf file is not generated properly is more than one
+  IPsec profile is used
+* :vytask:`T2131` Improve syslog remote host CLI definition
+* :vytask:`T2224` Update Linux Kernel to v4.19.114
+* :vytask:`T2286` IPoE server vulnerability
+* :vytask:`T2303` Unable to delete the image version that came from OVA
+* :vytask:`T2305` Add release name to "show version" command
+* :vytask:`T2311` Statically configured name servers may not take precedence
+  over ones from DHCP
+* :vytask:`T2327` Unable to create syslog server entry with different port
+* :vytask:`T2332` Backport node option for a syslog server
+* :vytask:`T2342` Bridge l2tpv3 + ethX errors
+* :vytask:`T2344` PPPoE server client static IP assignment silently fails
+* :vytask:`T2385` salt-minion: improve completion helpers
+* :vytask:`T2389` BGP community-list unknown command
+* :vytask:`T2398` op-mode "dhcp client leases interface" completion helper
+  misses interfaces
+* :vytask:`T2402` Live ISO should warn when configuring that changes won't
+  persist
+* :vytask:`T2443` NHRP: Add debugging information to syslog
+* :vytask:`T2448` `monitor protocol bgp` subcommands fail with 'command
+  incomplete'
+* :vytask:`T2458` Update FRR to 7.3.1
+* :vytask:`T2476` Bond member description change leads to network outage
+* :vytask:`T2478` login radius: use NAS-IP-Address if defined source address
+* :vytask:`T2482` Update PowerDNS recursor to 4.3.1 for CVE-2020-10995
+* :vytask:`T2517` vyos-container: link_filter: No such file or directory
+* :vytask:`T2526` Wake-On-Lan CLI implementation
+* :vytask:`T2528` "update dns dynamic" throws FileNotFoundError excepton
+* :vytask:`T2536` "show log dns forwarding" still refers to dnsmasq
+* :vytask:`T2538` Update Intel NIC drivers to recent release (preparation for
+  Kernel >=5.4)
+* :vytask:`T2545` Show physical device offloading capabilities for specified
+  ethernet interface
+* :vytask:`T2563` Wrong interface binding for Dell VEP 1445
+* :vytask:`T2605` SNMP service is not disabled by default
+* :vytask:`T2625` Provide generic Library for package builds
+* :vytask:`T2686` FRR: BGP: large-community configuration is not applied
+  properly after upgrading FRR to 7.3.x series
+* :vytask:`T2701` `vpn ipsec pfs enable` doesn't work with IKE groups
+* :vytask:`T2728` Protocol option ignored for IPSec peers in transport mode
+* :vytask:`T2734` WireGuard: fwmark CLI definition is inconsistent
+* :vytask:`T2757` "show system image version" contains additional new-line
+  character breaking output
+* :vytask:`T2797` Update Linux Kernel to v4.19.139
+* :vytask:`T2822` Update Linux Kernel to v4.19.141
+* :vytask:`T2829` PPPoE server: mppe setting is implemented as node instead of
+  leafNode
+* :vytask:`T2831` Update Linux Kernel to v4.19.142
+* :vytask:`T2852` rename dynamic dns interface breaks ddclient.cache permissions
+* :vytask:`T2853` Intel QAT acceleration does not work

docs/changelog/index.rst

@@ -0,0 +1,18 @@
+.. _release-notes:
+
+
+#########
+Changelog
+#########
+
+
+.. toctree::
+   :maxdepth: 1
+   :includehidden:
+
+   1.2.6
+   1.2.5
+   1.2.4
+   1.2.3
+   1.2.2
+   1.2.1

docs/cli.rst

@@ -1,19 +1,18 @@
 .. _cli:
 
-###
-CLI
-###
+######################
+Command Line Interface
+######################
 
 The VyOS :abbr:`CLI (Command-Line Interface)` comprises an operational and a
 configuration mode.
 
 Operational Mode
-================
+################
 
 Operational mode allows for commands to perform operational system tasks and
 view system and service status, while configuration mode allows for the
-modification of system configuration. The list of all operational level commands
-is available at :ref:`operational_level_commands`.
+modification of system configuration.
 
 The CLI provides a built-in help system. In the CLI the ``?`` key may be used
 to display available commands. The ``TAB`` key can be used to auto-complete
@@ -73,10 +72,7 @@ When viewing in page mode the following commands are available:
    in the event that the output has lines which exceed the terminal size.
 
 Configuration Mode
-==================
-
-The list of all operational level commands is available at
-:ref:`configuration_level_commands`.
+##################
 
 To enter configuration mode use the ``configure`` command:
 
@@ -97,3 +93,737 @@ To enter configuration mode use the ``configure`` command:
 
 See the configuration section of this document for more information on
 configuration mode.
+
+
+.. _configuration-overview:
+
+######################
+Configuration Overview
+######################
+
+VyOS makes use of a unified configuration file for the entire system's
+configuration: ``/config/config.boot``. This allows easy template
+creation, backup, and replication of system configuration. A system can
+thus also be easily cloned by simply copying the required configuration
+files.
+
+Terminology
+###########
+
+live
+A VyOS system has three major types of configurations:
+
+* **Active** or **running configuration** is the system configuration
+  that is loaded  and currently active (used by VyOS). Any change in
+  the configuration will have to be committed to belong to the
+  active/running configuration.
+
+* **Working configuration** is the one that is currently being modified
+  in configuration mode. Changes made to the working configuration do
+  not go into effect until the changes are committed with the
+  :cfgcmd:`commit` command. At which time the working configuration will
+  become the active or running configuration.
+
+* **Saved configuration** is the one saved to a file using the
+  :cfgcmd:`save` command. It allows you to keep safe a configuration for
+  future uses. There can be multiple configuration files. The default or
+  "boot" configuration is saved and loaded from the file
+  ``/config/config.boot``.
+
+Seeing and navigating the configuration
+=======================================
+
+.. opcmd:: show configuration
+
+   View the current active configuration, also known as the running
+   configuration, from the operational mode.
+
+   .. code-block:: none
+
+     vyos@vyos:~$ show configuration
+     interfaces {
+         ethernet eth0 {
+             address dhcp
+             hw-id 00:53:00:00:aa:01
+         }
+         loopback lo {
+         }
+     }
+     service {
+         ssh {
+             port 22
+         }
+     }
+     system {
+         config-management {
+             commit-revisions 20
+         }
+         console {
+             device ttyS0 {
+                 speed 9600
+             }
+         }
+         login {
+             user vyos {
+                 authentication {
+                     encrypted-password ****************
+                 }
+                 level admin
+             }
+         }
+         ntp {
+             server 0.pool.ntp.org {
+             }
+             server 1.pool.ntp.org {
+             }
+             server 2.pool.ntp.org {
+             }
+         }
+         syslog {
+             global {
+                 facility all {
+                     level notice
+                 }
+                 facility protocols {
+                     level debug
+                 }
+             }
+         }
+     }
+
+By default, the configuration is displayed in a hierarchy like the above
+example, this is only one of the possible ways to display the
+configuration. When the configuration is generated and the device is
+configured, changes are added through a collection of :cfgcmd:`set` and
+:cfgcmd:`delete` commands.
+
+.. opcmd:: show configuration commands
+
+   Get a collection of all the set commands required which led to the
+   running configuration.
+
+   .. code-block:: none
+
+     vyos@vyos:~$ show configuration commands
+     set interfaces ethernet eth0 address 'dhcp'
+     set interfaces ethernet eth0 hw-id '00:53:dd:44:3b:0f'
+     set interfaces loopback 'lo'
+     set service ssh port '22'
+     set system config-management commit-revisions '20'
+     set system console device ttyS0 speed '9600'
+     set system login user vyos authentication encrypted-password '$6$Vt68...QzF0'
+     set system login user vyos level 'admin'
+     set system ntp server '0.pool.ntp.org'
+     set system ntp server '1.pool.ntp.org'
+     set system ntp server '2.pool.ntp.org'
+     set system syslog global facility all level 'notice'
+     set system syslog global facility protocols level 'debug'
+
+Both these ``show`` commands should be executed when in operational
+mode, they do not work directly in configuration mode. There is a
+special way on how to :ref:`run_opmode_from_config_mode`.
+
+.. hint:: Use the ``show configuration commands | strip-private``
+   command when you want to hide private data. You may want to do so if
+   you want to share your configuration on the `forum`_.
+
+.. _`forum`: https://forum.vyos.io
+
+
+The config mode
+---------------
+
+When entering the configuration mode you are navigating inside a tree
+structure, to enter configuration mode enter the command
+:opcmd:`configure` when in operational mode.
+
+.. code-block:: none
+
+  vyos@vyos$ configure
+  [edit]
+  vyos@vyos#
+
+
+.. note:: When going into configuration mode, prompt changes from
+   ``$`` to ``#``.
+
+
+All commands executed here are relative to the configuration level you
+have entered. You can do everything from the top level, but commands
+will be quite lengthy when manually typing them.
+
+The current hierarchy level can be changed by the :cfgcmd:`edit`
+command.
+
+.. code-block:: none
+
+  [edit]
+  vyos@vyos# edit interfaces ethernet eth0
+
+  [edit interfaces ethernet eth0]
+  vyos@vyos#
+
+You are now in a sublevel relative to ``interfaces ethernet eth0``, all
+commands executed from this point on are relative to this sublevel. Use
+eithe the :cfgcmd:`top` or :cfgcmd:`exit` command to go back to the top
+of the hierarchy. You can also use the :cfgcmd:`up` command to move only
+one level up at a time.
+
+.. cfgcmd:: show
+
+The :cfgcmd:`show` command within configuration mode will show the
+working configuration indicating line changes with ``+`` for additions,
+``>`` for replacements and ``-`` for deletions.
+
+**Example:**
+
+.. code-block:: none
+
+ vyos@vyos:~$ configure
+ [edit]
+ vyos@vyos# show interfaces
+  ethernet eth0 {
+      description MY_OLD_DESCRIPTION
+      disable
+      hw-id 00:53:dd:44:3b:03
+  }
+  loopback lo {
+  }
+ [edit]
+ vyos@vyos# set interfaces ethernet eth0 address dhcp
+ [edit]
+ vyos@vyos# set interfaces ethernet eth0 description MY_NEW_DESCRIPTION
+ [edit]
+ vyos@vyos# delete interfaces ethernet eth0 disable
+ [edit]
+ vyos@vyos# show interfaces
+  ethernet eth0 {
+ +    address dhcp
+ >    description MY_NEW_DESCRIPTION
+ -    disable
+      hw-id 00:53:dd:44:3b:03
+  }
+  loopback lo {
+  }
+
+It is also possible to display all `set` commands within configuration
+mode using :cfgcmd:`show | commands`
+
+.. code-block:: none
+
+  vyos@vyos# show interfaces ethernet eth0 | commands
+  set address dhcp
+  set hw-id 00:53:ad:44:3b:03
+
+These commands are also relative to the level you are inside and only 
+relevant configuration blocks will be displayed when entering a
+sub-level.
+
+.. code-block:: none
+
+  [edit interfaces ethernet eth0]
+  vyos@vyos# show
+   address dhcp
+   hw-id 00:53:ad:44:3b:03
+
+Exiting from the configuration mode is done via the :cfgcmd:`exit`
+command from the top level, executing :cfgcmd:`exit` from within a
+sub-level takes you back to the top level.
+
+.. code-block:: none
+
+  [edit interfaces ethernet eth0]
+  vyos@vyos# exit
+  [edit]
+  vyos@vyos# exit
+  Warning: configuration changes have not been saved.
+
+
+Editing the configuration
+=========================
+
+The configuration can be edited by the use of :cfgcmd:`set` and
+:cfgcmd:`delete` commands from within configuration mode.
+
+.. cfgcmd:: set
+
+   Use this command to set the value of a parameter or to create a new
+   element.
+
+Configuration commands are flattened from the tree into 'one-liner'
+commands shown in :opcmd:`show configuration commands` from operation
+mode. Commands are relative to the level where they are executed and all
+redundant information from the current level is removed from the command
+entered.
+
+.. code-block:: none
+
+  [edit]
+  vyos@vyos# set interface ethernet eth0 address 192.0.2.100/24
+
+
+.. code-block:: none
+
+  [edit interfaces ethernet eth0]
+  vyos@vyos# set address 203.0.113.6/24
+
+
+These two commands above are essentially the same, just executed from
+different levels in the hierarchy.
+
+.. cfgcmd:: delete
+
+   To delete a configuration entry use the :cfgcmd:`delete` command,
+   this also deletes all sub-levels under the current level you've
+   specified in the :cfgcmd:`delete` command. Deleting an entry will
+   also result in the element reverting back to its default value if one
+   exists.
+
+   .. code-block:: none
+
+     [edit interfaces ethernet eth0]
+     vyos@vyos# delete address 192.0.2.100/24
+
+.. cfgcmd:: commit
+
+  Any change you do on the configuration, will not take effect until
+  committed using the :cfgcmd:`commit` command in configuration mode.
+
+  .. code-block:: none
+
+    vyos@vyos# commit
+    [edit]
+    vyos@vyos# exit
+    Warning: configuration changes have not been saved.
+    vyos@vyos:~$
+
+.. _save:
+
+.. cfgcmd:: save
+
+   Use this command to preserve configuration changes upon reboot. By
+   default it is stored at */config/config.boot*. In the case you want
+   to store the configuration file somewhere else, you can add a local
+   path, an SCP address, an FTP address or a TFTP address. 
+
+   .. code-block:: none
+
+     vyos@vyos# save
+     Saving configuration to '/config/config.boot'...
+     Done
+
+   .. code-block:: none
+
+     vyos@vyos# save [tab]
+     Possible completions:
+       <Enter>       Save to system config file
+       <file>        Save to file on local machine
+       scp://<user>:<passwd>@<host>:/<file> Save to file on remote machine
+       ftp://<user>:<passwd>@<host>/<file> Save to file on remote machine
+       tftp://<host>/<file>      Save to file on remote machine
+     vyos@vyos# save tftp://192.168.0.100/vyos-test.config.boot
+     Saving configuration to 'tftp://192.168.0.100/vyos-test.config.boot'...
+     ######################################################################## 100.0%
+     Done
+
+.. cfgcmd:: exit [discard]
+
+   Configuration mode can not be exited while uncommitted changes exist.
+   To exit configuration mode without applying changes, the
+   :cfgcmd:`exit discard` command must be used.
+
+   All changes in the working config will thus be lost.
+
+   .. code-block:: none
+
+     vyos@vyos# exit
+     Cannot exit: configuration modified.
+     Use 'exit discard' to discard the changes and exit.
+     [edit]
+     vyos@vyos# exit discard
+
+
+.. cfgcmd:: commit-confirm <minutes>
+
+   Use this command to temporarily commit your changes and set the
+   number of minutes available for validation. ``confirm`` must
+   be entered within those minutes, otherwise the system will reboot
+   into the previous configuration. The default value is 10 minutes.
+
+
+   What if you are doing something dangerous? Suppose you want to setup
+   a firewall, and you are not sure there are no mistakes that will lock
+   you out of your system. You can use confirmed commit. If you issue
+   the ``commit-confirm`` command, your changes will be commited, and if
+   you don't issue issue the ``confirm`` command in 10 minutes, your
+   system will reboot into previous config revision.
+
+   .. code-block:: none
+   
+      vyos@router# set interfaces ethernet eth0 firewall local name FromWorld
+      vyos@router# commit-confirm 
+      commit confirm will be automatically reboot in 10 minutes unless confirmed
+      Proceed? [confirm]y
+      [edit]
+      vyos@router# confirm 
+      [edit]
+
+
+   .. note:: A reboot because you did not enter ``confirm`` will not
+      take you necessarily to the *saved configuration*, but to the
+      point before the unfortunate commit.
+
+
+.. cfgcmd:: copy
+
+   Copy a configuration element.
+
+   You can copy and remove configuration subtrees. Suppose you set up a
+   firewall ruleset ``FromWorld`` with one rule that allows traffic from
+   specific subnet. Now you want to setup a similar rule, but for
+   different subnet. Change your edit level to
+   ``firewall name FromWorld`` and use ``copy rule 10 to rule 20``, then
+   modify rule 20.
+
+
+   .. code-block:: none
+   
+      vyos@router# show firewall name FromWorld 
+       default-action drop
+       rule 10 {
+           action accept
+           source {
+               address 203.0.113.0/24
+           }
+       }
+      [edit]
+      vyos@router# edit firewall name FromWorld 
+      [edit firewall name FromWorld]
+      vyos@router# copy rule 10 to rule 20
+      [edit firewall name FromWorld]
+      vyos@router# set rule 20 source address 198.51.100.0/24
+      [edit firewall name FromWorld]
+      vyos@router# commit
+      [edit firewall name FromWorld]
+
+
+.. cfgcmd:: rename
+
+   Rename a configuration element.
+
+   You can also rename config subtrees:
+
+   .. code-block:: none
+   
+      vyos@router# rename rule 10 to rule 5
+      [edit firewall name FromWorld]
+      vyos@router# commit
+      [edit firewall name FromWorld]
+
+   Note that ``show`` command respects your edit level and from this
+   level you can view the modified firewall ruleset with just ``show``
+   with no parameters.
+
+   .. code-block:: none
+   
+      vyos@router# show 
+       default-action drop
+       rule 5 {
+           action accept
+           source {
+               address 203.0.113.0/24
+           }
+       }
+       rule 20 {
+           action accept
+           source {
+               address 198.51.100.0/24
+           }
+       }
+
+
+.. cfgcmd:: comment <config node> "comment text"
+
+   Add comment as an annotation to a configuration node.
+
+   The ``comment`` command allows you to insert a comment above the
+   ``<config node>`` configuration section. When shown, comments are
+   enclosed with ``/*`` and ``*/`` as open/close delimiters. Comments
+   need to be commited, just like other config changes.
+
+   To remove an existing comment from your current configuration,
+   specify an empty string enclosed in double quote marks (``""``) as
+   the comment text.
+
+   Example:
+
+   .. code-block:: none
+
+     vyos@vyos# comment firewall all-ping "Yes I know this VyOS is cool"
+     vyos@vyos# commit
+     vyos@vyos# show
+      firewall {
+          /* Yes I know this VyOS is cool */
+          all-ping enable
+          broadcast-ping disable
+          ...
+      }
+
+   .. note:: An important thing to note is that since the comment is
+      added on top of the section, it will not appear if the ``show
+      <section>`` command is used. With the above example, the `show
+      firewall` command would return starting after the ``firewall
+      {`` line, hiding the comment.
+
+
+
+
+   
+
+.. _run_opmode_from_config_mode:
+
+Access opmode from config mode
+==============================
+
+When inside configuration mode you are not directly able to execute
+operational commands.
+
+.. cfgcmd:: run
+
+  Access to these commands are possible through the use of the
+  ``run [command]`` command. From this command you will have access to
+  everything accessible from operational mode.
+
+  Command completion and syntax help with ``?`` and ``[tab]`` will also
+  work.
+
+  .. code-block:: none
+
+    [edit]
+    vyos@vyos# run show interfaces
+    Codes: S - State, L - Link, u - Up, D - Down, A - Admin Down
+    Interface        IP Address                        S/L  Description
+    ---------        ----------                        ---  -----------
+    eth0             0.0.0.0/0                         u/u
+
+Managing configurations
+=======================
+
+VyOS comes with an integrated versioning system for the system
+configuration. It automatically maintains a backup of every previous
+configuration which has been committed to the system. The configurations
+are versioned locally for rollback but they can also be stored on a
+remote host for archiving/backup reasons.
+
+Local Archive
+-------------
+
+Revisions are stored on disk. You can view, compare and rollback them to
+any previous revisions if something goes wrong.
+
+.. opcmd:: show system commit
+
+   View all existing revisions on the local system.
+
+   .. code-block:: none
+
+     vyos@vyos:~$ show system commit
+     0   2015-03-30 08:53:03 by vyos via cli
+     1   2015-03-30 08:52:20 by vyos via cli
+     2   2015-03-26 21:26:01 by root via boot-config-loader
+     3   2015-03-26 20:43:18 by root via boot-config-loader
+     4   2015-03-25 11:06:14 by root via boot-config-loader
+     5   2015-03-25 01:04:28 by root via boot-config-loader
+     6   2015-03-25 00:16:47 by vyos via cli
+     7   2015-03-24 23:43:45 by root via boot-config-loader
+
+
+.. cfgcmd:: set system config-management commit-revisions <N>
+
+   You can specify the number of revisions stored on disk. N can be in
+   the range of 0 - 65535. When the number of revisions exceeds the
+   configured value, the oldest revision is removed. The default setting
+   for this value is to store 100 revisions locally.
+
+
+Compare configurations
+----------------------
+
+VyOS lets you compare different configurations.
+
+.. cfgcmd:: compare <saved | N> <M>
+
+   Use this command to spot what the differences are between different
+   configurations.
+
+   .. code-block:: none
+
+     vyos@vyos# compare [tab]
+     Possible completions:
+       <Enter>	Compare working & active configurations
+       saved		Compare working & saved configurations
+       <N>		Compare working with revision N
+       <N> <M>	Compare revision N with M
+       Revisions:
+         0	   2013-12-17 20:01:37 root by boot-config-loader
+         1	   2013-12-13 15:59:31 root by boot-config-loader
+         2	   2013-12-12 21:56:22 vyos by cli
+         3	   2013-12-12 21:55:11 vyos by cli
+         4	   2013-12-12 21:27:54 vyos by cli
+         5	   2013-12-12 21:23:29 vyos by cli
+         6	   2013-12-12 21:13:59 root by boot-config-loader
+         7	   2013-12-12 16:25:19 vyos by cli
+         8	   2013-12-12 15:44:36 vyos by cli
+         9	   2013-12-12 15:42:07 root by boot-config-loader
+         10   2013-12-12 15:42:06 root by init
+
+   The command :cfgcmd:`compare` allows you to compare different type of
+   configurations. It also lets you compare different revisions through
+   the :cfgcmd:`compare N M` command, where N and M are revision
+   numbers. The output will describe how the configuration N is when
+   compared to M indicating with a plus sign (``+``) the additional
+   parts N has when compared to M, and indicating with a minus sign
+   (``-``) the lacking parts N misses when compared to M.
+
+   .. code-block:: none
+
+     vyos@vyos# compare 0 6
+     [edit interfaces]
+     +dummy dum1 {
+     +    address 10.189.0.1/31
+     +}
+     [edit interfaces ethernet eth0]
+     +vif 99 {
+     +    address 10.199.0.1/31
+     +}
+     -vif 900 {
+     -    address 192.0.2.4/24
+     -}
+
+
+.. opcmd:: show system commit diff <number>
+
+   Show commit revision difference.
+
+
+The command above also lets you see the difference between two commits.
+By default the difference with the running config is shown.
+
+.. code-block:: none
+
+   vyos@router# run show system commit diff 4
+   [edit system]
+   +ipv6 {
+   +    disable-forwarding
+   +}
+
+This means four commits ago we did ``set system ipv6 disable-forwarding``.
+
+
+Rollback Changes
+----------------
+
+You can rollback configuration changes using the rollback command. This
+will apply the selected revision and trigger a system reboot.
+
+.. cfgcmd:: rollback <N>
+
+   Rollback to revision N (currently requires reboot)
+
+   .. code-block:: none
+
+     vyos@vyos# compare 1
+     [edit system]
+     >host-name vyos-1
+     [edit]
+
+     vyos@vyos# rollback 1
+     Proceed with reboot? [confirm][y]
+     Broadcast message from root@vyos-1 (pts/0) (Tue Dec 17 21:07:45 2013):
+     The system is going down for reboot NOW!
+
+Remote Archive
+--------------
+
+VyOS can upload the configuration to a remote location after each call
+to :cfgcmd:`commit`. You will have to set the commit-archive location.
+TFTP, FTP, SCP and SFTP servers are supported. Every time a
+:cfgcmd:`commit` is successfull the ``config.boot`` file will be copied
+to the defined destination(s). The filename used on the remote host will
+be ``config.boot-hostname.YYYYMMDD_HHMMSS``. 
+
+.. cfgcmd:: set system config-management commit-archive location <URI>
+
+   Specify remote location of commit archive as any of the below
+   :abbr:`URI (Uniform Resource Identifier)`
+
+   * ``scp://<user>:<passwd>@<host>:/<dir>``
+   * ``sftp://<user>:<passwd>@<host>/<dir>``
+   * ``ftp://<user>:<passwd>@<host>/<dir>``
+   * ``tftp://<host>/<dir>``
+
+.. note:: The number of revisions don't affect the commit-archive.
+
+.. note:: You may find VyOS not allowing the secure connection because
+   it cannot verify the legitimacy of the remote server. You can use
+   the workaround below to quickly add the remote host's SSH
+   fingerprint to your ``~/.ssh/known_hosts`` file:
+
+   .. code-block:: none
+
+     vyos@vyos# ssh-keyscan <host> >> ~/.ssh/known_hosts
+
+Saving and loading manually
+---------------------------
+
+You can use the ``save`` and ``load`` commands if you want to manually
+manage specific configuration files.
+
+When using the save_ command, you can add a specific location where
+to store your configuration file. And, when needed it, you will be able
+to load it with the ``load`` command:
+
+.. cfgcmd:: load <URI>
+
+   Use this command to load a configuration which will replace the
+   running configuration. Define the location of the configuration file
+   to be loaded. You can use a path to a local file, an SCP address, an
+   SFTP address, an FTP address, an HTTP address, an HTTPS address or a
+   TFTP address.
+
+  .. code-block:: none
+
+     vyos@vyos# load 
+     Possible completions:
+       <Enter>				        Load from system config file
+       <file>			        	Load from file on local machine
+       scp://<user>:<passwd>@<host>:/<file>	Load from file on remote machine
+       sftp://<user>:<passwd>@<host>/<file>	Load from file on remote machine
+       ftp://<user>:<passwd>@<host>/<file>	Load from file on remote machine
+       http://<host>/<file>			Load from file on remote machine
+       https://<host>/<file>			Load from file on remote machine
+       tftp://<host>/<file>			Load from file on remote machine
+     
+
+
+Restore Default
+---------------
+
+In the case you want to completely delete your configuration and restore
+the default one, you can enter the following command in configuration
+mode:
+
+.. code-block:: none
+
+  load /opt/vyatta/etc/config.boot.default
+
+You will be asked if you want to continue. If you accept, you will have
+to use :cfgcmd:`commit` if you want to make the changes active.
+
+Then you may want to :cfgcmd:`save` in order to delete the saved
+configuration too.
+
+.. note:: If you are remotely connected, you will lose your connection.
+   You may want to copy first the config, edit it to ensure
+   connectivity, and load the edited config.
+

docs/command-list-configuration.rst

@@ -1,7 +0,0 @@
-.. _configuration_level_commands:
-
-********************************
-Configuration Level Command List
-********************************
-
-.. cfgcmdlist::