prepare coverage.rst

docs/_ext/testcoverage.py

@@ -0,0 +1,351 @@
+'''
+generate json with all commands from xml for vyos documentation coverage
+
+'''
+
+
+import sys
+import os
+import json
+import re
+import logging
+
+from io import BytesIO
+from lxml import etree as ET
+import shutil
+
+default_constraint_err_msg = "Invalid value"
+validator_dir = ""
+
+
+input_data = [
+    {
+        "kind": "cfgcmd",
+        "input_dir": "_include/vyos-1x/interface-definitions/",
+        "schema_file": "_include/vyos-1x/schema/interface_definition.rng",
+        "files": []
+    },
+    {
+        "kind": "opcmd",
+        "input_dir": "_include/vyos-1x/op-mode-definitions/",
+        "schema_file": "_include/vyos-1x/schema/op-mode-definition.rng",
+        "files": []
+    }
+]
+
+node_data = {
+    'cfgcmd': {},
+    'opcmd': {},
+}
+
+def get_properties(p):
+    props = {}
+    props['valueless'] = False
+
+    try:
+        if p.find("valueless") is not None:
+            props['valueless'] = True
+    except:
+        pass
+
+    if p is None:
+        return props
+
+    # Get the help string
+    try:
+        props["help"] = p.find("help").text
+    except:
+        pass
+
+    # Get value help strings
+    try:
+        vhe = p.findall("valueHelp")
+        vh = []
+        for v in vhe:
+            vh.append( (v.find("format").text, v.find("description").text) )
+        props["val_help"] = vh
+    except:
+        props["val_help"] = []
+
+    # Get the constraint statements
+    error_msg = default_constraint_err_msg
+    # Get the error message if it's there
+    try:
+        error_msg = p.find("constraintErrorMessage").text
+    except:
+        pass
+    
+
+    vce = p.find("constraint")
+    vc = []
+    if vce is not None:
+        # The old backend doesn't support multiple validators in OR mode
+        # so we emulate it
+
+        regexes = []
+        regex_elements = vce.findall("regex")
+        if regex_elements is not None:
+            regexes = list(map(lambda e: e.text.strip(), regex_elements))
+        if "" in regexes:
+            print("Warning: empty regex, node will be accepting any value")
+
+        validator_elements = vce.findall("validator")
+        validators = []
+        if validator_elements is not None:
+            for v in validator_elements:
+                v_name = os.path.join(validator_dir, v.get("name"))
+
+                # XXX: lxml returns None for empty arguments
+                v_argument = None
+                try:
+                    v_argument = v.get("argument")
+                except:
+                    pass
+                if v_argument is None:
+                    v_argument = ""
+
+                validators.append("{0} {1}".format(v_name, v_argument))
+
+
+        regex_args = " ".join(map(lambda s: "--regex \\\'{0}\\\'".format(s), regexes))
+        validator_args = " ".join(map(lambda s: "--exec \\\"{0}\\\"".format(s), validators))
+        validator_script = '${vyos_libexec_dir}/validate-value.py'
+        validator_string = "exec \"{0} {1} {2} --value \\\'$VAR(@)\\\'\"; \"{3}\"".format(validator_script, regex_args, validator_args, error_msg)
+
+        props["constraint"] = validator_string
+
+    # Get the completion help strings
+    try:
+        che = p.findall("completionHelp")
+        ch = ""
+        for c in che:
+            scripts = c.findall("script")
+            paths = c.findall("path")
+            lists = c.findall("list")
+
+            # Current backend doesn't support multiple allowed: tags
+            # so we get to emulate it
+            comp_exprs = []
+            for i in lists:
+                comp_exprs.append("echo \"{0}\"".format(i.text))
+            for i in paths:
+                comp_exprs.append("/bin/cli-shell-api listNodes {0}".format(i.text))
+            for i in scripts:
+                comp_exprs.append("sh -c \"{0}\"".format(i.text))
+            comp_help = " && ".join(comp_exprs)
+            props["comp_help"] = comp_help
+    except:
+        props["comp_help"] = []
+
+    # Get priority
+    try:
+        props["priority"] = p.find("priority").text
+    except:
+        pass
+
+    # Get "multi"
+    if p.find("multi") is not None:
+        props["multi"] = True
+
+    # Get "valueless"
+    if p.find("valueless") is not None:
+        props["valueless"] = True
+
+    return props
+
+def process_node(n, f):
+
+    props_elem = n.find("properties")
+    children = n.find("children")
+    command = n.find("command")
+    children_nodes = []
+    owner = n.get("owner")
+    node_type = n.tag
+
+    name = n.get("name")
+    props = get_properties(props_elem)
+
+    if node_type != "node":
+        if "valueless" not in props.keys():
+            props["type"] = "txt"
+    if node_type == "tagNode":
+        props["tag"] = "True"
+    
+    if node_type == "node" and children is not None:
+        inner_nodes = children.iterfind("*")
+        index_child = 0
+        for inner_n in inner_nodes:
+            children_nodes.append(process_node(inner_n, f))
+            index_child = index_child + 1
+
+    if node_type == "tagNode" and children is not None:
+        inner_nodes = children.iterfind("*")
+        index_child = 0
+        for inner_n in inner_nodes:
+            children_nodes.append(process_node(inner_n, f))
+            index_child = index_child + 1
+    else:
+        # This is a leaf node
+        pass
+    
+    if command is not None:
+        test_command = True
+    else:
+        test_command = False
+    node = {
+        'name': name,
+        'type': node_type,
+        'children': children_nodes,
+        'props': props,
+        'command': test_command,
+        'filename': f
+    }
+    return node
+
+
+
+def create_commands(data, parent_list=[], level=0):
+    result = []
+    command = {
+        'name': [],
+        'help': None,
+        'tag_help': [],
+        'level': level,
+        'no_childs': False,
+        'filename': None
+    }
+    command['filename'] = data['filename']
+    command['name'].extend(parent_list)
+    command['name'].append(data['name'])
+
+    if data['type'] == 'tagNode':
+        command['name'].append("<" + data['name'] + ">")
+
+    if 'val_help' in data['props'].keys():
+        for val_help in data['props']['val_help']:
+            command['tag_help'].append(val_help)
+    
+    if len(data['children']) == 0:
+        command['no_childs'] = True
+    
+    if data['command']:
+        command['no_childs'] = True
+    
+    try:
+        help_text = data['props']['help']
+        command['help'] = re.sub(r"[\n\t]*", "", help_text)
+        
+    except:
+        command['help'] = ""
+    
+    command['valueless'] = data['props']['valueless']
+    
+    if 'children' in data.keys():
+        children_bool = True
+        for child in data['children']:
+            result.extend(create_commands(child, command['name'], level + 1))
+    
+    if command['no_childs']:
+        result.append(command)
+    
+
+
+    return result
+
+
+def include_file(line, input_dir):
+    string = ""
+    if "#include <include" in line.strip():
+        include_filename = line.strip().split('<')[1][:-1]
+        with open(input_dir + include_filename) as ifp:
+            iline = ifp.readline()
+            while iline:
+                string = string + include_file(iline.strip(), input_dir)
+                iline = ifp.readline()
+    else:
+        string = line
+    return string
+
+
+def get_working_commands():
+    for entry in input_data:
+        for (dirpath, dirnames, filenames) in os.walk(entry['input_dir']):
+            entry['files'].extend(filenames)
+            break
+
+        for f in entry['files']:
+
+            string = ""
+            with open(entry['input_dir'] + f) as fp:
+                line = fp.readline()
+                while line:                
+                    string = string + include_file(line.strip(), entry['input_dir'])
+                    line = fp.readline()
+            
+            try:
+                xml = ET.parse(BytesIO(bytes(string, 'utf-8')))
+            except Exception as e:
+                print("Failed to load interface definition file {0}".format(f))
+                print(e)
+                sys.exit(1)
+            
+            try:
+                relaxng_xml = ET.parse(entry['schema_file'])
+                validator = ET.RelaxNG(relaxng_xml)
+
+                if not validator.validate(xml):
+                    print(validator.error_log)
+                    print("Interface definition file {0} does not match the schema!".format(f))
+                    sys.exit(1)
+            except Exception as e:
+                print("Failed to load the XML schema {0}".format(entry['schema_file']))
+                print(e)
+                sys.exit(1)
+
+            root = xml.getroot()
+            nodes = root.iterfind("*")
+            for n in nodes:
+                node_data[entry['kind']][f] = process_node(n, f)
+
+    # build config tree and sort
+
+    config_tree_new = {
+        'cfgcmd': {},
+        'opcmd': {},
+    }
+
+    for kind in node_data:
+        for entry in node_data[kind]:
+            node_0 = node_data[kind][entry]['name']
+            
+            if node_0 not in config_tree_new[kind].keys():
+                config_tree_new[kind][node_0] = {
+                    'name': node_0,
+                    'type': node_data[kind][entry]['type'],
+                    'props': node_data[kind][entry]['props'],
+                    'children': [],
+                    'command': node_data[kind][entry]['command'],
+                    'filename': node_data[kind][entry]['filename'],
+                }
+            config_tree_new[kind][node_0]['children'].extend(node_data[kind][entry]['children'])
+    
+    result = {
+        'cfgcmd': [],
+        'opcmd': [],
+    }
+    for kind in  config_tree_new:
+        for e in config_tree_new[kind]:
+            result[kind].extend(create_commands(config_tree_new[kind][e]))
+    
+    for cmd in result['cfgcmd']:
+        cmd['cmd'] = " ".join(cmd['name'])
+    for cmd in result['opcmd']:
+        cmd['cmd'] = " ".join(cmd['name'])
+    return result
+
+
+
+if __name__ == "__main__":
+    res = get_working_commands()
+    print(json.dumps(res))
+    #print(res['cfgcmd'][0])

docs/_ext/vyos.py

@@ -1,25 +1,41 @@
 import re
-import io
+import json
 import os
 from docutils import io, nodes, utils, statemachine
-from docutils.utils.error_reporting import SafeString, ErrorString
 from docutils.parsers.rst.roles import set_classes
 from docutils.parsers.rst import Directive, directives
+
 from sphinx.util.docutils import SphinxDirective
 
+from testcoverage import get_working_commands
+
 
 def setup(app):
 
     app.add_config_value(
         'vyos_phabricator_url',
-        'https://phabricator.vyos.net/', ''
+        'https://phabricator.vyos.net/',
+        'html'
     )
+
+    app.add_config_value(
+        'vyos_working_commands',
+        get_working_commands(),
+        'html'
+    )
+    app.add_config_value(
+        'vyos_coverage',
+        {
+            'cfgcmd': [0,len(app.config.vyos_working_commands['cfgcmd'])],
+            'opcmd': [0,len(app.config.vyos_working_commands['opcmd'])]
+        },
+        'html'
+    )
+
     app.add_role('vytask', vytask_role)
     app.add_role('cfgcmd', cmd_role)
     app.add_role('opcmd', cmd_role)
 
-    print(app.config.vyos_phabricator_url)
-
     app.add_node(
         inlinecmd,
         html=(inlinecmd.visit_span, inlinecmd.depart_span),
@@ -46,9 +62,11 @@ def setup(app):
         text=(CmdHeader.visit_div, CmdHeader.depart_div)
     )
     app.add_node(CfgcmdList)
+    app.add_node(CfgcmdListCoverage)
     app.add_directive('cfgcmdlist', CfgcmdlistDirective)
 
     app.add_node(OpcmdList)
+    app.add_node(OpcmdListCoverage)
     app.add_directive('opcmdlist', OpcmdlistDirective)
 
     app.add_directive('cfgcmd', CfgCmdDirective)
@@ -56,15 +74,17 @@ def setup(app):
     app.add_directive('cmdinclude', CfgInclude)
     app.connect('doctree-resolved', process_cmd_nodes)
 
-
 class CfgcmdList(nodes.General, nodes.Element):
     pass
 
-
 class OpcmdList(nodes.General, nodes.Element):
     pass
 
-import json
+class CfgcmdListCoverage(nodes.General, nodes.Element):
+    pass
+
+class OpcmdListCoverage(nodes.General, nodes.Element):
+    pass
 
 class CmdHeader(nodes.General, nodes.Element):
 
@@ -200,8 +220,8 @@ class CfgInclude(Directive):
                               '(wrong locale?).' %
                               (self.name, SafeString(path)))
         except IOError:
-            raise self.severe(u'Problems with "%s" directive path.' %
+            raise self.severe(u'Problems with "%s" directive path:\n%s.' %
-                      (self.name))
+                      (self.name, ErrorString(error)))
         startline = self.options.get('start-line', None)
         endline = self.options.get('end-line', None)
         try:
@@ -277,7 +297,16 @@ class CfgInclude(Directive):
             return codeblock.run()
         
         new_include_lines = []
-
+        var_value0 = self.options.get('var0', '')
+        var_value1 = self.options.get('var1', '')
+        var_value2 = self.options.get('var2', '')
+        var_value3 = self.options.get('var3', '')
+        var_value4 = self.options.get('var4', '')
+        var_value5 = self.options.get('var5', '')
+        var_value6 = self.options.get('var6', '')
+        var_value7 = self.options.get('var7', '')
+        var_value8 = self.options.get('var8', '')
+        var_value9 = self.options.get('var9', '')
         for line in include_lines:
             for i in range(10):
                 value = self.options.get(f'var{i}','')
@@ -285,22 +314,41 @@ class CfgInclude(Directive):
                     line = re.sub('\s?{{\s?var' + str(i) + '\s?}}',value,line)
                 else:
                     line = re.sub('{{\s?var' + str(i) + '\s?}}',value,line)
-
             new_include_lines.append(line)
         self.state_machine.insert_input(new_include_lines, path)
         return []
 
 
 class CfgcmdlistDirective(Directive):
+    has_content = False
+    required_arguments = 0
+    option_spec = {
+        'show-coverage': directives.flag
+    }
 
     def run(self):
-        return [CfgcmdList('')]
+        cfglist = CfgcmdList()
+        cfglist['coverage'] = False
+        if 'show-coverage' in self.options:
+            cfglist['coverage'] = True
+        return [cfglist]
 
 
 class OpcmdlistDirective(Directive):
+    has_content = False
+    required_arguments = 0
+    option_spec = {
+        'show-coverage': directives.flag
+    }
 
     def run(self):
-        return [OpcmdList('')]
+        oplist = OpcmdList()
+        oplist['coverage'] = False
+        if 'show-coverage' in self.options:
+            oplist['coverage'] = True
+            
+        return [oplist]
+
 
 
 class CmdDirective(SphinxDirective):
@@ -309,6 +357,7 @@ class CmdDirective(SphinxDirective):
     custom_class = ''
 
     def run(self):        
+
         title_list = []
         content_list = []
         title_text = ''
@@ -386,7 +435,134 @@ class CfgCmdDirective(CmdDirective):
     custom_class = 'cfg'
 
 
-def process_cmd_node(app, cmd, fromdocname):
+def strip_cmd(cmd):
+    #cmd = re.sub('set','',cmd)
+    cmd = re.sub('\s\|\s','',cmd)
+    cmd = re.sub('<\S*>','',cmd)
+    cmd = re.sub('\[\S\]','',cmd)
+    cmd = re.sub('\s+','',cmd)
+    return cmd
+
+def build_row(app, fromdocname, rowdata):
+    row = nodes.row()
+    for cell in rowdata:
+        entry = nodes.entry()
+        row += entry
+        if isinstance(cell, list):
+            for item in cell:
+                if isinstance(item, dict):
+                    entry += process_cmd_node(app, item, fromdocname, '')
+                else:
+                    entry += nodes.paragraph(text=item)
+        elif isinstance(cell, bool):
+            if cell:
+                entry += nodes.paragraph(text="")
+                entry['classes'] = ['coverage-ok']
+            else:
+                entry += nodes.paragraph(text="")
+                entry['classes'] = ['coverage-fail']
+        else:
+            entry += nodes.paragraph(text=cell)
+    return row
+
+
+
+def process_coverage(app, fromdocname, doccmd, xmlcmd, cli_type):
+    coverage_list = {}
+    int_docs = 0
+    int_xml = 0
+    for cmd in doccmd:
+        coverage_item = {
+            'doccmd': None,
+            'xmlcmd': None,
+            'doccmd_item': None,
+            'xmlcmd_item': None,
+            'indocs': False,
+            'inxml': False,
+            'xmlfilename': None
+        }
+        coverage_item['doccmd'] = cmd['cmd']
+        coverage_item['doccmd_item'] = cmd
+        coverage_item['indocs'] = True
+        int_docs += 1
+        coverage_list[strip_cmd(cmd['cmd'])] = dict(coverage_item)
+    
+    for cmd in xmlcmd:
+        
+        strip = strip_cmd(cmd['cmd'])
+        if strip not in coverage_list.keys():
+            coverage_item = {
+                'doccmd': None,
+                'xmlcmd': None,
+                'doccmd_item': None,
+                'xmlcmd_item': None,
+                'indocs': False,
+                'inxml': False,
+                'xmlfilename': None
+            }
+            coverage_item['xmlcmd'] = cmd['cmd']
+            coverage_item['xmlcmd_item'] = cmd
+            coverage_item['inxml'] = True
+            coverage_item['xmlfilename'] = cmd['filename']
+            int_xml += 1
+            coverage_list[strip] = dict(coverage_item)
+        else:
+            #print("===BEGIN===")
+            #print(cmd)
+            #print(coverage_list[strip])
+            #print(strip)
+            #print("===END====")
+            coverage_list[strip]['xmlcmd'] = cmd['cmd']
+            coverage_list[strip]['xmlcmd_item'] = cmd
+            coverage_list[strip]['inxml'] = True
+            coverage_list[strip]['xmlfilename'] = cmd['filename']
+            int_xml += 1
+
+
+    
+
+    table = nodes.table()
+    tgroup = nodes.tgroup(cols=3)
+    table += tgroup
+
+    header = (f'{int_docs}/{len(coverage_list)} in Docs', f'{int_xml}/{len(coverage_list)} in XML', 'Command')
+    colwidths = (1, 1, 8)
+    table = nodes.table()
+    tgroup = nodes.tgroup(cols=len(header))
+    table += tgroup
+    for colwidth in colwidths:
+        tgroup += nodes.colspec(colwidth=colwidth)
+    thead = nodes.thead()
+    tgroup += thead
+    thead += build_row(app, fromdocname, header)
+    tbody = nodes.tbody()
+    tgroup += tbody
+    for entry in sorted(coverage_list):
+        body_text_list = []
+        if coverage_list[entry]['indocs']:
+            body_text_list.append(coverage_list[entry]['doccmd_item'])
+        else:
+            body_text_list.append('Not documented yet')
+
+        if coverage_list[entry]['inxml']:
+            body_text_list.append("------------------")
+            body_text_list.append(str(coverage_list[entry]['xmlfilename']) + ":")
+            body_text_list.append(coverage_list[entry]['xmlcmd'])
+        else:
+            body_text_list.append('Nothing found in XML Definitions')
+
+            
+        tbody += build_row(app, fromdocname, 
+            (
+                coverage_list[entry]['indocs'],
+                coverage_list[entry]['inxml'],
+                body_text_list
+            )
+        )
+
+    return table
+
+def process_cmd_node(app, cmd, fromdocname, cli_type):
     para = nodes.paragraph()
     newnode = nodes.reference('', '')
     innernode = cmd['cmdnode']
@@ -401,22 +577,46 @@ def process_cmd_node(app, cmd, fromdocname):
 
 
 def process_cmd_nodes(app, doctree, fromdocname):
+    try:
         env = app.builder.env
         
         for node in doctree.traverse(CfgcmdList):
             content = []
-
+            if node.attributes['coverage']:
+                node.replace_self(
+                    process_coverage(
+                        app,
+                        fromdocname,
+                        env.vyos_cfgcmd,
+                        app.config.vyos_working_commands['cfgcmd'],
+                        'cfgcmd'
+                        )
+                    )
+            else:
                 for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']):
-            content.append(process_cmd_node(app, cmd, fromdocname))
+                    content.append(process_cmd_node(app, cmd, fromdocname, 'cfgcmd'))                
                 node.replace_self(content)
             
         for node in doctree.traverse(OpcmdList):
             content = []
-
+            if node.attributes['coverage']:
+                node.replace_self(
+                    process_coverage(
+                        app,
+                        fromdocname,
+                        env.vyos_opcmd,
+                        app.config.vyos_working_commands['opcmd'],
+                        'opcmd'
+                        )
+                    )
+            else:
                 for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']):
-            content.append(process_cmd_node(app, cmd, fromdocname))
+                    content.append(process_cmd_node(app, cmd, fromdocname, 'opcmd'))
                 node.replace_self(content)
 
+    except Exception as inst:
+        print(inst)
+
 
 def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     app = inliner.document.settings.env.app

docs/coverage.rst

@@ -0,0 +1,43 @@
+:orphan:
+
+########
+Coverage
+########
+
+Overview over all commands, which are documented in the ``.. cfgcmd::`` or ``.. opcmd::`` Directives.
+
+| The build process take all xml definition files from `vyos-1x <https://github.com/vyos/vyos-1x>`_  and extract each leaf command or executable command.
+| After this the commands are compare and shown in the follwoing two tables.
+| The script compare only the fixed part of a command. All varables or values will be erase and then compare:
+
+for example there are these two commands:
+
+  * documentation: ``interfaces ethernet <interface> address <address | dhcp | dhcpv6>```
+  * xml: ``interface ethernet <ethernet> address <address>``
+
+Now the script earse all in between ``<`` and ``>`` and simply compare the strings.
+
+**There are 2 kind of problems:**   
+
+| ``Not documented yet``
+| A XML command are not found in ``.. cfgcmd::`` or ``.. opcmd::`` Commands
+| The command should be documented
+
+| ``Nothing found in XML Definitions``: 
+| ``.. cfgcmd::`` or ``.. opcmd::`` Command are not found in a XML command
+| Maybe the command where changed in the XML Definition, or the feature is not anymore in VyOS
+| Some commands are not yet translated to XML
+
+
+Configuration Commands
+======================
+
+.. cfgcmdlist::
+    :show-coverage:
+
+
+Operational Commands
+====================
+
+.. opcmdlist::
+    :show-coverage: