prepare coverage.rst

This commit is contained in:
rebortg 2020-11-26 22:12:04 +01:00
parent ab14c6a43a
commit de9eaec2cf
3 changed files with 626 additions and 32 deletions

351
docs/_ext/testcoverage.py Normal file
View File

@ -0,0 +1,351 @@
'''
generate json with all commands from xml for vyos documentation coverage
'''
import sys
import os
import json
import re
import logging
from io import BytesIO
from lxml import etree as ET
import shutil
default_constraint_err_msg = "Invalid value"
validator_dir = ""
input_data = [
{
"kind": "cfgcmd",
"input_dir": "_include/vyos-1x/interface-definitions/",
"schema_file": "_include/vyos-1x/schema/interface_definition.rng",
"files": []
},
{
"kind": "opcmd",
"input_dir": "_include/vyos-1x/op-mode-definitions/",
"schema_file": "_include/vyos-1x/schema/op-mode-definition.rng",
"files": []
}
]
node_data = {
'cfgcmd': {},
'opcmd': {},
}
def get_properties(p):
props = {}
props['valueless'] = False
try:
if p.find("valueless") is not None:
props['valueless'] = True
except:
pass
if p is None:
return props
# Get the help string
try:
props["help"] = p.find("help").text
except:
pass
# Get value help strings
try:
vhe = p.findall("valueHelp")
vh = []
for v in vhe:
vh.append( (v.find("format").text, v.find("description").text) )
props["val_help"] = vh
except:
props["val_help"] = []
# Get the constraint statements
error_msg = default_constraint_err_msg
# Get the error message if it's there
try:
error_msg = p.find("constraintErrorMessage").text
except:
pass
vce = p.find("constraint")
vc = []
if vce is not None:
# The old backend doesn't support multiple validators in OR mode
# so we emulate it
regexes = []
regex_elements = vce.findall("regex")
if regex_elements is not None:
regexes = list(map(lambda e: e.text.strip(), regex_elements))
if "" in regexes:
print("Warning: empty regex, node will be accepting any value")
validator_elements = vce.findall("validator")
validators = []
if validator_elements is not None:
for v in validator_elements:
v_name = os.path.join(validator_dir, v.get("name"))
# XXX: lxml returns None for empty arguments
v_argument = None
try:
v_argument = v.get("argument")
except:
pass
if v_argument is None:
v_argument = ""
validators.append("{0} {1}".format(v_name, v_argument))
regex_args = " ".join(map(lambda s: "--regex \\\'{0}\\\'".format(s), regexes))
validator_args = " ".join(map(lambda s: "--exec \\\"{0}\\\"".format(s), validators))
validator_script = '${vyos_libexec_dir}/validate-value.py'
validator_string = "exec \"{0} {1} {2} --value \\\'$VAR(@)\\\'\"; \"{3}\"".format(validator_script, regex_args, validator_args, error_msg)
props["constraint"] = validator_string
# Get the completion help strings
try:
che = p.findall("completionHelp")
ch = ""
for c in che:
scripts = c.findall("script")
paths = c.findall("path")
lists = c.findall("list")
# Current backend doesn't support multiple allowed: tags
# so we get to emulate it
comp_exprs = []
for i in lists:
comp_exprs.append("echo \"{0}\"".format(i.text))
for i in paths:
comp_exprs.append("/bin/cli-shell-api listNodes {0}".format(i.text))
for i in scripts:
comp_exprs.append("sh -c \"{0}\"".format(i.text))
comp_help = " && ".join(comp_exprs)
props["comp_help"] = comp_help
except:
props["comp_help"] = []
# Get priority
try:
props["priority"] = p.find("priority").text
except:
pass
# Get "multi"
if p.find("multi") is not None:
props["multi"] = True
# Get "valueless"
if p.find("valueless") is not None:
props["valueless"] = True
return props
def process_node(n, f):
props_elem = n.find("properties")
children = n.find("children")
command = n.find("command")
children_nodes = []
owner = n.get("owner")
node_type = n.tag
name = n.get("name")
props = get_properties(props_elem)
if node_type != "node":
if "valueless" not in props.keys():
props["type"] = "txt"
if node_type == "tagNode":
props["tag"] = "True"
if node_type == "node" and children is not None:
inner_nodes = children.iterfind("*")
index_child = 0
for inner_n in inner_nodes:
children_nodes.append(process_node(inner_n, f))
index_child = index_child + 1
if node_type == "tagNode" and children is not None:
inner_nodes = children.iterfind("*")
index_child = 0
for inner_n in inner_nodes:
children_nodes.append(process_node(inner_n, f))
index_child = index_child + 1
else:
# This is a leaf node
pass
if command is not None:
test_command = True
else:
test_command = False
node = {
'name': name,
'type': node_type,
'children': children_nodes,
'props': props,
'command': test_command,
'filename': f
}
return node
def create_commands(data, parent_list=[], level=0):
result = []
command = {
'name': [],
'help': None,
'tag_help': [],
'level': level,
'no_childs': False,
'filename': None
}
command['filename'] = data['filename']
command['name'].extend(parent_list)
command['name'].append(data['name'])
if data['type'] == 'tagNode':
command['name'].append("<" + data['name'] + ">")
if 'val_help' in data['props'].keys():
for val_help in data['props']['val_help']:
command['tag_help'].append(val_help)
if len(data['children']) == 0:
command['no_childs'] = True
if data['command']:
command['no_childs'] = True
try:
help_text = data['props']['help']
command['help'] = re.sub(r"[\n\t]*", "", help_text)
except:
command['help'] = ""
command['valueless'] = data['props']['valueless']
if 'children' in data.keys():
children_bool = True
for child in data['children']:
result.extend(create_commands(child, command['name'], level + 1))
if command['no_childs']:
result.append(command)
return result
def include_file(line, input_dir):
string = ""
if "#include <include" in line.strip():
include_filename = line.strip().split('<')[1][:-1]
with open(input_dir + include_filename) as ifp:
iline = ifp.readline()
while iline:
string = string + include_file(iline.strip(), input_dir)
iline = ifp.readline()
else:
string = line
return string
def get_working_commands():
for entry in input_data:
for (dirpath, dirnames, filenames) in os.walk(entry['input_dir']):
entry['files'].extend(filenames)
break
for f in entry['files']:
string = ""
with open(entry['input_dir'] + f) as fp:
line = fp.readline()
while line:
string = string + include_file(line.strip(), entry['input_dir'])
line = fp.readline()
try:
xml = ET.parse(BytesIO(bytes(string, 'utf-8')))
except Exception as e:
print("Failed to load interface definition file {0}".format(f))
print(e)
sys.exit(1)
try:
relaxng_xml = ET.parse(entry['schema_file'])
validator = ET.RelaxNG(relaxng_xml)
if not validator.validate(xml):
print(validator.error_log)
print("Interface definition file {0} does not match the schema!".format(f))
sys.exit(1)
except Exception as e:
print("Failed to load the XML schema {0}".format(entry['schema_file']))
print(e)
sys.exit(1)
root = xml.getroot()
nodes = root.iterfind("*")
for n in nodes:
node_data[entry['kind']][f] = process_node(n, f)
# build config tree and sort
config_tree_new = {
'cfgcmd': {},
'opcmd': {},
}
for kind in node_data:
for entry in node_data[kind]:
node_0 = node_data[kind][entry]['name']
if node_0 not in config_tree_new[kind].keys():
config_tree_new[kind][node_0] = {
'name': node_0,
'type': node_data[kind][entry]['type'],
'props': node_data[kind][entry]['props'],
'children': [],
'command': node_data[kind][entry]['command'],
'filename': node_data[kind][entry]['filename'],
}
config_tree_new[kind][node_0]['children'].extend(node_data[kind][entry]['children'])
result = {
'cfgcmd': [],
'opcmd': [],
}
for kind in config_tree_new:
for e in config_tree_new[kind]:
result[kind].extend(create_commands(config_tree_new[kind][e]))
for cmd in result['cfgcmd']:
cmd['cmd'] = " ".join(cmd['name'])
for cmd in result['opcmd']:
cmd['cmd'] = " ".join(cmd['name'])
return result
if __name__ == "__main__":
res = get_working_commands()
print(json.dumps(res))
#print(res['cfgcmd'][0])

View File

@ -1,25 +1,41 @@
import re
import io
import json
import os
from docutils import io, nodes, utils, statemachine
from docutils.utils.error_reporting import SafeString, ErrorString
from docutils.parsers.rst.roles import set_classes
from docutils.parsers.rst import Directive, directives
from sphinx.util.docutils import SphinxDirective
from testcoverage import get_working_commands
def setup(app):
app.add_config_value(
'vyos_phabricator_url',
'https://phabricator.vyos.net/', ''
'https://phabricator.vyos.net/',
'html'
)
app.add_config_value(
'vyos_working_commands',
get_working_commands(),
'html'
)
app.add_config_value(
'vyos_coverage',
{
'cfgcmd': [0,len(app.config.vyos_working_commands['cfgcmd'])],
'opcmd': [0,len(app.config.vyos_working_commands['opcmd'])]
},
'html'
)
app.add_role('vytask', vytask_role)
app.add_role('cfgcmd', cmd_role)
app.add_role('opcmd', cmd_role)
print(app.config.vyos_phabricator_url)
app.add_node(
inlinecmd,
html=(inlinecmd.visit_span, inlinecmd.depart_span),
@ -46,9 +62,11 @@ def setup(app):
text=(CmdHeader.visit_div, CmdHeader.depart_div)
)
app.add_node(CfgcmdList)
app.add_node(CfgcmdListCoverage)
app.add_directive('cfgcmdlist', CfgcmdlistDirective)
app.add_node(OpcmdList)
app.add_node(OpcmdListCoverage)
app.add_directive('opcmdlist', OpcmdlistDirective)
app.add_directive('cfgcmd', CfgCmdDirective)
@ -56,15 +74,17 @@ def setup(app):
app.add_directive('cmdinclude', CfgInclude)
app.connect('doctree-resolved', process_cmd_nodes)
class CfgcmdList(nodes.General, nodes.Element):
pass
class OpcmdList(nodes.General, nodes.Element):
pass
import json
class CfgcmdListCoverage(nodes.General, nodes.Element):
pass
class OpcmdListCoverage(nodes.General, nodes.Element):
pass
class CmdHeader(nodes.General, nodes.Element):
@ -200,8 +220,8 @@ class CfgInclude(Directive):
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError:
raise self.severe(u'Problems with "%s" directive path.' %
(self.name))
raise self.severe(u'Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
@ -277,7 +297,16 @@ class CfgInclude(Directive):
return codeblock.run()
new_include_lines = []
var_value0 = self.options.get('var0', '')
var_value1 = self.options.get('var1', '')
var_value2 = self.options.get('var2', '')
var_value3 = self.options.get('var3', '')
var_value4 = self.options.get('var4', '')
var_value5 = self.options.get('var5', '')
var_value6 = self.options.get('var6', '')
var_value7 = self.options.get('var7', '')
var_value8 = self.options.get('var8', '')
var_value9 = self.options.get('var9', '')
for line in include_lines:
for i in range(10):
value = self.options.get(f'var{i}','')
@ -285,22 +314,41 @@ class CfgInclude(Directive):
line = re.sub('\s?{{\s?var' + str(i) + '\s?}}',value,line)
else:
line = re.sub('{{\s?var' + str(i) + '\s?}}',value,line)
new_include_lines.append(line)
self.state_machine.insert_input(new_include_lines, path)
return []
class CfgcmdlistDirective(Directive):
has_content = False
required_arguments = 0
option_spec = {
'show-coverage': directives.flag
}
def run(self):
return [CfgcmdList('')]
cfglist = CfgcmdList()
cfglist['coverage'] = False
if 'show-coverage' in self.options:
cfglist['coverage'] = True
return [cfglist]
class OpcmdlistDirective(Directive):
has_content = False
required_arguments = 0
option_spec = {
'show-coverage': directives.flag
}
def run(self):
return [OpcmdList('')]
oplist = OpcmdList()
oplist['coverage'] = False
if 'show-coverage' in self.options:
oplist['coverage'] = True
return [oplist]
class CmdDirective(SphinxDirective):
@ -309,6 +357,7 @@ class CmdDirective(SphinxDirective):
custom_class = ''
def run(self):
title_list = []
content_list = []
title_text = ''
@ -386,7 +435,134 @@ class CfgCmdDirective(CmdDirective):
custom_class = 'cfg'
def process_cmd_node(app, cmd, fromdocname):
def strip_cmd(cmd):
#cmd = re.sub('set','',cmd)
cmd = re.sub('\s\|\s','',cmd)
cmd = re.sub('<\S*>','',cmd)
cmd = re.sub('\[\S\]','',cmd)
cmd = re.sub('\s+','',cmd)
return cmd
def build_row(app, fromdocname, rowdata):
row = nodes.row()
for cell in rowdata:
entry = nodes.entry()
row += entry
if isinstance(cell, list):
for item in cell:
if isinstance(item, dict):
entry += process_cmd_node(app, item, fromdocname, '')
else:
entry += nodes.paragraph(text=item)
elif isinstance(cell, bool):
if cell:
entry += nodes.paragraph(text="")
entry['classes'] = ['coverage-ok']
else:
entry += nodes.paragraph(text="")
entry['classes'] = ['coverage-fail']
else:
entry += nodes.paragraph(text=cell)
return row
def process_coverage(app, fromdocname, doccmd, xmlcmd, cli_type):
coverage_list = {}
int_docs = 0
int_xml = 0
for cmd in doccmd:
coverage_item = {
'doccmd': None,
'xmlcmd': None,
'doccmd_item': None,
'xmlcmd_item': None,
'indocs': False,
'inxml': False,
'xmlfilename': None
}
coverage_item['doccmd'] = cmd['cmd']
coverage_item['doccmd_item'] = cmd
coverage_item['indocs'] = True
int_docs += 1
coverage_list[strip_cmd(cmd['cmd'])] = dict(coverage_item)
for cmd in xmlcmd:
strip = strip_cmd(cmd['cmd'])
if strip not in coverage_list.keys():
coverage_item = {
'doccmd': None,
'xmlcmd': None,
'doccmd_item': None,
'xmlcmd_item': None,
'indocs': False,
'inxml': False,
'xmlfilename': None
}
coverage_item['xmlcmd'] = cmd['cmd']
coverage_item['xmlcmd_item'] = cmd
coverage_item['inxml'] = True
coverage_item['xmlfilename'] = cmd['filename']
int_xml += 1
coverage_list[strip] = dict(coverage_item)
else:
#print("===BEGIN===")
#print(cmd)
#print(coverage_list[strip])
#print(strip)
#print("===END====")
coverage_list[strip]['xmlcmd'] = cmd['cmd']
coverage_list[strip]['xmlcmd_item'] = cmd
coverage_list[strip]['inxml'] = True
coverage_list[strip]['xmlfilename'] = cmd['filename']
int_xml += 1
table = nodes.table()
tgroup = nodes.tgroup(cols=3)
table += tgroup
header = (f'{int_docs}/{len(coverage_list)} in Docs', f'{int_xml}/{len(coverage_list)} in XML', 'Command')
colwidths = (1, 1, 8)
table = nodes.table()
tgroup = nodes.tgroup(cols=len(header))
table += tgroup
for colwidth in colwidths:
tgroup += nodes.colspec(colwidth=colwidth)
thead = nodes.thead()
tgroup += thead
thead += build_row(app, fromdocname, header)
tbody = nodes.tbody()
tgroup += tbody
for entry in sorted(coverage_list):
body_text_list = []
if coverage_list[entry]['indocs']:
body_text_list.append(coverage_list[entry]['doccmd_item'])
else:
body_text_list.append('Not documented yet')
if coverage_list[entry]['inxml']:
body_text_list.append("------------------")
body_text_list.append(str(coverage_list[entry]['xmlfilename']) + ":")
body_text_list.append(coverage_list[entry]['xmlcmd'])
else:
body_text_list.append('Nothing found in XML Definitions')
tbody += build_row(app, fromdocname,
(
coverage_list[entry]['indocs'],
coverage_list[entry]['inxml'],
body_text_list
)
)
return table
def process_cmd_node(app, cmd, fromdocname, cli_type):
para = nodes.paragraph()
newnode = nodes.reference('', '')
innernode = cmd['cmdnode']
@ -401,22 +577,46 @@ def process_cmd_node(app, cmd, fromdocname):
def process_cmd_nodes(app, doctree, fromdocname):
try:
env = app.builder.env
for node in doctree.traverse(CfgcmdList):
content = []
if node.attributes['coverage']:
node.replace_self(
process_coverage(
app,
fromdocname,
env.vyos_cfgcmd,
app.config.vyos_working_commands['cfgcmd'],
'cfgcmd'
)
)
else:
for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']):
content.append(process_cmd_node(app, cmd, fromdocname))
content.append(process_cmd_node(app, cmd, fromdocname, 'cfgcmd'))
node.replace_self(content)
for node in doctree.traverse(OpcmdList):
content = []
if node.attributes['coverage']:
node.replace_self(
process_coverage(
app,
fromdocname,
env.vyos_opcmd,
app.config.vyos_working_commands['opcmd'],
'opcmd'
)
)
else:
for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']):
content.append(process_cmd_node(app, cmd, fromdocname))
content.append(process_cmd_node(app, cmd, fromdocname, 'opcmd'))
node.replace_self(content)
except Exception as inst:
print(inst)
def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
app = inliner.document.settings.env.app

43
docs/coverage.rst Normal file
View File

@ -0,0 +1,43 @@
:orphan:
########
Coverage
########
Overview over all commands, which are documented in the ``.. cfgcmd::`` or ``.. opcmd::`` Directives.
| The build process take all xml definition files from `vyos-1x <https://github.com/vyos/vyos-1x>`_ and extract each leaf command or executable command.
| After this the commands are compare and shown in the follwoing two tables.
| The script compare only the fixed part of a command. All varables or values will be erase and then compare:
for example there are these two commands:
* documentation: ``interfaces ethernet <interface> address <address | dhcp | dhcpv6>```
* xml: ``interface ethernet <ethernet> address <address>``
Now the script earse all in between ``<`` and ``>`` and simply compare the strings.
**There are 2 kind of problems:**
| ``Not documented yet``
| A XML command are not found in ``.. cfgcmd::`` or ``.. opcmd::`` Commands
| The command should be documented
| ``Nothing found in XML Definitions``:
| ``.. cfgcmd::`` or ``.. opcmd::`` Command are not found in a XML command
| Maybe the command where changed in the XML Definition, or the feature is not anymore in VyOS
| Some commands are not yet translated to XML
Configuration Commands
======================
.. cfgcmdlist::
:show-coverage:
Operational Commands
====================
.. opcmdlist::
:show-coverage: