diff --git a/src/documentation_builder/.coveragerc b/src/documentation_builder/.coveragerc new file mode 100644 index 0000000000..1c8076c8f4 --- /dev/null +++ b/src/documentation_builder/.coveragerc @@ -0,0 +1,10 @@ +[run] +omit = + # omit anything in a .local directory anywhere + */.local/* + # omit everything in /usr + /usr/* + # omit the test directory + */test/* + # omit everything in /tmp + /tmp/* diff --git a/src/documentation_builder/.gitignore b/src/documentation_builder/.gitignore new file mode 100644 index 0000000000..75e4e429c6 --- /dev/null +++ b/src/documentation_builder/.gitignore @@ -0,0 +1,26 @@ +## aptana/eclipse +*.project +*.pydevproject +## python +*.pyc +*.pyo +.eggs +## emacs +*~ +.#* +*.orig +*.bak +## Various build products +*.rpm +*.o +*.old +*.DS_Store +*.egg +*.egg-info +.coverage +htmlcov/ +## Python __init__.py files for distributed packages +MANIFEST +dist/ +build/ +setup.cfg diff --git a/src/documentation_builder/.prospector.yaml b/src/documentation_builder/.prospector.yaml new file mode 100644 index 0000000000..110d1e3a7a --- /dev/null +++ b/src/documentation_builder/.prospector.yaml @@ -0,0 +1,17 @@ +strictness: veryhigh +test-warnings: true +doc-warnings: true +member-warnings: true + +ignore-paths: + - build + - htmlcov + +pep8: + disable: + options: + max-line-length: 120 + +pep257: + disable: + - D203 diff --git a/src/documentation_builder/MANIFEST.in b/src/documentation_builder/MANIFEST.in new file mode 100644 index 0000000000..46ba85b2ca --- /dev/null +++ b/src/documentation_builder/MANIFEST.in @@ -0,0 +1,3 @@ +include lib/quattordocbuild/tt/pan.tt +include lib/quattordocbuild/tt/toc.tt +include bin/build-quattor-documentation.sh diff --git a/src/documentation_builder/README.rst b/src/documentation_builder/README.rst new file mode 100644 index 0000000000..22a080f2bb --- /dev/null +++ b/src/documentation_builder/README.rst @@ -0,0 +1,54 @@ +Documentation-builder +--------------------- + +Documentation builder for the Quattor repositories. + +:: + $ quattor-documentation-builder --help + Usage: quattor-documentation-builder [options] + + + Documentation-builder generates markdown documentation. + + It get this from: + - configuration-modules-core perl documentation + - configuration-modules-grid perl documentation + - CAF perl documentation + - CCM perl documentation + - schema pan annotations + and creates a index for the website on http://quattor.org. + @author: Wouter Depypere (Ghent University) + + Options: + -h, --shorthelp show short help message and exit + -H OUTPUT_FORMAT, --help=OUTPUT_FORMAT + show full help message and exit + --confighelp show help as annotated configfile + + Main options (configfile section MAIN): + -p, --codify_paths Put paths inside code tags. (def True) + -i INDEX_NAME, --index_name=INDEX_NAME + Filename for the index/toc for the components. (def mkdocs.yml) + -c, --maven_compile + Execute a maven clean and maven compile before generating the documentation. (def False) + -m MODULES_LOCATION, --modules_location=MODULES_LOCATION + The location of the repo checkout. + -o OUTPUT_LOCATION, --output_location=OUTPUT_LOCATION + The location where the output markdown files should be written to. + -r, --remove_emails + Remove email addresses from generated md files. (def True) + -R, --remove_headers + Remove unneeded headers from files (MAINTAINER and AUTHOR). (def True) + -w, --remove_whitespace + Remove whitespace ( ) from md files. (def True) + -s, --small_titles Decrease the title size in the md files. (def True) + + Debug and logging options (configfile section MAIN): + -d, --debug Enable debug log mode (def False) + --info Enable info log mode (def False) + --quiet Enable quiet/warning log mode (def False) + + + +It makes some assumpions on several repositories being in place. +To help set this up a helper script was added **build-quattor-documentation.sh** which builds the whole documentation from latest master. diff --git a/src/documentation_builder/bin/build-quattor-documentation.sh b/src/documentation_builder/bin/build-quattor-documentation.sh new file mode 100755 index 0000000000..f8af10f3d5 --- /dev/null +++ b/src/documentation_builder/bin/build-quattor-documentation.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +for com in 'mkdocs' 'mvn' 'pod2markdown' 'bundle'; do + command -v $com >/dev/null 2>&1 || { echo >&2 "I require $com but it's not installed. Aborting."; exit 1; } +done + +tmpdir="/tmp/quattor-documentation" + +# Create temporary working directory +mkdir -p $tmpdir/{src,output} +cd $tmpdir/src + +# Clone required github repositories +for REPO in CAF configuration-modules-core configuration-modules-grid CCM maven-tools; do + git clone https://github.com/quattor/$REPO.git +done +cd .. + +# Build the whole documentation +quattor-documentation-builder -c -m $tmpdir/src/ -o $tmpdir/output/ --info || { echo 'Something went wrong building documentation.' ; exit 1 ; } + +# get required index which is not generated +curl https://raw.githubusercontent.com/quattor/documentation/master/docs/index.md -o $tmpdir/output/docs/index.md + +cd $tmpdir/output +mkdocs build --clean + +# Get some tests up +curl https://raw.githubusercontent.com/quattor/documentation/master/Gemfile -o Gemfile +bundle + +bundle exec htmlproofer --check-html ./site/ --file-ignore ./site/base.html,./site/breadcrumbs.html,./site/footer.html,./site/toc.html,./site/versions.html || { echo 'build test errors detected. stopping.' ; exit 1 ; } diff --git a/src/documentation_builder/bin/quattor-documentation-builder b/src/documentation_builder/bin/quattor-documentation-builder new file mode 100644 index 0000000000..ed416d7252 --- /dev/null +++ b/src/documentation_builder/bin/quattor-documentation-builder @@ -0,0 +1,48 @@ +#!/usr/bin/env python2 +""" +Documentation-builder generates markdown documentation. + +It get this from: + - configuration-modules-core perl documentation + - configuration-modules-grid perl documentation + - CAF perl documentation + - CCM perl documentation + - schema pan annotations + and creates a index for the website on http://quattor.org. +@author: Wouter Depypere (Ghent University) +""" + +from vsc.utils import fancylogger +from vsc.utils.generaloption import simple_option +from quattordocbuild.builder import build_documentation + +logger = fancylogger.getLogger() + + +def main(repolocation, outputlocation, maven_compile, cleanup_options): + """Main run of the script.""" + build_documentation(repolocation, cleanup_options, maven_compile, outputlocation) + + +if __name__ == '__main__': + OPTIONS = { + 'modules_location': ('The location of the repo checkout.', None, 'store', None, 'm'), + 'output_location': ('The location where the output markdown files should be written to.', None, 'store', None, 'o'), + 'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None, 'store_true', False, 'c'), + 'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'), + 'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'), + 'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'), + 'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's'), + 'codify_paths': ('Put paths inside code tags.', None, 'store_true', True, 'p'), + } + GO = simple_option(OPTIONS) + logger.info("Starting main.") + cleanup_options = { + 'remove_emails': GO.options.remove_emails, + 'remove_whitespace': GO.options.remove_whitespace, + 'remove_headers': GO.options.remove_headers, + 'small_titles': GO.options.small_titles, + 'codify_paths': GO.options.codify_paths, + } + main(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile, cleanup_options) + logger.info("Done.") diff --git a/src/documentation_builder/lib/__init__.py b/src/documentation_builder/lib/__init__.py new file mode 100644 index 0000000000..28fb39055f --- /dev/null +++ b/src/documentation_builder/lib/__init__.py @@ -0,0 +1 @@ +"""Empty __init__.py.""" diff --git a/src/documentation_builder/lib/quattordocbuild/__init__.py b/src/documentation_builder/lib/quattordocbuild/__init__.py new file mode 100644 index 0000000000..6f9a20fcfe --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/__init__.py @@ -0,0 +1 @@ +"""Empty __init.py__.""" diff --git a/src/documentation_builder/lib/quattordocbuild/builder.py b/src/documentation_builder/lib/quattordocbuild/builder.py new file mode 100644 index 0000000000..d772982b50 --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/builder.py @@ -0,0 +1,183 @@ +"""Build documentation from quattor sources.""" + +import os +import sys +import re +from template import Template, TemplateException +from sourcehandler import get_source_files +from markdownhandler import generate_markdown, cleanup_content +from config import build_repository_map +from vsc.utils import fancylogger + +logger = fancylogger.getLogger() + + +def build_documentation(repository_location, cleanup_options, compile, output_location): + """Build the whole documentation from quattor repositories.""" + if not check_input(repository_location, output_location): + sys.exit(1) + if not check_commands(compile): + sys.exit(1) + repository_map = build_repository_map(repository_location) + if not repository_map: + sys.exit(1) + + markdownlist = {} + + for repository in repository_map.keys(): + logger.info("Building documentation for %s." % repository) + fullpath = os.path.join(repository_location, repository) + if repository_map[repository]["subdir"]: + fullpath = os.path.join(fullpath, repository_map[repository]["subdir"]) + logger.info("Path: %s." % fullpath) + sources = get_source_files(fullpath, compile) + logger.debug("Sources:" % sources) + markdown = generate_markdown(sources) + cleanup_content(markdown, cleanup_options) + markdownlist[repository] = markdown + + site_pages = build_site_structure(markdownlist, repository_map) + site_pages = make_interlinks(site_pages) + write_site(site_pages, output_location, "docs") + return True + + +def which(command): + """Check if given command is available for the current user on this system.""" + found = False + for direct in os.getenv("PATH").split(':'): + if os.path.exists(os.path.join(direct, command)): + found = True + + return found + + +def check_input(sourceloc, outputloc): + """Check input and locations.""" + logger.info("Checking if the given paths exist.") + if not sourceloc: + logger.error("Repo location not specified.") + return False + if not outputloc: + logger.error("output location not specified") + return False + if not os.path.exists(sourceloc): + logger.error("Repo location %s does not exist" % sourceloc) + return False + if not os.path.exists(outputloc): + logger.error("Output location %s does not exist" % outputloc) + return False + if not os.listdir(outputloc) == []: + logger.error("Output location %s is not empty." % outputloc) + return False + return True + + +def check_commands(runmaven): + """Check required binaries.""" + if runmaven: + if not which("mvn"): + logger.error("The command mvn is not available on this system, please install maven.") + return False + if not which("pod2markdown"): + logger.error("The command pod2markdown is not available on this system, please install pod2markdown.") + return False + return True + + +def build_site_structure(markdownlist, repository_map): + """Make a mapping of files with their new names for the website.""" + sitepages = {} + for repo, markdowns in markdownlist.iteritems(): + sitesection = repository_map[repo]['sitesection'] + + sitepages[sitesection] = {} + + targets = repository_map[repo]['targets'] + for source, markdown in markdowns.iteritems(): + found = False + for target in targets: + if target in source and not found: + newname = source.split(target)[-1] + newname = os.path.splitext(newname)[0].replace("/", "::") + ".md" + sitepages[sitesection][newname] = markdown + found = True + if not found: + logger.error("No suitable target found for %s in %s." % (source, targets)) + return sitepages + + +def make_interlinks(pages): + """Make links in the content based on pagenames.""" + logger.info("Creating interlinks.") + newpages = pages + for subdir in pages: + for page in pages[subdir]: + basename = os.path.splitext(page)[0] + link = '../%s/%s' % (subdir, page) + regxs = [] + regxs.append("`%s`" % basename) + regxs.append("`%s::%s`" % (subdir, basename)) + + cpans = "https://metacpan.org/pod/" + + if subdir == 'CCM': + regxs.append("\[{2}::{0}\]\({1}{2}::{0}\)".format(basename, cpans, "EDG::WP4::CCM")) + if subdir == 'Unittest': + regxs.append("\[{2}::{0}\]\({1}{2}::{0}\)".format(basename, cpans, "Test")) + if subdir in ['components', 'components-grid']: + regxs.append("\[{2}::{0}\]\({1}{2}::{0}\)".format(basename, cpans, "NCM::Component")) + regxs.append("`ncm-%s`" % basename) + regxs.append("ncm-%s" % basename) + + for regex in regxs: + newpages = replace_regex_link(newpages, regex, basename, link) + + return newpages + + +def replace_regex_link(pages, regex, basename, link): + """Replace links in a bunch of pages based on a regex.""" + regex = r'( |^|\n)%s([,. $])' % regex + for subdir in pages: + for page in pages[subdir]: + content = pages[subdir][page] + if (basename not in page or basename == "Quattor") and basename in content: + content = re.sub(regex, "\g<1>[%s](%s)\g<2>" % (basename, link), content) + pages[subdir][page] = content + return pages + + +def write_site(sitepages, location, docsdir): + """Write the pages for the website to disk and build a toc.""" + toc = {} + for subdir, pages in sitepages.iteritems(): + toc[subdir] = set() + fullsubdir = os.path.join(location, docsdir, subdir) + if not os.path.exists(fullsubdir): + os.makedirs(fullsubdir) + for pagename, content in pages.iteritems(): + with open(os.path.join(fullsubdir, pagename), 'w') as fih: + fih.write(content) + + toc[subdir].add(pagename) + + # Sort the toc, ignore the case. + toc[subdir] = sorted(toc[subdir], key=lambda s: s.lower()) + + write_toc(toc, location) + + +def write_toc(toc, location): + """Write the toc to disk.""" + try: + name = 'toc.tt' + template = Template({'INCLUDE_PATH': os.path.join(os.path.dirname(__file__), 'tt')}) + tocfile = template.process(name, {'toc': toc}) + except TemplateException as e: + msg = "Failed to render template %s with data %s: %s." % (name, toc, e) + logger.error(msg) + raise TemplateException('render', msg) + + with open(os.path.join(location, "mkdocs.yml"), 'w') as fih: + fih.write(tocfile) diff --git a/src/documentation_builder/lib/quattordocbuild/config.py b/src/documentation_builder/lib/quattordocbuild/config.py new file mode 100644 index 0000000000..d74b39889e --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/config.py @@ -0,0 +1,76 @@ +"""Build and check configuration for quattor documentation.""" + +import os +import ConfigParser +from vsc.utils import fancylogger + +logger = fancylogger.getLogger() +cfgfile = ".docbuilder.cfg" + + +def check_repository_map(repository_map): + """Check if a repository mapping is valid.""" + logger.info("Checking repository map.") + if repository_map is None: + logger.error("Repository map is None.") + return False + if len(repository_map) == 0: + logger.error("Repository map is an empty list.") + return False + for repository in repository_map.keys(): + keys = repository_map[repository].keys() + for opt in ['targets', 'sitesection']: + if opt not in keys: + logger.error("Respository %s does not have a '%s' in repository_map." % (repository, opt)) + return False + if type(repository_map[repository]['targets']) is not list: + logger.error("Repository %s targets is not a list." % repository) + return False + return True + + +def build_repository_map(location): + """Build a repository mapping for repository_location.""" + logger.info("Building repository map in %s." % location) + root_dirs = [f for f in os.listdir(location) if os.path.isdir(os.path.join(location, f))] + repomap = {} + for repo in root_dirs: + logger.info("Checking %s." % repo) + fullcfgfile = os.path.join(location, repo, cfgfile) + if os.path.isfile(fullcfgfile): + cfg = read_config(fullcfgfile) + if cfg: + repomap[repo] = cfg + else: + logger.warning("Invalid config file found in %s. Not using it." % repo) + else: + logger.warning("No config file found in %s. Not using it." % repo) + + if check_repository_map(repomap): + return repomap + else: + return False + + +def read_config(configfile): + """Read a given config file.""" + config = ConfigParser.ConfigParser() + config.read(configfile) + + section = 'docbuilder' + options = {} + for option in ['sitesection', 'targets']: + if config.has_option(section, option): + options[option] = config.get(section, option) + else: + logger.error("Config %s does not have required option '%s'." % (configfile, option)) + return False + # filter out empty strings from a list of strings + options['targets'] = filter(None, options['targets'].split(',')) + + if config.has_option(section, "subdir"): + options["subdir"] = config.get(section, "subdir") + else: + options["subdir"] = None + + return options diff --git a/src/documentation_builder/lib/quattordocbuild/markdownhandler.py b/src/documentation_builder/lib/quattordocbuild/markdownhandler.py new file mode 100644 index 0000000000..bf4058d104 --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/markdownhandler.py @@ -0,0 +1,115 @@ +"""Module to handle markdown operations.""" + +import re + +from vsc.utils import fancylogger +from vsc.utils.run import run_asyncloop +from panhandler import markdown_from_pan + +logger = fancylogger.getLogger() + +MAILREGEX = re.compile(("([a-zA-Z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+\/=?^_`" + "{|}~-]+)*(@|\sat\s)(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?(\.|" + "\sdot\s))+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)")) +PATHREGEX = re.compile(r'(\s+)((?:/[\w{}]+)+\.?\w*)(\s*)') +EXAMPLEMAILS = ["example", "username", "system.admin"] + + +def generate_markdown(sources): + """Generate markdown.""" + logger.info("Generating md files.") + + markdownlist = {} + + for source in sources: + logger.debug("Parsing %s." % source) + if source.endswith(".pan"): + markdown = markdown_from_pan(source) + if markdown is not None: + markdownlist[source] = markdown + else: + markdown = markdown_from_perl(source) + if markdown is not None: + markdownlist[source] = markdown + + return markdownlist + + +def markdown_from_perl(podfile): + """ + Take a perl file and converts it to a markdown with the help of pod2markdown. + + Returns True if pod2markdown worked, False if it failed. + """ + logger.info("Making markdown from perl: %s." % podfile) + ec, output = run_asyncloop("pod2markdown %s" % podfile) + logger.debug(output) + if ec != 0 or output == "\n": + logger.warning("pod2markdown failed on %s." % podfile) + return None + else: + return output + + +def cleanup_content(markdown, cleanup_options): + """Run several cleaners on the content we get from perl files.""" + for source, content in markdown.iteritems(): + if not source.endswith('.pan'): + for fn in ['remove_emails', 'remove_headers', 'small_titles', 'remove_whitespace', 'codify_paths']: + if cleanup_options[fn]: + content = globals()[fn](content) + markdown[source] = content + + return markdown + + +def remove_emails(markdown): + """Remove email adresses from markdown.""" + replace = False + for email in re.findall(MAILREGEX, markdown): + logger.debug("Found %s." % email[0]) + replace = True + if email[0].startswith('//'): + replace = False + for ignoremail in EXAMPLEMAILS: + if ignoremail in email[0]: + replace = False + + if replace: + logger.debug("Removed it from line.") + markdown = markdown.replace(email[0], '') + + return markdown + + +def remove_headers(markdown): + """Remove MAINTAINER and AUTHOR headers from md files.""" + for header in ['# AUTHOR', '# MAINTAINER']: + if header in markdown: + markdown = markdown.replace(header, '') + + return markdown + + +def remove_whitespace(markdown): + """Remove extra whitespace (3 line breaks).""" + if '\n\n\n' in markdown: + markdown = markdown.replace('\n\n\n', '\n') + markdown = remove_whitespace(markdown) + + return markdown + + +def small_titles(markdown): + """Make titles smaller, eg replace '# ' with '### ' at the start of a line.""" + markdown = re.sub(r'\n# |^# ', '\n### ', markdown) + markdown = re.sub(r'\n## |^##', '\n#### ', markdown) + return markdown + + +def codify_paths(markdown): + """Put paths into code tags.""" + markdown, counter = PATHREGEX.subn(r'\1`\2`\3', markdown) + if counter > 0: + markdown = codify_paths(markdown) + return markdown diff --git a/src/documentation_builder/lib/quattordocbuild/panhandler.py b/src/documentation_builder/lib/quattordocbuild/panhandler.py new file mode 100644 index 0000000000..89bffcea2a --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/panhandler.py @@ -0,0 +1,169 @@ +"""Handle pan files for documentation.""" + +import os +import re +import tempfile +import shutil +from template import Template, TemplateException +from vsc.utils import fancylogger +from vsc.utils.run import run_asyncloop +from lxml import etree + +logger = fancylogger.getLogger() +namespace = "{http://quattor.org/pan/annotations}" + + +def markdown_from_pan(panfile): + """Make markdown from a pan annotated file.""" + logger.info("Making markdown from pan: %s." % panfile) + content = get_content_from_pan(panfile) + basename = get_basename(panfile) + output = render_template(content, basename) + if len(output) == 0: + return None + else: + return output + + +def render_template(content, basename): + """Render the template.""" + try: + name = 'pan.tt' + template = Template({'INCLUDE_PATH': os.path.join(os.path.dirname(__file__), 'tt')}) + output = template.process(name, {'content': content, 'basename': basename}) + except TemplateException as e: + msg = "Failed to render template %s with data %s: %s." % (name, content, e) + logger.error(msg) + raise TemplateException('render', msg) + return output + + +def get_content_from_pan(panfile): + """Return the information of all types and functions from a pan annotated file.""" + content = {} + tempdir = tempfile.mkdtemp() + directory, filename = os.path.split(panfile) + built = build_annotations(filename, directory, tempdir) + if built: + xmlroot = validate_annotations(os.path.join(tempdir, "%s.annotation.xml" % filename)) + if xmlroot is not None: + types, functions = get_types_and_functions(xmlroot) + if types is not None: + content['types'] = [] + for type in types: + content['types'].append(parse_type(type)) + + if functions is not None: + content['functions'] = [] + for function in functions: + content['functions'].append(parse_function(function)) + shutil.rmtree(tempdir) + return content + + +def build_annotations(file, basedir, outputdir): + """Build pan annotations.""" + panccommand = ["panc-annotations", "--output-dir", outputdir, "--base-dir", basedir] + panccommand.append(file) + logger.debug("Running %s." % panccommand) + ec, output = output = run_asyncloop(panccommand) + logger.debug(output) + if ec == 0 and os.path.exists(os.path.join(outputdir, "%s.annotation.xml" % file)): + return True + else: + logger.warning("Something went wrong running '%s'." % panccommand) + return False + + +def validate_annotations(file): + """ + Check if a pan annotations file is usable. + + e.g. XML is parsable and the root element is not empty. + If it is usable, return the xml root element. + """ + xml = etree.parse(file) + root = xml.getroot() + + if len(root) == 0: + logger.debug("%s is empty, skipping it." % file) + return None + else: + return root + + +def get_types_and_functions(root): + """Return a list of types and functions from a root element.""" + types = root.findall('%stype' % namespace) + functions = root.findall('%sfunction' % namespace) + + logger.debug(types) + logger.debug(functions) + + if len(types) == 0 and len(functions) == 0: + logger.debug("%s has no usable content, skipping it." % file) + return None, None + + return types, functions + + +def find_description(element): + """Search for the desc tag, even if it is in documentation tag.""" + desc = element.find("./%sdocumentation/%sdesc" % (namespace, namespace)) + if desc is None: + desc = element.find("./%sdesc" % namespace) + return desc + + +def parse_type(type): + """Parse a type from an XML Element Tree.""" + typeinfo = {} + typeinfo['name'] = type.get('name') + desc = find_description(type) + if desc is not None: + typeinfo['desc'] = desc.text + + typeinfo['fields'] = [] + + for field in type.findall(".//%sfield" % namespace): + fieldinfo = {} + fieldinfo['name'] = field.get('name') + desc = find_description(field) + if desc is not None: + fieldinfo['desc'] = desc.text + + fieldinfo['required'] = field.get('required') + basetype = field.find(".//%sbasetype" % namespace) + fieldtype = basetype.get('name') + fieldinfo['type'] = fieldtype + if fieldtype == "long" and basetype.get('range'): + fieldinfo['range'] = basetype.get('range') + + typeinfo['fields'].append(fieldinfo) + + return typeinfo + + +def parse_function(function): + """Parse a function from an XML Element Tree.""" + functinfo = {} + functinfo['name'] = function.get('name') + desc = find_description(function) + if desc is not None: + functinfo['desc'] = desc.text + functinfo['args'] = [] + for arg in function.findall(".//%sarg" % namespace): + functinfo['args'].append(arg.text) + + return functinfo + + +def get_basename(path): + """Return a base name from a path and regular expression.""" + regex = ".*/(.*?)/target/.*" + result = re.search(regex, path) + result = result.group(1) + if "ncm-" in result: + result = result.replace("ncm-", "") + + return result diff --git a/src/documentation_builder/lib/quattordocbuild/sourcehandler.py b/src/documentation_builder/lib/quattordocbuild/sourcehandler.py new file mode 100644 index 0000000000..3882b63200 --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/sourcehandler.py @@ -0,0 +1,120 @@ +""" +Module to handle 'source' related actions. + +It contains functions to run maven clean and compile, +scan a whole tree for files and find '.pan' files and perl +files, where when there is a duplicate between '.pod' and +'.pl', the pod file has the preference. +""" + +import os + +from vsc.utils import fancylogger +from vsc.utils.run import run_asyncloop + +logger = fancylogger.getLogger() + + +def maven_clean_compile(location): + """Execute mvn clean and mvn compile in the given modules_location.""" + logger.info("Doing maven clean compile in %s." % location) + ec, output = run_asyncloop("mvn clean compile", startpath=location) + logger.debug(output) + return ec + + +def is_wanted_dir(path, files): + """ + Check if the directory matches required criteria. + + - It must be in 'target directory' + - It should contain files + - it shoud contain one of the following in the path: + - doc/pod + - lib/perl + - pan + """ + if 'target' not in path: + return False + if len(files) == 0: + return False + if True not in [substr in path for substr in ["doc/pod", "lib/perl", "pan"]]: + return False + return True + + +def is_wanted_file(path, filename): + """ + Check if the file matches one of the criteria. + + - a perl file based on extension + - a pan file based on extension (.pan) + - a perl file based on shebang + """ + if True in [filename.endswith(ext) for ext in [".pod", ".pm", ".pl"]]: + return True + if filename.endswith(".pan"): + return True + if len(filename.split(".")) < 2: + with open(os.path.join(path, filename), 'r') as pfile: + if 'perl' in pfile.readline(): + return True + return False + + +def handle_duplicates(file, path, fulllist): + """Handle duplicates, pod takes preference over pm.""" + if "doc/pod" in path: + duplicate = path.replace('doc/pod', 'lib/perl') + if file.endswith('.pod'): + duplicate = duplicate.replace(".pod", ".pm") + if duplicate in fulllist: + fulllist[fulllist.index(duplicate)] = path + return fulllist + + if "lib/perl" in path: + duplicate = path.replace('lib/perl', 'doc/pod') + if file.endswith('.pm'): + duplicate = duplicate.replace(".pm", ".pod") + if duplicate not in fulllist: + fulllist.append(path) + return fulllist + else: + return fulllist + fulllist.append(path) + return fulllist + + +def list_source_files(location): + """ + Return a list of source_files in a location. + + Try to filter out: + - Unwanted locations + - Unwanted files + - Duplicates, pod takes precedence over pm. + """ + logger.info("Looking for source files.") + finallist = [] + for path, _, files in os.walk(location): + if not is_wanted_dir(path, files): + continue + + for file in files: + if is_wanted_file(path, file): + fullpath = os.path.join(path, file) + finallist = handle_duplicates(file, fullpath, finallist) + return finallist + + +def get_source_files(location, compile): + """Run maven compile and get all source files.""" + if compile: + ec = maven_clean_compile(location) + if ec != 0: + logger.error("Something went wrong running maven in %s." % location) + return None + + sources = list_source_files(location) + logger.info("Found %s source files." % len(sources)) + return sources diff --git a/src/documentation_builder/lib/quattordocbuild/tt/pan.tt b/src/documentation_builder/lib/quattordocbuild/tt/pan.tt new file mode 100644 index 0000000000..9c922d927d --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/tt/pan.tt @@ -0,0 +1,45 @@ +[% IF content.types.defined and content.types.size -%] + +### Types + +[% FOREACH type IN content.types -%] + - `/software/[% basename %]/[% type.name %]` +[% IF type.desc.defined -%] + - Description: [% type.desc %] +[% END -%] +[% IF type.fields.defined -%] +[% FOREACH field IN type.fields -%] + - `/software/[% basename %]/[% type.name %]/[% field.name %]` +[% IF field.desc.defined -%] + - Description: [% field.desc %] +[% END -%] +[% IF field.required == true -%] + - Required +[% ELSE -%] + - Optional +[% END -%] + - Type: [% field.type %] +[% IF field.range.defined -%] + - Range: [% field.range %] +[% END -%] +[% END -%] +[% END -%] +[% END -%] +[% END -%] +[% IF content.functions.defined and content.functions.size -%] + +### Functions + +[% FOREACH function IN content.functions -%] + - [% function.name %] +[% IF function.desc.defined -%] + - Description: [% function.desc %] +[% END -%] +[% IF function.args.size -%] + - Arguments: +[% FOREACH arg IN function.args -%] + - [% arg %] +[% END -%] +[% END -%] +[% END -%] +[% END -%] diff --git a/src/documentation_builder/lib/quattordocbuild/tt/toc.tt b/src/documentation_builder/lib/quattordocbuild/tt/toc.tt new file mode 100644 index 0000000000..8a2408bb7a --- /dev/null +++ b/src/documentation_builder/lib/quattordocbuild/tt/toc.tt @@ -0,0 +1,12 @@ +site_name: Quattor Documentation + +theme: 'readthedocs' + +pages: +- introduction: 'index.md' +[% FOREACH subdir_pages IN toc.pairs.sort -%] +- [% subdir_pages.key %]: +[%- FOREACH page IN subdir_pages.value %] + - [% page.split('.').first %]: [% subdir_pages.key %]/[% page -%] +[% END %] +[% END -%] diff --git a/src/documentation_builder/setup.py b/src/documentation_builder/setup.py new file mode 100644 index 0000000000..78d4eaa278 --- /dev/null +++ b/src/documentation_builder/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +Basic setup.py for building the documentation-builder. + +@author: Wouter Depypere +""" + +from setuptools import setup, find_packages + +if __name__ == '__main__': + setup( + name = 'documentation-builder', + description = 'Documentation Builder for Quattor', + url='https://github.com/quattor/release/src/documentation_builder/', + version = '0.0.1', + author = 'Wouter Depypere', + author_email = 'wouter.depypere@ugent.be', + packages = find_packages('lib'), + package_dir={'':'lib'}, + scripts=['bin/quattor-documentation-builder', 'bin/build-quattor-documentation.sh'], + install_requires = [ + 'vsc-utils', + 'vsc-base', + 'Template-Python', + 'lxml', + ], + test_suite = "test", + tests_require = ["prospector"], + include_package_data=True, + ) diff --git a/src/documentation_builder/test/__init__.py b/src/documentation_builder/test/__init__.py new file mode 100644 index 0000000000..28fb39055f --- /dev/null +++ b/src/documentation_builder/test/__init__.py @@ -0,0 +1 @@ +"""Empty __init__.py.""" diff --git a/src/documentation_builder/test/builder.py b/src/documentation_builder/test/builder.py new file mode 100644 index 0000000000..1e9496af85 --- /dev/null +++ b/src/documentation_builder/test/builder.py @@ -0,0 +1,149 @@ +"""Test module for builder.py.""" + +import sys +import os +import shutil +import filecmp +from tempfile import mkdtemp +from unittest import TestCase, main, TestLoader +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))) # noqa +from quattordocbuild import builder + + +class BuilderTest(TestCase): + """Test class for builder.py.""" + + def setUp(self): + """Set up temp dir for tests.""" + self.tmpdir = mkdtemp() + + def tearDown(self): + """Remove temp dir.""" + shutil.rmtree(self.tmpdir) + + def test_which(self): + """Test which function.""" + self.assertFalse(builder.which('testtest1234')) + self.assertTrue(builder.which('python')) + + def test_check_input(self): + """Test check_input function.""" + self.assertFalse(builder.check_input("", "")) + self.assertFalse(builder.check_input(self.tmpdir, "")) + self.assertFalse(builder.check_input("/nonexistingdir", "")) + self.assertTrue(builder.check_input(self.tmpdir, self.tmpdir)) + os.makedirs(os.path.join(self.tmpdir, "test")) + self.assertTrue(builder.check_input(self.tmpdir, os.path.join(self.tmpdir, "test"))) + + def test_check_commands(self): + """Test check_commands function.""" + self.assertTrue(builder.check_commands(True)) + + def test_build_site_structure(self): + """Test build_site_structure function.""" + repomap = { + "configuration-modules-core": { + "sitesection": "components", + "targets": ["/NCM/Component/", "/components/", "/pan/quattor/"] + }, + "CCM": { + "sitesection": "CCM", + "targets": ["EDG/WP4/CCM/"], + }, + } + testdata = {'CCM': {'/tmp/qdoc/src/CCM/target/doc/pod/EDG/WP4/CCM/Fetch/Download.pod': + "# NAME\n\nEDG::WP4::CC"}, + 'configuration-modules-core': + {'/tmp/doc/src/configuration-modules-core/ncm-profile/target/pan/components/profile/functions.pan': + u'\n### Functions\n', + '/tmp/doc/src/configuration-modules-core/ncm-fmonagent/target/doc/pod/NCM/Component/fmonagent.pod': + 'Hello', + '/tmp/doc/src/configuration-modules-core/ncm-freeipa/target/pan/quattor/aii/freeipa/schema.pan': + 'Hello2' + }} + expected_response = {'CCM': {'Fetch::Download.md': '# NAME\n\nEDG::WP4::CC'}, + 'components': {'aii::freeipa::schema.md': 'Hello2', + 'fmonagent.md': 'Hello', + 'profile::functions.md': u'\n### Functions\n'}} + self.assertEquals(builder.build_site_structure(testdata, repomap), expected_response) + + def test_make_interlinks(self): + """Test make_interlinks function.""" + # Replace one reference + test_data = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to `fmonagent`.'}} + expected = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to [fmonagent](../components-grid/fmonagent.md).'}} + self.assertEquals(builder.make_interlinks(test_data), expected) + + # Replace two references + test_data = {'comps-gr': {'fmnt.md': ''}, + 'comps': {'icinga.md': 'refr `fmnt` and `fmnt`.'}} + expected = {'comps-gr': {'fmnt.md': ''}, + 'comps': {'icinga.md': 'refr [fmnt](../comps-gr/fmnt.md) and [fmnt](../comps-gr/fmnt.md).'}} + self.assertEquals(builder.make_interlinks(test_data), expected) + + # Replace ncm- reference + test_data = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to `ncm-fmonagent`.'}} + expected = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to [fmonagent](../components-grid/fmonagent.md).'}} + self.assertEquals(builder.make_interlinks(test_data), expected) + + # Replace newline reference + test_data = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to \n`ncm-fmonagent`.'}} + expected = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to \n[fmonagent](../components-grid/fmonagent.md).'}} + self.assertEquals(builder.make_interlinks(test_data), expected) + + # Replace linked wrong reference + test_data = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer \ +to [NCM::Component::FreeIPA::Client](https://metacpan.org/pod/NCM::Component::FreeIPA::Client).', + 'FreeIPA::Client': 'Allo'}} + expected = {'components-grid': {'fmonagent.md': ''}, + 'components': {'icinga.md': 'I refer to [FreeIPA::Client](../components/FreeIPA::Client).', + 'FreeIPA::Client': 'Allo'}} + + self.assertEquals(builder.make_interlinks(test_data), expected) + + # Don't replace in own page + test_data = {'comps-grid': {'fmonagent.md': 'ref to `fmonagent`.'}, + 'comps': {'icinga.md': 'ref to `icinga` and `ncm-icinga`.'}} + self.assertEquals(builder.make_interlinks(test_data), test_data) + + def test_write_site(self): + """Test write_site function.""" + input = {'CCM': {'fetch::download.md': '# NAME\n\nEDG::WP4::CC'}, + 'components': {'fmonagent.md': 'Hello', + 'profile::functions.md': u'\n### Functions\n'}} + + sitedir = os.path.join(self.tmpdir, "docs") + builder.write_site(input, self.tmpdir, "docs") + self.assertTrue(os.path.exists(os.path.join(sitedir, 'components'))) + self.assertTrue(os.path.exists(os.path.join(sitedir, 'components/profile::functions.md'))) + self.assertTrue(os.path.exists(os.path.join(sitedir, 'components/fmonagent.md'))) + self.assertTrue(os.path.exists(os.path.join(sitedir, 'CCM'))) + self.assertTrue(os.path.exists(os.path.join(sitedir, 'CCM/fetch::download.md'))) + + def test_write_toc(self): + """Test write_toc function.""" + toc = {'CCM': set(['fetch::download.md']), 'components': set(['fmonagent.md', 'profile::functions.md'])} + builder.write_toc(toc, self.tmpdir) + with open(os.path.join(self.tmpdir, "mkdocs.yml")) as fih: + print fih.read() + self.assertTrue(filecmp.cmp('test/testdata/mkdocs.yml', os.path.join(self.tmpdir, "mkdocs.yml"))) + + toc = {'components': set(['profile::functions.md', 'fmonagent.md']), 'CCM': set(['fetch::download.md'])} + builder.write_toc(toc, self.tmpdir) + with open(os.path.join(self.tmpdir, "mkdocs.yml")) as fih: + print fih.read() + self.assertTrue(filecmp.cmp('test/testdata/mkdocs.yml', os.path.join(self.tmpdir, "mkdocs.yml"))) + + def suite(self): + """Return all the testcases in this module.""" + return TestLoader().loadTestsFromTestCase(BuilderTest) + +if __name__ == '__main__': + main() diff --git a/src/documentation_builder/test/config.py b/src/documentation_builder/test/config.py new file mode 100644 index 0000000000..7f45678116 --- /dev/null +++ b/src/documentation_builder/test/config.py @@ -0,0 +1,78 @@ +"""Test module for config.py.""" + +import sys +import os +import shutil +from tempfile import mkdtemp +from unittest import TestCase, main, TestLoader +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))) # noqa +from quattordocbuild import config + + +class ConfigTest(TestCase): + """Test class for config.py.""" + + def setUp(self): + """Set up temp dir for tests.""" + self.tmpdir = mkdtemp() + + def tearDown(self): + """Remove temp dir.""" + shutil.rmtree(self.tmpdir) + + def test_check_repository_map(self): + """Test check repository_map function.""" + self.assertFalse(config.check_repository_map(None)) + self.assertFalse(config.check_repository_map({})) + self.assertFalse(config.check_repository_map({"test": {}})) + self.assertFalse(config.check_repository_map({"test": {"sitesection": "test"}})) + self.assertFalse(config.check_repository_map({"test": {"targets": ["test"]}})) + repomap = {'test': {'sitesection': 'components', 'targets': ['/NCM/Component', 'components']}} + self.assertTrue(config.check_repository_map(repomap)) + + def test_build_repository_map(self): + """Test build_repository_map function.""" + testdir1 = os.path.join(self.tmpdir, "repo") + testdir2 = os.path.join(self.tmpdir, "repo1") + os.makedirs(testdir1) + os.makedirs(testdir2) + self.assertFalse(config.build_repository_map(self.tmpdir)) + open(os.path.join(testdir1, config.cfgfile), 'a').close() + self.assertFalse(config.build_repository_map(self.tmpdir)) + + def test_read_config(self): + """Test read_config function.""" + testfile = os.path.join(self.tmpdir, config.cfgfile) + with open(testfile, 'w') as fih: + fih.write("\n") + self.assertFalse(config.read_config(testfile)) + with open(testfile, 'a') as fih: + fih.write("[docbuilder]\nsitesection=test\n") + self.assertFalse(config.read_config(testfile)) + with open(testfile, 'a') as fih: + fih.write("targets=test") + self.assertEquals(config.read_config(testfile), {'sitesection': 'test', + 'subdir': None, + 'targets': ['test']}) + with open(testfile, 'a') as fih: + fih.write(",test2") + self.assertEquals(config.read_config(testfile), {'sitesection': 'test', + 'subdir': None, + 'targets': ['test', 'test2']}) + with open(testfile, 'a') as fih: + fih.write(", \n") + self.assertEquals(config.read_config(testfile), {'sitesection': 'test', + 'subdir': None, + 'targets': ['test', 'test2']}) + with open(testfile, 'a') as fih: + fih.write("subdir=test3\n") + self.assertEquals(config.read_config(testfile), {'sitesection': 'test', + 'subdir': 'test3', + 'targets': ['test', 'test2']}) + + def suite(self): + """Return all the testcases in this module.""" + return TestLoader().loadTestsFromTestCase(ConfigTest) + +if __name__ == '__main__': + main() diff --git a/src/documentation_builder/test/markdownhandler.py b/src/documentation_builder/test/markdownhandler.py new file mode 100644 index 0000000000..46acc9895d --- /dev/null +++ b/src/documentation_builder/test/markdownhandler.py @@ -0,0 +1,114 @@ +"""Test class for markdownhandler.""" + +import sys +import os +import shutil +import filecmp +from tempfile import mkdtemp +from unittest import TestCase, main, TestLoader + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))) # noqa +from quattordocbuild import markdownhandler as mdh + + +class MarkdownHandlerTest(TestCase): + """Test class for markdownhandler.""" + + def setUp(self): + """Set up temmp dir for tests.""" + self.tmpdir = mkdtemp() + + def tearDown(self): + """Remove temp dir.""" + shutil.rmtree(self.tmpdir) + + def test_markdown_from_perl(self): + """Test convert_pod_to_markdown function.""" + testinput = "test/testdata/pod_test_input.pod" + testoutput = os.path.join(self.tmpdir, "test.md") + expectedoutput = "test/testdata/markdown_from_pod.md" + + # Get False from bogus input + self.assertFalse(mdh.markdown_from_perl("test")) + + # Verify content + output = mdh.markdown_from_perl(testinput) + file = open(testoutput, 'w') + file.write(output) + file.close() + self.assertTrue(filecmp.cmp(testoutput, expectedoutput)) + + def test_generate_markdown(self): + """Test generate_markdown.""" + testdir = os.path.join(self.tmpdir, "testdata/target") + testfile1 = os.path.join(testdir, "pan_annotated_schema.pan") + testfile2 = os.path.join(testdir, "pod_test_input.pod") + os.makedirs(testdir) + shutil.copy("test/testdata/pan_annotated_schema.pan", testfile1) + shutil.copy("test/testdata/pod_test_input.pod", testfile2) + output = mdh.generate_markdown([testfile1, testfile2]) + self.assertEquals(len(output), 2) + self.assertEquals(sorted(output.keys()), [testfile1, testfile2]) + + def test_remove_emails(self): + """Test remove_emails function.""" + mailtext = "Hello: mail@mailtest.mail mister." + self.assertEquals(mdh.remove_emails(mailtext), "Hello: mister.") + mailtext = "Hello: //mail@mailtest.mail mister." + self.assertEquals(mdh.remove_emails(mailtext), mailtext) + mailtext = "Hello: example@example.com" + self.assertEquals(mdh.remove_emails(mailtext), mailtext) + + def test_remove_headers(self): + """Test remove_headers function.""" + headertext = "This is a \n test TEXT with AUTH and ication." + self.assertEquals(mdh.remove_headers(headertext), headertext) + headertext = "This is # MAINTAINER and # AUTHOR text \n with newlines." + self.assertEquals(mdh.remove_headers(headertext), "This is and text \n with newlines.") + + def test_remove_whitespace(self): + """Test remove_whitespace function.""" + whitetext = "this \n\n\n\n\n is a bit too much." + self.assertEquals(mdh.remove_whitespace(whitetext), "this \n is a bit too much.") + whitetext = "this \n is much better \n\n." + self.assertEquals(mdh.remove_whitespace(whitetext), whitetext) + + def test_small_titles(self): + """Test small_titles function.""" + titletext = "\n\n# very big title\n it is." + self.assertEquals(mdh.small_titles(titletext), '\n\n### very big title\n it is.') + titletext = "\n## Smaller title." + self.assertEquals(mdh.small_titles(titletext), '\n#### Smaller title.') + titletext = "# test with set() and ## others Should \n# Deliver \n # not someting weird." + expected = '\n### test with set() and ## others Should \n### Deliver \n # not someting weird.' + self.assertEquals(mdh.small_titles(titletext), expected) + + def test_codify_paths(self): + """Test codify_paths function.""" + txt = "leave / me alone//" + self.assertEquals(mdh.codify_paths(txt), txt) + txt = " /but/please/change/me/yes" + self.assertEquals(mdh.codify_paths(txt), ' `/but/please/change/me/yes`') + txt = "\n /even/if/i/have/multiline/in/me/ \n" + self.assertEquals(mdh.codify_paths(txt), '\n `/even/if/i/have/multiline/in/me`/ \n') + txt = " /or/even/if/i/have \n\n /several/paths/that/have/coding " + self.assertEquals(mdh.codify_paths(txt), ' `/or/even/if/i/have` \n\n `/several/paths/that/have/coding` ') + + def test_cleanup_content(self): + """Test cleanup_content.""" + opts = { + 'codify_paths': True, + 'remove_emails': True, + 'remove_whitespace': True, + 'remove_headers': True, + 'small_titles': True + } + text = {"/tmp/testfile": "# MAINTAINER \n# Title \n\n\n\n\n /path/to/test/on \n test@test.com"} + self.assertEquals(mdh.cleanup_content(text, opts), {'/tmp/testfile': ' \n### Title \n `/path/to/test/on` \n '}) + + def suite(self): + """Return all the testcases in this module.""" + return TestLoader().loadTestsFromTestCase(MarkdownHandlerTest) + +if __name__ == '__main__': + main() diff --git a/src/documentation_builder/test/panhandler.py b/src/documentation_builder/test/panhandler.py new file mode 100644 index 0000000000..d3384cb47b --- /dev/null +++ b/src/documentation_builder/test/panhandler.py @@ -0,0 +1,223 @@ +"""Tests for panhandler.""" + +import sys +import os +import shutil +import filecmp +from tempfile import mkdtemp +from unittest import TestCase, main, TestLoader +from lxml import etree + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))) # noqa +from quattordocbuild import panhandler as panh + + +class PanHandlerTest(TestCase): + """Test class for panhandler.""" + + def setUp(self): + """Set up temmp dir for tests.""" + self.tmpdir = mkdtemp() + + def tearDown(self): + """Remove temp dir.""" + shutil.rmtree(self.tmpdir) + + def test_build_annotations(self): + """Test build_annotations function.""" + # Get False with bogus input + self.assertFalse(panh.build_annotations("testfile.pan", "test123", "test1234")) + + # Get True with valid input + self.assertTrue(panh.build_annotations("pan_annotated_schema.pan", "test/testdata/", self.tmpdir)) + + outputfile = os.path.join(self.tmpdir, "pan_annotated_schema.pan.annotation.xml") + # Test if the output file exists + self.assertTrue(os.path.exists(outputfile)) + + # Verify the content + self.assertTrue(filecmp.cmp("test/testdata/pan_annotated_output.xml", outputfile)) + + def test_validate_annotations(self): + """Test validate_annotations function.""" + # Test we skip empty files + self.assertIsNone(panh.validate_annotations("test/testdata/pan_empty_annotated_output.xml")) + # Test we get valid content back + self.assertIsNotNone(panh.validate_annotations("test/testdata/pan_annotated_output.xml")) + + def test_get_types_and_functions(self): + """Test get_types_and_functions function.""" + root = panh.validate_annotations("test/testdata/pan_annotated_output.xml") + self.assertIsNotNone(panh.get_types_and_functions(root)) + + el1 = self.create_element("test", "test") + self.assertEquals(panh.get_types_and_functions(el1), (None, None)) + + def create_element(self, name, text): + """Create a XML element from name and text.""" + element = etree.Element(name) + element.text = text + return element + + def test_find_description(self): + """Test find_description function.""" + # Test for an emtpy description + el1 = self.create_element("test", "test") + self.assertIsNone(panh.find_description(el1)) + + # Test for a description tag + el2 = self.create_element("%sdesc" % panh.namespace, "test") + el1.append(el2) + self.assertEquals(panh.find_description(el1).text, "test") + + # Test for a description tag within a documentation tag + el3 = self.create_element("test", "test") + el4 = self.create_element("%sdocumentation" % panh.namespace, "test") + el2.text = "test2" + el4.append(el2) + el3.append(el4) + self.assertEquals(panh.find_description(el3).text, "test2") + + def test_parse_type(self): + """Test parse_type function.""" + # Test empty element + el1 = self.create_element("type", "") + self.assertEquals(panh.parse_type(el1), {'fields': [], 'name': None}) + + # Test name + el1.attrib['name'] = "test" + self.assertEquals(panh.parse_type(el1), {'fields': [], 'name': 'test'}) + + # Test description + el2 = self.create_element("%sdesc" % panh.namespace, "testdesc") + el1.append(el2) + self.assertEquals(panh.parse_type(el1), {'desc': 'testdesc', 'fields': [], 'name': 'test'}) + + # Test field + el3 = self.create_element("basetype", "") + el4 = self.create_element("%sfield" % panh.namespace, "") + el4.attrib['name'] = 'debug' + el4.attrib['required'] = 'true' + el5 = self.create_element("%sbasetype" % panh.namespace, "") + el5.attrib['name'] = 'string' + el4.append(el5) + el3.append(el4) + el1.append(el3) + expectedoutput = {'desc': 'testdesc', + 'fields': [{'name': 'debug', 'required': 'true', 'type': 'string'}], + 'name': 'test'} + self.assertEquals(panh.parse_type(el1), expectedoutput) + + def test_parse_function(self): + """Test parse_function function.""" + # Test empty input + el1 = self.create_element("function", "") + self.assertEquals(panh.parse_function(el1), {'args': [], 'name': None}) + + # Test name + el1.attrib['name'] = "testfunction" + self.assertEquals(panh.parse_function(el1), {'args': [], 'name': 'testfunction'}) + + # Test description + el2 = self.create_element("%sdesc" % panh.namespace, "testdesc") + el1.append(el2) + self.assertEquals(panh.parse_function(el1), {'args': [], 'name': 'testfunction', 'desc': 'testdesc'}) + + # Test args + el3 = self.create_element("%sarg" % panh.namespace, "testargument") + el4 = self.create_element("%sarg" % panh.namespace, "testargument2") + el2.append(el3) + el2.append(el4) + expectedoutput = {'args': ['testargument', 'testargument2'], 'name': 'testfunction', 'desc': 'testdesc'} + self.assertEquals(panh.parse_function(el1), expectedoutput) + + def test_render_template(self): + """Test render_template function.""" + content = panh.get_content_from_pan("test/testdata/pan_annotated_schema.pan") + testfile = open(os.path.join(self.tmpdir, "test.md"), 'w') + + output = panh.render_template(content, "component-test") + print output + for line in output: + testfile.write(line) + testfile.close() + + self.assertTrue(filecmp.cmp("test/testdata/markdown_from_pan.md", testfile.name)) + + # Test with only fields + content = {'functions': [{'args': ['first number to add'], 'name': 'add'}]} + + output = panh.render_template(content, "component-test") + print output + + expectedoutput = "\n### Functions\n\n - add\n - Arguments:\n - first number to add\n" + self.assertEquals(output, expectedoutput) + + # Test with only Types + content = {'types': [{'fields': [{'required': 'false', 'type': 'string', 'name': 'ca'}], 'name': 'testtype'}]} + output = panh.render_template(content, "component-test") + print output + expectedoutput = "\n### Types\n\n - `/software/component-test/testtype`\n - `/software/component-test/testtype/ca`\n\ + - Optional\n - Type: string\n" + self.assertEquals(output, expectedoutput) + + def test_get_content_from_pan(self): + """Test get_content_from_pan function.""" + # Test with empty pan input file. + self.assertEqual(panh.get_content_from_pan("test/testdata/pan_empty_input.pan"), {}) + + expectedresult = {'functions': + [{'args': ['first number to add', + 'second number to add'], + 'name': 'add', + 'desc': 'simple addition of two numbers'}], + 'types': [ + {'fields': [ + {'range': '0..1', + 'required': 'true', + 'type': 'long', + 'name': 'debug', + 'desc': 'Test long.'}, + {'required': 'false', + 'type': 'string', + 'name': 'ca_dir', + 'desc': 'Test string'}], + 'name': 'testtype', + 'desc': 'test type.'}]} + + # Test with valid input. + self.assertEqual(panh.get_content_from_pan("test/testdata/pan_annotated_schema.pan"), expectedresult) + + def test_markdown_from_pan(self): + """Test markdown_from_pan function.""" + # Test valid input + testdir = os.path.join(self.tmpdir, "testdata/target") + testfile = os.path.join(testdir, "pan_annotated_schema.pan") + os.makedirs(testdir) + shutil.copy("test/testdata/pan_annotated_schema.pan", testfile) + testoutput = panh.markdown_from_pan(testfile) + print testoutput + self.assertEqual(len(testoutput), 484) + self.assertTrue("Types" in testoutput) + self.assertTrue("Functions" in testoutput) + + # Test invalid input + testdir = os.path.join(self.tmpdir, "testdata2/target") + testfile = os.path.join(testdir, "pan_empty_input.pan") + os.makedirs(testdir) + shutil.copy("test/testdata/pan_empty_input.pan", testfile) + self.assertIsNone(panh.markdown_from_pan(testfile)) + + def test_get_basename(self): + """Test get_basename function.""" + test = "/tmp/test/src/modules-core/ncm-useraccess/target/pan/components/useraccess/config-common.pan" + self.assertEquals(panh.get_basename(test), "useraccess") + test = "/tmp/test/src/modules-grid/ncm-condorconfig/target/pan/components/condorconfig/schema.pan" + self.assertEquals(panh.get_basename(test), "condorconfig") + + def suite(self): + """Return all the testcases in this module.""" + return TestLoader().loadTestsFromTestCase(PanHandlerTest) + +if __name__ == '__main__': + main() diff --git a/src/documentation_builder/test/sourcehandler.py b/src/documentation_builder/test/sourcehandler.py new file mode 100644 index 0000000000..9388a63790 --- /dev/null +++ b/src/documentation_builder/test/sourcehandler.py @@ -0,0 +1,118 @@ +"""Test class for sourcehandler.""" +import os +import sys +import shutil +from tempfile import mkdtemp +from unittest import TestCase, main, TestLoader + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))) # noqa +from quattordocbuild import sourcehandler + + +class SourcehandlerTest(TestCase): + """Test class for sourcehandler.""" + + def setUp(self): + """Set up temp dir for tests.""" + self.tmpdir = mkdtemp() + + def tearDown(self): + """Remove temp dir.""" + shutil.rmtree(self.tmpdir) + + def test_maven_clean_compile(self): + """Test maven_clean_compile.""" + repoloc = os.path.join(self.tmpdir, "test") + os.makedirs(repoloc) + + # test if it fails on a empty dir + self.assertNotEqual(sourcehandler.maven_clean_compile(repoloc), 0) + + # test if it can run a basic pom.xml and return 0 + file = open(os.path.join(repoloc, "pom.xml"), "w") + file.write('4.0.0test') + file.write('test1') + file.close() + + self.assertEqual(sourcehandler.maven_clean_compile(repoloc), 0) + + def test_is_wanted_file(self): + """Test is_wanted_file function.""" + # Test valid extensions + for extension in ['pod', 'pm', 'pl', 'pan']: + testfile = "test.%s" % extension + self.assertTrue(sourcehandler.is_wanted_file('', testfile)) + + # Test invalid extensions + for extension in ['tpl', 'txt', 'xml']: + testfile = "test.%s" % extension + self.assertFalse(sourcehandler.is_wanted_file('', testfile)) + + # Test valid shebang + testfilename = "test" + file = open(os.path.join(self.tmpdir, testfilename), "w") + file.write('#!/usr/bin/perl\n') + file.close() + self.assertTrue(sourcehandler.is_wanted_file(self.tmpdir, testfilename)) + + # Test invalid shebang + file = open(os.path.join(self.tmpdir, testfilename), "w") + file.write('#!/usr/bin/python\n') + file.close() + self.assertFalse(sourcehandler.is_wanted_file(self.tmpdir, testfilename)) + + def test_is_wanted_dir(self): + """Test is_wanted_dir function.""" + # Test we get False if target is not in the given path + self.assertFalse(sourcehandler.is_wanted_dir('/bogusdir/test', [])) + + # Test for False on empty fileslist + self.assertFalse(sourcehandler.is_wanted_dir('/tmp/target/test/', [])) + + # Test for wrong subdir + self.assertFalse(sourcehandler.is_wanted_dir('/tmp/target/test/', ['schema.pan'])) + + # Test for a correct path + self.assertTrue(sourcehandler.is_wanted_dir('/tmp/target/doc/pod', ['schema.pan'])) + + def test_handle_duplicates(self): + """Test handle_duplicates function.""" + testperlfile = 'test/lib/perl/test.pm' + testpodfile = 'test/doc/pod/test.pod' + + # Add a correct item to an empty list + self.assertEquals(sourcehandler.handle_duplicates('test.pm', testperlfile, []), + ['test/lib/perl/test.pm']) + + # Add a pod file when a pm file is in the list, get a list with only the pod file back + self.assertEquals(sourcehandler.handle_duplicates('test.pod', testpodfile, [testperlfile]), [testpodfile]) + + # Add a pm file when a pod file is in the list, get a list with only the pod file back + self.assertEquals(sourcehandler.handle_duplicates('test.pm', testperlfile, [testpodfile]), [testpodfile]) + + def test_list_source_files(self): + """Test list_source_files function.""" + # Test a bogus dir + self.assertEquals(sourcehandler.list_source_files(self.tmpdir), []) + + # Test a correct dir + testfile = 'test.pod' + fulltestdir = os.path.join(self.tmpdir, 'target/doc/pod') + os.makedirs(fulltestdir) + file = open(os.path.join(fulltestdir, testfile), 'w') + file.write("test\n") + file.close() + + self.assertEquals(sourcehandler.list_source_files(fulltestdir), [os.path.join(fulltestdir, testfile)]) + + def test_get_source_files(self): + """Test get_source_files function.""" + self.assertEquals(sourcehandler.get_source_files(self.tmpdir, False), []) + self.assertFalse(sourcehandler.get_source_files(self.tmpdir, True)) + + def suite(self): + """Return all the testcases in this module.""" + return TestLoader().loadTestsFromTestCase(SourcehandlerTest) + +if __name__ == '__main__': + main() diff --git a/src/documentation_builder/test/test_common.py b/src/documentation_builder/test/test_common.py new file mode 100644 index 0000000000..2d2e5abb64 --- /dev/null +++ b/src/documentation_builder/test/test_common.py @@ -0,0 +1,49 @@ +""" +Test class for common tests. + +Mainly runs prospector on project. +""" + +import sys +import os +from prospector.run import Prospector +from prospector.config import ProspectorConfig +from unittest import TestCase, main, TestLoader +import pprint + + +class CommonTest(TestCase): + """Class for all common tests.""" + + def setUp(self): + """Cleanup after running a test.""" + self.orig_sys_argv = sys.argv + self.REPO_BASE_DIR = os.path.dirname(os.path.abspath(sys.argv[0])) + super(CommonTest, self).setUp() + + def tearDown(self): + """Cleanup after running a test.""" + sys.argv = self.orig_sys_argv + super(CommonTest, self).tearDown() + + def test_prospector(self): + """Run prospector on project.""" + sys.argv = ['fakename'] + sys.argv.append(self.REPO_BASE_DIR) + + config = ProspectorConfig() + prospector = Prospector(config) + prospector.execute() + + failures = [] + for msg in prospector.get_messages(): + failures.append(msg.as_dict()) + + self.assertFalse(failures, "prospector failures: %s" % pprint.pformat(failures)) + + def suite(self): + """Return all the testcases in this module.""" + return TestLoader().loadTestsFromTestCase(CommonTest) + +if __name__ == '__main__': + main() diff --git a/src/documentation_builder/test/testdata/markdown_from_pan.md b/src/documentation_builder/test/testdata/markdown_from_pan.md new file mode 100644 index 0000000000..2207f0a6f4 --- /dev/null +++ b/src/documentation_builder/test/testdata/markdown_from_pan.md @@ -0,0 +1,22 @@ + +### Types + + - `/software/component-test/testtype` + - Description: test type. + - `/software/component-test/testtype/debug` + - Description: Test long. + - Optional + - Type: long + - Range: 0..1 + - `/software/component-test/testtype/ca_dir` + - Description: Test string + - Optional + - Type: string + +### Functions + + - add + - Description: simple addition of two numbers + - Arguments: + - first number to add + - second number to add diff --git a/src/documentation_builder/test/testdata/markdown_from_pod.md b/src/documentation_builder/test/testdata/markdown_from_pod.md new file mode 100644 index 0000000000..affd7e85b2 --- /dev/null +++ b/src/documentation_builder/test/testdata/markdown_from_pod.md @@ -0,0 +1,7 @@ +# Test pod file + +## Really a pod file + +This is some text + +- this is an item diff --git a/src/documentation_builder/test/testdata/mkdocs.yml b/src/documentation_builder/test/testdata/mkdocs.yml new file mode 100644 index 0000000000..3a5daadd2a --- /dev/null +++ b/src/documentation_builder/test/testdata/mkdocs.yml @@ -0,0 +1,11 @@ +site_name: Quattor Documentation + +theme: 'readthedocs' + +pages: +- introduction: 'index.md' +- CCM: + - fetch::download: CCM/fetch::download.md +- components: + - fmonagent: components/fmonagent.md + - profile::functions: components/profile::functions.md diff --git a/src/documentation_builder/test/testdata/pan_annotated_output.xml b/src/documentation_builder/test/testdata/pan_annotated_output.xml new file mode 100644 index 0000000000..a494453848 --- /dev/null +++ b/src/documentation_builder/test/testdata/pan_annotated_output.xml @@ -0,0 +1,24 @@ + diff --git a/src/documentation_builder/test/testdata/pan_annotated_schema.pan b/src/documentation_builder/test/testdata/pan_annotated_schema.pan new file mode 100644 index 0000000000..420efcdf0f --- /dev/null +++ b/src/documentation_builder/test/testdata/pan_annotated_schema.pan @@ -0,0 +1,18 @@ +declaration template pan_annotated_schema; + +@documentation {test type.} +type testtype = { + @{Test long.} + 'debug' : long(0..1) = 0 + @{Test string} + 'ca_dir' ? string +}; + +@documentation{ + desc = simple addition of two numbers + arg = first number to add + arg = second number to add +} +function add = { + ARGV[0] + ARGV[1]; +}; diff --git a/src/documentation_builder/test/testdata/pan_empty_annotated_output.xml b/src/documentation_builder/test/testdata/pan_empty_annotated_output.xml new file mode 100644 index 0000000000..9783608acb --- /dev/null +++ b/src/documentation_builder/test/testdata/pan_empty_annotated_output.xml @@ -0,0 +1,2 @@ + diff --git a/src/documentation_builder/test/testdata/pan_empty_input.pan b/src/documentation_builder/test/testdata/pan_empty_input.pan new file mode 100644 index 0000000000..c5f22430f6 --- /dev/null +++ b/src/documentation_builder/test/testdata/pan_empty_input.pan @@ -0,0 +1 @@ +declaration template paninput; diff --git a/src/documentation_builder/test/testdata/pod_test_input.pod b/src/documentation_builder/test/testdata/pod_test_input.pod new file mode 100644 index 0000000000..50657df431 --- /dev/null +++ b/src/documentation_builder/test/testdata/pod_test_input.pod @@ -0,0 +1,15 @@ +=pod + +=head1 Test pod file + +=head2 Really a pod file + +This is some text + +=over 4 + +=item this is an item + +=back + +=cut diff --git a/src/releasing/quattorpoddoc.py b/src/releasing/quattorpoddoc.py deleted file mode 100755 index e478ab7ca9..0000000000 --- a/src/releasing/quattorpoddoc.py +++ /dev/null @@ -1,513 +0,0 @@ -#!/usr/bin/env python2 -""" -quattorpoddoc generates markdown documentation from: - - configuration-modules-core perl documentation - - configuration-modules-grid perl documentation - - CAF perl documentation - - CCM perl documentation - - schema pan annotations - and creates a index for the website on http://quattor.org. - -@author: Wouter Depypere (Ghent University) - -""" -import sys -import os -import re -from vsc.utils.generaloption import simple_option -from vsc.utils import fancylogger -from vsc.utils.run import run_asyncloop -import tempfile -from lxml import etree -import shutil - -MAILREGEX = re.compile(("([a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`" - "{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|" - "\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)")) -PATHREGEX = re.compile(r'(\s+)((?:/[\w{}]+)+\.?\w*)(\s*)') -EXAMPLEMAILS = ["example", "username", "system.admin"] -logger = fancylogger.getLogger() -DOCDIR = "docs" - -REPOMAP = { - "configuration-modules-core": { - "sitesubdir": "components", - "target": "/NCM/Component/" - }, - "configuration-modules-grid":{ - "sitesubdir": "components-grid", - "target": "/NCM/Component/" - }, - "CAF": { - "sitesubdir": "CAF", - "target": "/CAF/" - }, - "CCM": { - "sitesubdir": "CCM", - "target": "EDG/WP4/CCM/" - }, - } - - -def maven_clean_compile(repo_location, repository): - """ - Executes mvn clean and mvn compile in the given modules_location. - """ - repoloc = os.path.join(repo_location, repository) - logger.info("Doing maven clean compile in %s." % repoloc) - output = run_asyncloop("mvn clean compile", startpath=repoloc) - logger.debug(output) - - -def generate_mds(repo, sources, location): - """ - Takes a list of components with podfiles and generates a md file for it. - """ - logger.info("Generating md files.") - mdfiles = set() - - comppath = os.path.join(location, DOCDIR, REPOMAP[repo]['sitesubdir']) - if not os.path.exists(comppath): - os.makedirs(comppath) - - for source in sources: - tpl = False - if source.endswith(".tpl"): - logger.warning("%s file extension is '.tpl', should become '.pan'." % source) - tpl = True - - if source.endswith(".pan") or tpl: - mdfile = create_md_from_pan(source, comppath) - if mdfile is not None: - mdfiles.add(mdfile) - else: - sourcename = source.split(REPOMAP[repo]['target'])[-1] - mdfile = os.path.splitext(sourcename)[0].replace("/", "::").lower() + ".md" - - convert_pod_to_markdown(source, os.path.join(comppath, mdfile)) - mdfiles.add(mdfile) - - logger.info("Written %s md files." % len(mdfiles)) - return mdfiles - -def find_description(sub, namespace): - """ - Search for the desc tag, even if it is in documentation tag. - """ - doc = sub.find("./%sdocumentation/%sdesc" % (namespace, namespace)) - if doc is None: - doc = sub.find("./%sdesc" % namespace) - return doc - -def create_md_from_pan(source, comppath): - """ - Takes a pan schema, creates the pan annotations and parses them to markdown. - """ - modname = os.path.basename(os.path.split(source)[0]) - templatename = os.path.splitext(os.path.basename(source))[0] - mdfile = "%s::%s.md" % (modname, templatename) - tmpdir = tempfile.mkdtemp() - logger.debug("Temporary directory: %s" % tmpdir) - panccommand = ["panc-annotations", "--output-dir", tmpdir, "--base-dir"] - panccommand.extend(os.path.split(source)) - output = run_asyncloop(panccommand) - logger.debug(output) - namespace = "{http://quattor.org/pan/annotations}" - - tpl = "%s.pan.annotation.xml" % templatename - xml = etree.parse(os.path.join(tmpdir, tpl)) - root = xml.getroot() - - if len(root) == 0: - logger.debug("%s would be empty, skipping it." % mdfile) - return None - - stypes = root.findall('%stype' % namespace) - deffunctions = root.findall('%sfunction' % namespace) - - if len(stypes) == 0 and len(deffunctions) == 0: - logger.debug("%s has no usable content, skipping it." % mdfile) - return None - - mdtext = [] - - mdtext.append("# Types\n") - for stype in stypes: - name = stype.get('name') - mdtext.append("- /software/%s/%s" % (modname, name)) - - doc = find_description(stype, namespace) - if doc is not None: - mdtext.append("%s- decription: %s" % (" "*4, doc.text)) - - for field in stype.findall(".//%sfield" % namespace): - mdtext.append("%s- /software/%s/%s/%s" % (" "*4, modname, name, field.get('name'))) - - doct = find_description(field, namespace) - if doct is not None: - mdtext.append("%s- description: %s" % (" "*8, doct.text)) - - required = field.get('required') - if required == "true": - mdtext.append("%s- required" % (" "*8)) - else: - mdtext.append("%s- optional" % (" "*8)) - - for basetype in field.findall(".//%sbasetype" % namespace): - fieldtype = basetype.get('name') - mdtext.append("%s- type: %s" % (" "*8, fieldtype)) - if fieldtype == "long" and basetype.get('range'): - fieldrange = basetype.get('range') - mdtext.append("%s- range: %s" % (" "*8, fieldrange)) - mdtext.append("\n") - - if deffunctions: - mdtext.append("\n# Functions\n") - root.findall('%sfunction' % namespace) - for fnname in deffunctions: - name = fnname.get('name') - mdtext.append("- %s" % name) - - doc = find_description(fnname, namespace) - if doc is not None: - mdtext.append("%s- description: %s " % (" "*4, doc.text)) - - for arg in fnname.findall(".//%sarg" % namespace): - mdtext.append("%s- arg: %s" % (" "*4, arg.text)) - - with open(os.path.join(comppath, mdfile), "w") as fih: - fih.write("\n".join(mdtext)) - - logger.debug("Removing temporary directory: %s" % tmpdir) - shutil.rmtree(tmpdir) - return mdfile - - -def convert_pod_to_markdown(podfile, outputfile): - """ - Takes a podfile and converts it to a markdown with the help of pod2markdown. - """ - logger.debug("Running pod2markdown on %s." % podfile) - output = run_asyncloop("pod2markdown %s" % podfile) - logger.debug("writing output to %s." % outputfile) - logger.debug(output) - with open(outputfile, "w") as fih: - fih.write(output[1]) - - -def generate_toc(mdfiles, outputloc, indexname): - """ - Generates a TOC for the parsed components. - """ - logger.info("Generating TOC as %s." % os.path.join(outputloc, indexname)) - - with open(os.path.join(outputloc, indexname), "w") as fih: - fih.write("site_name: Quattor Documentation\n\n") - fih.write("theme: 'readthedocs'\n\n") - fih.write("pages:\n") - fih.write("- introduction: 'index.md'\n") - - for subdivision in sorted(mdfiles.keys()): - fih.write("- %s:\n" % subdivision) - for page in sorted(mdfiles[subdivision]): - linkname = page.split(".")[0] - linktopage = "%s/%s" % (subdivision, page) - write_if_exists(outputloc, linktopage, linkname, fih) - fih.write("\n") - - -def write_if_exists(outputloc, linktopage, linkname, fih): - """ - Checks if the MD exists before adding it to the TOC. - """ - if os.path.exists(os.path.join(outputloc, DOCDIR, linktopage)): - logger.debug("Adding %s to toc." % linkname) - fih.write(" - %s: '%s'\n" % (linkname, linktopage)) - else: - logger.warn("Expected %s but it does not exist. Not adding to toc." - % os.path.join(outputloc, DOCDIR, linktopage)) - - -def remove_mail(mdfiles, outputloc): - """ - Removes the email addresses from the markdown files. - """ - logger.info("Removing emailaddresses from md files.") - counter = 0 - for subdivision in mdfiles.keys(): - for mdfile in mdfiles[subdivision]: - mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile) - with open(mdfileloc, 'r') as fih: - mdcontent = fih.read() - replace = False - for email in re.findall(MAILREGEX, mdcontent): - logger.debug("Found %s." % email[0]) - replace = True - if email[0].startswith('//'): - replace = False - for ignoremail in EXAMPLEMAILS: - if ignoremail in email[0]: - replace = False - - if replace: - logger.debug("Removed it from line.") - mdcontent = mdcontent.replace(email[0], '') - with open(mdfileloc, 'w') as fih: - fih.write(mdcontent) - counter += 1 - logger.info("Removed %s email addresses." % counter) - - -def remove_whitespace(mdfiles, outputloc): - """ - Removes extra whitespace (\n\n\n). - """ - logger.info("Removing extra whitespace from md files.") - counter = 0 - for subdivision in mdfiles.keys(): - for mdfile in mdfiles[subdivision]: - mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile) - with open(mdfileloc, 'r') as fih: - mdcontent = fih.read() - if '\n\n\n' in mdcontent: - logger.debug("Removing whitespace in %s." % mdfile) - mdcontent = mdcontent.replace('\n\n\n', '\n') - with open(mdfileloc, 'w') as fih: - fih.write(mdcontent) - counter += 1 - logger.info("Removed extra whitespace from %s files." % counter) - - -def decrease_title_size(mdfiles, outputloc): - """ - Makes titles smaller, e.g. replace "# " with "### ". - """ - logger.info("Downsizing titles in md files.") - counter = 0 - for subdivision in mdfiles.keys(): - for mdfile in mdfiles[subdivision]: - mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile) - with open(mdfileloc, 'r') as fih: - mdcontent = fih.read() - if '# ' in mdcontent: - logger.debug("Making titles smaller in %s." % mdfile) - mdcontent = mdcontent.replace('# ', '### ') - with open(mdfileloc, 'w') as fih: - fih.write(mdcontent) - counter += 1 - logger.info("Downsized titles in %s files." % counter) - - -def remove_headers(mdfiles, outputloc): - """ - Removes MAINTAINER and AUTHOR headers from md files. - """ - logger.info("Removing AUTHOR and MAINTAINER headers from md files.") - counter = 0 - for subdivision in mdfiles.keys(): - for mdfile in mdfiles[subdivision]: - mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile) - with open(mdfileloc, 'r') as fih: - mdcontent = fih.read() - if '# MAINTAINER' in mdcontent: - logger.debug("Removing # MAINTAINER in %s." % mdfile) - mdcontent = mdcontent.replace('# MAINTAINER', '') - with open(mdfileloc, 'w') as fih: - fih.write(mdcontent) - counter += 1 - if '# AUTHOR' in mdcontent: - logger.debug("Removing # AUTHOR in %s." % mdfile) - mdcontent = mdcontent.replace('# AUTHOR', '') - with open(mdfileloc, 'w') as fih: - fih.write(mdcontent) - counter += 1 - - logger.info("Removed %s unused headers." % counter) - - -def codify_paths(mdfiles, outputloc): - """ - Puts paths inside code tags - """ - logger.info("Putting paths inside code tags.") - counter = 0 - for subdivision in mdfiles.keys(): - for mdfile in mdfiles[subdivision]: - mdfileloc = os.path.join(outputloc, DOCDIR, subdivision, mdfile) - with open(mdfileloc, 'r') as fih: - mdcontent = fih.read() - - logger.debug("Tagging paths in %s." % mdfile) - mdcontent, counter = PATHREGEX.subn(r'\1`\2`\3', mdcontent) - with open(mdfileloc, 'w') as fih: - fih.write(mdcontent) - - logger.info("Code tagged %s paths." % counter) - - -def check_input_and_commands(sourceloc, outputloc, runmaven): - """ - Check if the directories are in place. - Check if the required binaries are in place. - """ - logger.info("Checking if the given paths exist.") - if not sourceloc: - logger.error("Repo location not specified.") - sys.exit(1) - if not outputloc: - logger.error("output location not specified") - sys.exit(1) - if not os.path.exists(sourceloc): - logger.error("Repo location %s does not exist" % sourceloc) - sys.exit(1) - for repo in REPOMAP.keys(): - if not os.path.exists(os.path.join(sourceloc, repo)): - logger.error("Repo location %s does not exist" % os.path.join(sourceloc, repo)) - sys.exit(1) - if not os.path.exists(outputloc): - logger.error("Output location %s does not exist" % outputloc) - sys.exit(1) - - logger.info("Checking if required binaries are installed.") - if runmaven: - if not which("mvn"): - logger.error("The command mvn is not available on this system, please install maven.") - sys.exit(1) - if not which("pod2markdown"): - logger.error("The command pod2markdown is not available on this system, please install pod2markdown.") - sys.exit(1) - - -def list_perl_modules(module_location): - """ - return a list of perl modules in module_location. - Try to filter out duplicates, pod takes precedence over pm. - """ - finallist = [] - for path, _, files in os.walk(module_location): - if not is_wanted_dir(path, files): - continue - - for mfile in files: - if is_wanted_file(path, mfile): - fname = os.path.join(path, mfile) - duplicate = "" - if "doc/pod" in fname: - duplicate = fname.replace('doc/pod', 'lib/perl') - if mfile.endswith('.pod'): - duplicate = duplicate.replace(".pod", ".pm") - if duplicate in finallist: - finallist[finallist.index(duplicate)] = fname - continue - - duplicate = "" - if "lib/perl" in fname: - duplicate = fname.replace('lib/perl', 'doc/pod') - if mfile.endswith('.pm'): - duplicate = duplicate.replace(".pm", ".pod") - if duplicate not in finallist: - finallist.append(fname) - else: - continue - finallist.append(os.path.join(path, mfile)) - - return finallist - - -def is_wanted_dir(path, files): - """ - Check if the directory matches required criteria: - - It must be in 'target directory' - - It should contain files - - it shoud contain one of the following in the path: - - doc/pod - - lib/perl - - pan - """ - if 'target' not in path: return False - if len(files) == 0: return False - if True not in [substr in path for substr in ["doc/pod", "lib/perl", "pan"]]: return False - return True - - -def is_wanted_file(path, filename): - """ - Check if the file matches one of the criteria: - - a perl file based on extension - - schema.pan - - a perl file based on shebang - """ - if True in [filename.endswith(ext) for ext in [".pod", ".pm", ".pl"]]: return True - if filename.endswith(".pan"): return True - if len(filename.split(".")) < 2: - with open(os.path.join(path, filename), 'r') as pfile: - if 'perl' in pfile.readline(): return True - return False - - -def which(command): - """ - Check if given command is available for the current user on this system. - """ - found = False - for direct in os.getenv("PATH").split(':'): - if os.path.exists(os.path.join(direct, command)): - found = True - - return found - - -def main(repoloc, outputloc): - """ - Main run of the script. - """ - mdfiles = {} - for repo in REPOMAP.keys(): - logger.info("Processing %s." % repo) - if GO.options.maven_compile: - logger.info("Doing maven clean and compile.") - maven_clean_compile(repoloc, repo) - else: - logger.info("Skipping maven clean and compile.") - - pmodules = list_perl_modules(os.path.join(repoloc, repo)) - mdfiles[REPOMAP[repo]['sitesubdir']] = generate_mds(repo, pmodules, outputloc) - - generate_toc(mdfiles, outputloc, GO.options.index_name) - - if GO.options.remove_emails: - remove_mail(mdfiles, outputloc) - - if GO.options.remove_headers: - remove_headers(mdfiles, outputloc) - - if GO.options.small_titles: - decrease_title_size(mdfiles, outputloc) - - if GO.options.remove_whitespace: - remove_whitespace(mdfiles, outputloc) - - if GO.options.codify_paths: - codify_paths(mdfiles, outputloc) - - -if __name__ == '__main__': - OPTIONS = { - 'modules_location': ('The location of the repo checkout.', None, 'store', None, 'm'), - 'output_location': ('The location where the output markdown files should be written to.', None, 'store', None, 'o'), - 'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None, 'store_true', False, 'c'), - 'index_name': ('Filename for the index/toc for the components.', None, 'store', 'mkdocs.yml', 'i'), - 'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'), - 'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'), - 'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'), - 'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's'), - 'codify_paths': ('Put paths inside code tags.', None, 'store_true', True, 'p'), - } - GO = simple_option(OPTIONS) - logger.info("Starting main.") - - check_input_and_commands(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile) - - main(GO.options.modules_location, GO.options.output_location) - - logger.info("Done.")