rewrite/cleanup for documentation builder

src/documentation_builder/.coveragerc

@@ -0,0 +1,10 @@
+[run]
+omit =
+    # omit anything in a .local directory anywhere
+    */.local/*
+    # omit everything in /usr
+    /usr/*
+    # omit the test directory
+    */test/*
+    # omit everything in /tmp
+    /tmp/*

src/documentation_builder/.gitignore

@@ -0,0 +1,26 @@
+## aptana/eclipse
+*.project
+*.pydevproject
+## python
+*.pyc
+*.pyo
+.eggs
+## emacs
+*~
+.#*
+*.orig
+*.bak
+## Various build products
+*.rpm
+*.o
+*.old
+*.DS_Store
+*.egg
+*.egg-info
+.coverage
+htmlcov/
+## Python __init__.py files for distributed packages
+MANIFEST
+dist/
+build/
+setup.cfg

src/documentation_builder/.prospector.yaml

@@ -0,0 +1,17 @@
+strictness: veryhigh
+test-warnings: true
+doc-warnings: true
+member-warnings: true
+
+ignore-paths:
+  - build
+  - htmlcov
+
+pep8:
+    disable:
+    options:
+        max-line-length: 120
+
+pep257:
+    disable:
+     - D203

src/documentation_builder/MANIFEST.in

@@ -0,0 +1,3 @@
+include lib/quattordocbuild/pan.j2
+include lib/quattordocbuild/toc.j2
+include bin/build-quattor-documentation.sh

src/documentation_builder/README.rst

@@ -0,0 +1,54 @@
+Documentation-builder
+---------------------
+
+Documentation builder for the Quattor repositories.
+
+::
+ $ quattor-documentation-builder --help
+ Usage: quattor-documentation-builder [options]
+
+
+  Documentation-builder generates markdown documentation.
+
+  It get this from:
+   - configuration-modules-core perl documentation
+   - configuration-modules-grid perl documentation
+   - CAF perl documentation
+   - CCM perl documentation
+   - schema pan annotations
+   and creates a index for the website on http://quattor.org.
+  @author: Wouter Depypere (Ghent University)
+
+ Options:
+   -h, --shorthelp       show short help message and exit
+   -H OUTPUT_FORMAT, --help=OUTPUT_FORMAT
+                         show full help message and exit
+   --confighelp          show help as annotated configfile
+
+  Main options (configfile section MAIN):
+    -p, --codify_paths  Put paths inside code tags. (def True)
+    -i INDEX_NAME, --index_name=INDEX_NAME
+                        Filename for the index/toc for the components. (def mkdocs.yml)
+    -c, --maven_compile
+                        Execute a maven clean and maven compile before generating the documentation. (def False)
+    -m MODULES_LOCATION, --modules_location=MODULES_LOCATION
+                        The location of the repo checkout.
+    -o OUTPUT_LOCATION, --output_location=OUTPUT_LOCATION
+                        The location where the output markdown files should be written to.
+    -r, --remove_emails
+                        Remove email addresses from generated md files. (def True)
+    -R, --remove_headers
+                        Remove unneeded headers from files (MAINTAINER and AUTHOR). (def True)
+    -w, --remove_whitespace
+                        Remove whitespace (   ) from md files. (def True)
+    -s, --small_titles  Decrease the title size in the md files. (def True)
+
+  Debug and logging options (configfile section MAIN):
+    -d, --debug         Enable debug log mode (def False)
+    --info              Enable info log mode (def False)
+    --quiet             Enable quiet/warning log mode (def False)
+
+
+
+It makes some assumpions on several repositories being in place.
+To help set this up a helper script was added **build-quattor-documentation.sh** which builds the whole documentation from latest master.

src/documentation_builder/bin/build-quattor-documentation.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+VERSION="master"
+
+#!/bin/bash
+while getopts "h:v:" arg; do
+  case $arg in
+    h)
+      echo "usage: use '-v' to specify another branch than master."
+      ;;
+    s)
+      VERSION=$OPTARG
+      ;;
+  esac
+done
+
+for com in 'mkdocs' 'mvn' 'pod2markdown' 'bundle'; do
+    command -v $com >/dev/null 2>&1 || { echo >&2 "I require $com but it's not installed.  Aborting."; exit 1; }
+done
+
+tmpdir="/tmp/quattor-documentation"
+
+# Create temporary working directory
+mkdir -p $tmpdir/{src,output}
+cd $tmpdir/src
+
+# Clone required github repositories
+for REPO in CAF configuration-modules-core configuration-modules-grid CCM ;  do
+    git clone https://github.com/quattor/$REPO.git
+    cd $REPO
+    tag=`git tag -l | grep "$VERSION$"`
+    git checkout -q $tag
+    cd ..
+done
+cd ..
+
+# Build the whole documentation
+quattor-documentation-builder -c -m $tmpdir/src/ -o $tmpdir/output/ --info || { echo 'Something went wrong building documentation.' ; exit 1 ; }
+
+# get required index which is not generated
+curl https://raw.githubusercontent.com/quattor/documentation/master/docs/index.md -o $tmpdir/output/docs/index.md
+
+cd $tmpdir/output
+mkdocs build --clean
+
+# Get some tests up
+curl https://raw.githubusercontent.com/quattor/documentation/master/Gemfile -o Gemfile
+bundle
+
+bundle exec htmlproofer  --check-html ./site/ --file-ignore ./site/base.html,./site/breadcrumbs.html,./site/footer.html,./site/toc.html,./site/versions.html || { echo 'build test errors detected. stopping.' ; exit 1 ; }

src/documentation_builder/bin/quattor-documentation-builder

@@ -0,0 +1,69 @@
+#!/usr/bin/env python2
+"""
+Documentation-builder generates markdown documentation.
+
+It get this from:
+ - configuration-modules-core perl documentation
+ - configuration-modules-grid perl documentation
+ - CAF perl documentation
+ - CCM perl documentation
+ - schema pan annotations
+ and creates a index for the website on http://quattor.org.
+@author: Wouter Depypere (Ghent University)
+"""
+
+import sys
+import os
+
+from vsc.utils import fancylogger
+from vsc.utils.generaloption import simple_option
+from quattordocbuild.builder import build_documentation
+
+logger = fancylogger.getLogger()
+
+REPOMAP = {
+    "configuration-modules-core": {
+        "sitesubdir": "components",
+        "targets": ["/NCM/Component/", "/components/", "/pan/quattor/"]
+        },
+    "configuration-modules-grid":{
+        "sitesubdir": "components-grid",
+        "targets": ["/NCM/Component/", "/components/"]
+        },
+    "CAF": {
+        "sitesubdir": "CAF",
+        "targets": ["/CAF/",]
+        },
+    "CCM": {
+        "sitesubdir": "CCM",
+        "targets": ["EDG/WP4/CCM/",]
+        },
+}
+
+def main(repolocation, outputlocation, maven_compile, cleanup_options):
+    """Main run of the script."""
+    build_documentation(repolocation, REPOMAP, cleanup_options, maven_compile, outputlocation)
+
+if __name__ == '__main__':
+    OPTIONS = {
+        'modules_location': ('The location of the repo checkout.', None, 'store', None, 'm'),
+        'output_location': ('The location where the output markdown files should be written to.', None, 'store', None, 'o'),
+        'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None, 'store_true', False, 'c'),
+        'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'),
+        'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'),
+        'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'),
+        'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's'),
+        'codify_paths': ('Put paths inside code tags.', None, 'store_true', True, 'p'),
+    }
+    GO = simple_option(OPTIONS)
+    logger.info("Starting main.")
+    cleanup_options = {
+        'remove_emails': GO.options.remove_emails,
+        'remove_whitespace': GO.options.remove_whitespace,
+        'remove_headers': GO.options.remove_headers,
+        'small_titles': GO.options.small_titles,
+        'codify_paths': GO.options.codify_paths,
+        }
+    main(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile, cleanup_options)
+    logger.info("Done.")
+

src/documentation_builder/lib/__init__.py

@@ -0,0 +1 @@
+"""Empty __init__.py."""

src/documentation_builder/lib/quattordocbuild/__init__.py

@@ -0,0 +1 @@
+"""Empty __init.py__."""

src/documentation_builder/lib/quattordocbuild/builder.py

@@ -0,0 +1,153 @@
+"""Build documentation from quattor sources."""
+
+import os
+import sys
+import jinja2
+from sourcehandler import get_source_files
+from markdownhandler import generate_markdown, cleanup_content
+from vsc.utils import fancylogger
+
+logger = fancylogger.getLogger()
+
+
+def check_repository_map(repository_map):
+    """Check if a repository mapping is valid."""
+    logger.info("Checking repository map.")
+    if repository_map is None:
+        logger.error("Repository map is empty.")
+        return False
+    if len(repository_map) == 0:
+        logger.error("Repository map is empty.")
+        return False
+    for repository in repository_map.keys():
+        keys = repository_map[repository].keys()
+        for opt in ['targets', 'sitesubdir']:
+            if opt not in keys:
+                logger.error("Respository %s does not have a '%s' in repository_map." % (repository, opt))
+                return False
+        if type(repository_map[repository]['targets']) is not list:
+            logger.error("Repository %s targets is not a list." % repository)
+            return False
+    return True
+
+
+def build_documentation(repository_location, repository_map, cleanup_options, compile, output_location):
+    """Build the whole documentation from quattor repositories."""
+    if not check_repository_map(repository_map):
+        sys.exit(1)
+    if not check_input(repository_map, repository_location, output_location):
+        sys.exit(1)
+    if not check_commands(compile):
+        sys.exit(1)
+
+    markdownlist = {}
+
+    for repository in repository_map.keys():
+        logger.info("Building documentation for %s." % repository)
+        fullpath = os.path.join(repository_location, repository)
+        logger.info("Path: %s." % fullpath)
+        sources = get_source_files(fullpath, compile)
+        logger.debug("Sources:" % sources)
+        markdown = generate_markdown(sources)
+        cleanup_content(markdown, cleanup_options)
+        markdownlist[repository] = markdown
+
+    site_pages = build_site_structure(markdownlist, repository_map)
+    write_site(site_pages, output_location, "docs")
+    return True
+
+
+def which(command):
+    """Check if given command is available for the current user on this system."""
+    found = False
+    for direct in os.getenv("PATH").split(':'):
+        if os.path.exists(os.path.join(direct, command)):
+            found = True
+
+    return found
+
+
+def check_input(repository_map, sourceloc, outputloc):
+    """Check input and locations."""
+    logger.info("Checking if the given paths exist.")
+    if not sourceloc:
+        logger.error("Repo location not specified.")
+        return False
+    if not outputloc:
+        logger.error("output location not specified")
+        return False
+    if not os.path.exists(sourceloc):
+        logger.error("Repo location %s does not exist" % sourceloc)
+        return False
+    for repo in repository_map.keys():
+        if not os.path.exists(os.path.join(sourceloc, repo)):
+            logger.error("Repo location %s does not exist" % os.path.join(sourceloc, repo))
+            return False
+    if not os.path.exists(outputloc):
+        logger.error("Output location %s does not exist" % outputloc)
+        return False
+    if not os.listdir(outputloc) == []:
+        logger.error("Output location %s is not empty." % outputloc)
+        return False
+    return True
+
+
+def check_commands(runmaven):
+    """Check required binaries."""
+    if runmaven:
+        if not which("mvn"):
+            logger.error("The command mvn is not available on this system, please install maven.")
+            return False
+    if not which("pod2markdown"):
+        logger.error("The command pod2markdown is not available on this system, please install pod2markdown.")
+        return False
+    return True
+
+
+def build_site_structure(markdownlist, repository_map):
+    """Make a mapping of files with their new names for the website."""
+    sitepages = {}
+    for repo, markdowns in markdownlist.iteritems():
+        sitesubdir = repository_map[repo]['sitesubdir']
+
+        sitepages[sitesubdir] = {}
+
+        targets = repository_map[repo]['targets']
+        for source, markdown in markdowns.iteritems():
+            found = False
+            for target in targets:
+                if target in source and not found:
+                    newname = source.split(target)[-1]
+                    newname = os.path.splitext(newname)[0].replace("/", "::").lower() + ".md"
+                    sitepages[sitesubdir][newname] = markdown
+                    found = True
+            if not found:
+                logger.error("No suitable target found for %s in %s." % (source, targets))
+    return sitepages
+
+
+def write_site(sitepages, location, docsdir):
+    """Write the pages for the website to disk and build a toc."""
+    toc = {}
+    for subdir, pages in sitepages.iteritems():
+        toc[subdir] = set()
+        fullsubdir = os.path.join(location, docsdir, subdir)
+        if not os.path.exists(fullsubdir):
+            os.makedirs(fullsubdir)
+        for pagename, content in pages.iteritems():
+            with open(os.path.join(fullsubdir, pagename), 'w') as fih:
+                fih.write(content)
+
+            toc[subdir].add(pagename)
+
+    write_toc(toc, location)
+
+
+def write_toc(toc, location):
+    """Write the toc to disk."""
+    loader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
+    jenv = jinja2.Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)
+    template = jenv.get_template('toc.j2')
+    tocfile = template.render(toc=toc)
+    with open(os.path.join(location, "mkdocs.yml"), 'w') as fih:
+        fih.write(tocfile)