rewrite/cleanup for documentation builder

src/documentation_builder/.coveragerc

@@ -0,0 +1,10 @@
+[run]
+omit =
+    # omit anything in a .local directory anywhere
+    */.local/*
+    # omit everything in /usr
+    /usr/*
+    # omit the test directory
+    */test/*
+    # omit everything in /tmp
+    /tmp/*

src/documentation_builder/.gitignore

@@ -0,0 +1,26 @@
+## aptana/eclipse
+*.project
+*.pydevproject
+## python
+*.pyc
+*.pyo
+.eggs
+## emacs
+*~
+.#*
+*.orig
+*.bak
+## Various build products
+*.rpm
+*.o
+*.old
+*.DS_Store
+*.egg
+*.egg-info
+.coverage
+htmlcov/
+## Python __init__.py files for distributed packages
+MANIFEST
+dist/
+build/
+setup.cfg

src/documentation_builder/.prospector.yaml

@@ -0,0 +1,17 @@
+strictness: veryhigh
+test-warnings: true
+doc-warnings: true
+member-warnings: true
+
+ignore-paths:
+  - build
+  - htmlcov
+
+pep8:
+    disable:
+    options:
+        max-line-length: 120
+
+pep257:
+    disable:
+     - D203

src/documentation_builder/MANIFEST.in

@@ -0,0 +1,3 @@
+include lib/quattordocbuild/tt/pan.tt
+include lib/quattordocbuild/tt/toc.tt
+include bin/build-quattor-documentation.sh

src/documentation_builder/README.rst

@@ -0,0 +1,54 @@
+Documentation-builder
+---------------------
+
+Documentation builder for the Quattor repositories.
+
+::
+ $ quattor-documentation-builder --help
+ Usage: quattor-documentation-builder [options]
+
+
+  Documentation-builder generates markdown documentation.
+
+  It get this from:
+   - configuration-modules-core perl documentation
+   - configuration-modules-grid perl documentation
+   - CAF perl documentation
+   - CCM perl documentation
+   - schema pan annotations
+   and creates a index for the website on http://quattor.org.
+  @author: Wouter Depypere (Ghent University)
+
+ Options:
+   -h, --shorthelp       show short help message and exit
+   -H OUTPUT_FORMAT, --help=OUTPUT_FORMAT
+                         show full help message and exit
+   --confighelp          show help as annotated configfile
+
+  Main options (configfile section MAIN):
+    -p, --codify_paths  Put paths inside code tags. (def True)
+    -i INDEX_NAME, --index_name=INDEX_NAME
+                        Filename for the index/toc for the components. (def mkdocs.yml)
+    -c, --maven_compile
+                        Execute a maven clean and maven compile before generating the documentation. (def False)
+    -m MODULES_LOCATION, --modules_location=MODULES_LOCATION
+                        The location of the repo checkout.
+    -o OUTPUT_LOCATION, --output_location=OUTPUT_LOCATION
+                        The location where the output markdown files should be written to.
+    -r, --remove_emails
+                        Remove email addresses from generated md files. (def True)
+    -R, --remove_headers
+                        Remove unneeded headers from files (MAINTAINER and AUTHOR). (def True)
+    -w, --remove_whitespace
+                        Remove whitespace (   ) from md files. (def True)
+    -s, --small_titles  Decrease the title size in the md files. (def True)
+
+  Debug and logging options (configfile section MAIN):
+    -d, --debug         Enable debug log mode (def False)
+    --info              Enable info log mode (def False)
+    --quiet             Enable quiet/warning log mode (def False)
+
+
+
+It makes some assumpions on several repositories being in place.
+To help set this up a helper script was added **build-quattor-documentation.sh** which builds the whole documentation from latest master.

src/documentation_builder/bin/build-quattor-documentation.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+VERSION="master"
+
+#!/bin/bash
+while getopts "h:v:" arg; do
+  case $arg in
+    h)
+      echo "usage: use '-v' to specify another branch than master."
+      ;;
+    s)
+      VERSION=$OPTARG
+      ;;
+  esac
+done
+
+for com in 'mkdocs' 'mvn' 'pod2markdown' 'bundle'; do
+    command -v $com >/dev/null 2>&1 || { echo >&2 "I require $com but it's not installed.  Aborting."; exit 1; }
+done
+
+tmpdir="/tmp/quattor-documentation"
+
+# Create temporary working directory
+mkdir -p $tmpdir/{src,output}
+cd $tmpdir/src
+
+# Clone required github repositories
+for REPO in CAF configuration-modules-core configuration-modules-grid CCM ;  do
+    git clone https://github.com/quattor/$REPO.git
+    cd $REPO
+    tag=`git tag -l | grep "$VERSION$"`
+    git checkout -q $tag
+    cd ..
+done
+cd ..
+
+# Build the whole documentation
+quattor-documentation-builder -c -m $tmpdir/src/ -o $tmpdir/output/ --info || { echo 'Something went wrong building documentation.' ; exit 1 ; }
+
+# get required index which is not generated
+curl https://raw.githubusercontent.com/quattor/documentation/master/docs/index.md -o $tmpdir/output/docs/index.md
+
+cd $tmpdir/output
+mkdocs build --clean
+
+# Get some tests up
+curl https://raw.githubusercontent.com/quattor/documentation/master/Gemfile -o Gemfile
+bundle
+
+bundle exec htmlproofer  --check-html ./site/ --file-ignore ./site/base.html,./site/breadcrumbs.html,./site/footer.html,./site/toc.html,./site/versions.html || { echo 'build test errors detected. stopping.' ; exit 1 ; }

src/documentation_builder/bin/quattor-documentation-builder

@@ -0,0 +1,69 @@
+#!/usr/bin/env python2
+"""
+Documentation-builder generates markdown documentation.
+
+It get this from:
+ - configuration-modules-core perl documentation
+ - configuration-modules-grid perl documentation
+ - CAF perl documentation
+ - CCM perl documentation
+ - schema pan annotations
+ and creates a index for the website on http://quattor.org.
+@author: Wouter Depypere (Ghent University)
+"""
+
+import sys
+import os
+
+from vsc.utils import fancylogger
+from vsc.utils.generaloption import simple_option
+from quattordocbuild.builder import build_documentation
+
+logger = fancylogger.getLogger()
+
+REPOMAP = {
+    "configuration-modules-core": {
+        "sitesubdir": "components",
+        "targets": ["/NCM/Component/", "/components/", "/pan/quattor/"]
+        },
+    "configuration-modules-grid":{
+        "sitesubdir": "components-grid",
+        "targets": ["/NCM/Component/", "/components/"]
+        },
+    "CAF": {
+        "sitesubdir": "CAF",
+        "targets": ["/CAF/",]
+        },
+    "CCM": {
+        "sitesubdir": "CCM",
+        "targets": ["EDG/WP4/CCM/",]
+        },
+}
+
+def main(repolocation, outputlocation, maven_compile, cleanup_options):
+    """Main run of the script."""
+    build_documentation(repolocation, REPOMAP, cleanup_options, maven_compile, outputlocation)
+
+if __name__ == '__main__':
+    OPTIONS = {
+        'modules_location': ('The location of the repo checkout.', None, 'store', None, 'm'),
+        'output_location': ('The location where the output markdown files should be written to.', None, 'store', None, 'o'),
+        'maven_compile': ('Execute a maven clean and maven compile before generating the documentation.', None, 'store_true', False, 'c'),
+        'remove_emails': ('Remove email addresses from generated md files.', None, 'store_true', True, 'r'),
+        'remove_whitespace': ('Remove whitespace (\n\n\n) from md files.', None, 'store_true', True, 'w'),
+        'remove_headers': ('Remove unneeded headers from files (MAINTAINER and AUTHOR).', None, 'store_true', True, 'R'),
+        'small_titles': ('Decrease the title size in the md files.', None, 'store_true', True, 's'),
+        'codify_paths': ('Put paths inside code tags.', None, 'store_true', True, 'p'),
+    }
+    GO = simple_option(OPTIONS)
+    logger.info("Starting main.")
+    cleanup_options = {
+        'remove_emails': GO.options.remove_emails,
+        'remove_whitespace': GO.options.remove_whitespace,
+        'remove_headers': GO.options.remove_headers,
+        'small_titles': GO.options.small_titles,
+        'codify_paths': GO.options.codify_paths,
+        }
+    main(GO.options.modules_location, GO.options.output_location, GO.options.maven_compile, cleanup_options)
+    logger.info("Done.")
+

src/documentation_builder/lib/__init__.py

@@ -0,0 +1 @@
+"""Empty __init__.py."""

src/documentation_builder/lib/quattordocbuild/__init__.py

@@ -0,0 +1 @@
+"""Empty __init.py__."""