orbiter document command (#38)

* feat: documentation command

* feat: add documentation to docker test

* fix: small fix for docker-run-binary test

.gitignore

@@ -21,3 +21,4 @@ workflow
 *.pyz
 orbiter-*
 .python-version
+translation_ruleset.html

justfile

@@ -95,30 +95,46 @@ docker-build-binary:
     just build-binary
     EOF
 
-docker-run-binary REPO='orbiter-community-translations' RULESET='orbiter_translations.oozie.xml_demo.translation_ruleset':
+docker-run-binary REPO='orbiter-community-translations'  DEMO="https://raw.githubusercontent.com/astronomer/orbiter-community-translations/refs/heads/main/tests/control_m/demo/workflow/demo.xml" RULESET='orbiter_translations.control_m.xml_demo.translation_ruleset' PLATFORM="linux/amd64":
     #!/usr/bin/env bash
     set -euxo pipefail
-    cat <<"EOF" | docker run --platform linux/amd64 -v `pwd`:/data -w /data -i ubuntu /bin/bash
+    cat <<"EOF" | docker run --platform {{PLATFORM}} -v `pwd`:/data -w /data -i ubuntu /bin/bash
+    echo "[SETUP]" && \
     echo "setting up certificates for https" && \
-    apt update && apt install -y ca-certificates && update-ca-certificates --fresh && \
+    apt update && apt install -y curl ca-certificates && update-ca-certificates --fresh && \
     echo "sourcing .env" && \
     set -a && source .env && set +a && \
     chmod +x ./orbiter-linux-x86_64 && \
+    echo "downloading {{DEMO}}" && \
+    mkdir -p workflow && pushd workflow && \
+    curl -ssLO {{DEMO}} && popd && \
+    echo "[INSTALL]" && \
     echo "[ORBITER LIST-RULESETS]" && \
     ./orbiter-linux-x86_64 list-rulesets && \
     mkdir -p workflow && \
     echo "[ORBITER INSTALL]" && \
     LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 install --repo={{REPO}} && \
     echo "[ORBITER TRANSLATE]" && \
-    LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 translate workflow/ output/ --ruleset {{RULESET}}
+    LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 translate workflow/ output/ --ruleset {{RULESET}} --no-format && \
+    echo "[ORBITER DOCUMENT]" && \
+    LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 document --ruleset {{RULESET}} && \
+    head translation_ruleset.html
     EOF
 
-docker-run-python REPO='orbiter-community-translations' RULESET='orbiter_translations.oozie.xml_demo.translation_ruleset':
+docker-run-python REPO='orbiter-community-translations' DEMO="https://raw.githubusercontent.com/astronomer/orbiter-community-translations/refs/heads/main/tests/control_m/demo/workflow/demo.xml" RULESET='orbiter_translations.control_m.xml_demo.translation_ruleset' PLATFORM="linux/amd64":
     #!/usr/bin/env bash
     set -euxo pipefail
-    cat <<"EOF" | docker run --platform linux/amd64 -v `pwd`:/data -w /data -i python /bin/bash
+    cat <<"EOF" | docker run --platform {{PLATFORM}} -v `pwd`:/data -w /data -i python /bin/bash
+    echo "[SETUP]" && \
     echo "sourcing .env" && \
     set -a && source .env && set +a && \
+    echo "installing cargo (pendulum needs rust)" && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    export PATH="$HOME/.cargo/bin:$PATH" && \
+    echo "downloading {{DEMO}}" && \
+    mkdir -p workflow && pushd workflow && \
+    curl -ssLO {{DEMO}} && popd && \
+    echo "[INSTALL]" && \
     echo "installing orbiter" && \
     pip install '.'
     echo "[ORBITER LIST-RULESETS]" && \
@@ -127,5 +143,8 @@ docker-run-python REPO='orbiter-community-translations' RULESET='orbiter_transla
     echo "[ORBITER INSTALL]" && \
     LOG_LEVEL=DEBUG orbiter install --repo={{REPO}} && \
     echo "[ORBITER TRANSLATE]" && \
-    LOG_LEVEL=DEBUG orbiter translate workflow/ output/ --ruleset {{RULESET}}
+    LOG_LEVEL=DEBUG orbiter translate workflow/ output/ --ruleset {{RULESET}} && \
+    echo "[ORBITER DOCUMENT]" && \
+    LOG_LEVEL=DEBUG orbiter document --ruleset {{RULESET}} && \
+    head translation_ruleset.html
     EOF

orbiter/__main__.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -399,5 +400,100 @@ def list_rulesets():
     )
 
 
+@orbiter.command(
+    help="Write Translation module documentation as HTML. Translations must already be installed with `orbiter install`"
+)
+@click.option(
+    *RULESET_ARGS,
+    multiple=True,
+    required=True,
+    prompt="Translation module to document? (e.g. `orbiter_translation.oozie.xml_demo`)",
+    help="Translation module to document (e.g `orbiter_translation.oozie.xml_demo`), can be supplied multiple times",
+)
+@click.option(
+    "--output-file",
+    "-o",
+    type=click.File(mode="w", atomic=True, lazy=True),
+    default="translation_ruleset.html",
+    help="HTML file to write to. Defaults to `translation_ruleset.html`. Use `-` to write to stdout",
+    show_default=True,
+)
+def document(ruleset: list[str], output_file):
+    from mkdocs.commands import build
+    from mkdocs.config.defaults import MkDocsConfig
+    import tempfile
+    import htmlark
+    from io import StringIO
+
+    index_md = """# Documentation\n"""
+    for rs in ruleset:
+        # remove .translation_ruleset, if it's the end of the input
+        rs = re.sub(r"\.translation_ruleset$", "", rs)
+        index_md += f"::: {rs}\n"
+
+    with tempfile.TemporaryDirectory() as tmpdir_read, tempfile.TemporaryDirectory() as tmpdir_write:
+        logger.debug(f"Writing index.md to {tmpdir_read}")
+        (Path(tmpdir_read) / "index.md").write_text(index_md)
+
+        with StringIO(f"""
+site_name: Translation Documentation
+docs_dir: {tmpdir_read}
+site_dir: {tmpdir_write}
+theme:
+  name: material
+  palette:
+    scheme: slate
+  features:
+    - toc.integrate
+    - content.code.copy
+markdown_extensions:
+  - pymdownx.magiclink
+  - pymdownx.superfences:
+  - pymdownx.saneheaders
+  - pymdownx.highlight:
+      use_pygments: true
+      anchor_linenums: true
+  - pymdownx.inlinehilite
+  - admonition
+  - pymdownx.details
+plugins:
+- mkdocstrings:
+    handlers:
+      python:
+        options:
+          docstring_style: sphinx
+          show_root_heading: true
+          separate_signature: true
+          show_signature_annotations: true
+          signature_crossrefs: true
+          unwrap_annotated: true
+          show_object_full_path: true
+          show_symbol_type_toc: true
+          show_symbol_type_heading: true
+          show_if_no_docstring: true
+          merge_init_into_class: true
+          summary: true
+          group_by_category: true
+          show_bases: false
+""") as f:
+            # Copypaste from mkdocs.commands.build
+            logger.info("Building documentation...")
+            logger.debug("Injecting mkdocs config directly...")
+            cfg = MkDocsConfig(config_file_path=None)
+            cfg.load_file(f)
+            cfg.load_dict({})
+            cfg.validate()
+            cfg.plugins.on_startup(command="build", dirty=False)
+            logger.debug(f"Building mkdocs to {tmpdir_write}/ ...")
+            try:
+                build.build(cfg, dirty=False)
+            finally:
+                cfg.plugins.on_shutdown()
+
+            # Use htmlark to convert the page to a single HTML file with css/js/etc included inline
+            logger.info(f"Writing documentation to {output_file.name}...")
+            output_file.write(htmlark.convert_page(f"{tmpdir_write}/index.html", ignore_errors=True))
+
+
 if __name__ == "__main__":
     orbiter()

pyproject.toml

@@ -27,7 +27,6 @@ description = "Astronomer Orbiter can take your legacy workloads and land them s
 readme = "README.md"
 authors = [
     { name = "Fritz Davenport", email = "[email protected]" },
-    { name = "CETA Team", email = "[email protected]" },
     { name = "Astronomer", email = "[email protected]" },
 ]
 license = { file = "LICENSE" }
@@ -79,6 +78,13 @@ dependencies = [
 
     # backport
     "eval_type_backport; python_version < '3.10'",
+
+    # for 'document' command
+    "mkdocs",
+    "mkdocs-material",
+    "pygments",
+    "mkdocstrings-python",
+    "htmlark"
 ]
 
 [project.optional-dependencies]