From 00c8fd2b6bb47da6c3dbb10a90d860eccf4ac917 Mon Sep 17 00:00:00 2001 From: fritz-astronomer <80706212+fritz-astronomer@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:32:08 -0500 Subject: [PATCH 1/3] feat: documentation command --- .gitignore | 1 + orbiter/__main__.py | 96 +++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 8 +++- 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 94e914b..6971198 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ workflow *.pyz orbiter-* .python-version +translation_ruleset.html diff --git a/orbiter/__main__.py b/orbiter/__main__.py index e4a6108..be8f131 100644 --- a/orbiter/__main__.py +++ b/orbiter/__main__.py @@ -2,6 +2,7 @@ import logging import os +import re import subprocess import sys from pathlib import Path @@ -399,5 +400,100 @@ def list_rulesets(): ) +@orbiter.command( + help="Write Translation module documentation as HTML. Translations must already be installed with `orbiter install`" +) +@click.option( + *RULESET_ARGS, + multiple=True, + required=True, + prompt="Translation module to document? (e.g. `orbiter_translation.oozie.xml_demo`)", + help="Translation module to document (e.g `orbiter_translation.oozie.xml_demo`), can be supplied multiple times", +) +@click.option( + "--output-file", + "-o", + type=click.File(mode="w", atomic=True, lazy=True), + default="translation_ruleset.html", + help="HTML file to write to. Defaults to `translation_ruleset.html`. Use `-` to write to stdout", + show_default=True, +) +def document(ruleset: list[str], output_file): + from mkdocs.commands import build + from mkdocs.config.defaults import MkDocsConfig + import tempfile + import htmlark + from io import StringIO + + index_md = """# Documentation\n""" + for rs in ruleset: + # remove .translation_ruleset, if it's the end of the input + rs = re.sub(r"\.translation_ruleset$", "", rs) + index_md += f"::: {rs}\n" + + with tempfile.TemporaryDirectory() as tmpdir_read, tempfile.TemporaryDirectory() as tmpdir_write: + logger.debug(f"Writing index.md to {tmpdir_read}") + (Path(tmpdir_read) / "index.md").write_text(index_md) + + with StringIO(f""" +site_name: Translation Documentation +docs_dir: {tmpdir_read} +site_dir: {tmpdir_write} +theme: + name: material + palette: + scheme: slate + features: + - toc.integrate + - content.code.copy +markdown_extensions: + - pymdownx.magiclink + - pymdownx.superfences: + - pymdownx.saneheaders + - pymdownx.highlight: + use_pygments: true + anchor_linenums: true + - pymdownx.inlinehilite + - admonition + - pymdownx.details +plugins: +- mkdocstrings: + handlers: + python: + options: + docstring_style: sphinx + show_root_heading: true + separate_signature: true + show_signature_annotations: true + signature_crossrefs: true + unwrap_annotated: true + show_object_full_path: true + show_symbol_type_toc: true + show_symbol_type_heading: true + show_if_no_docstring: true + merge_init_into_class: true + summary: true + group_by_category: true + show_bases: false +""") as f: + # Copypaste from mkdocs.commands.build + logger.info("Building documentation...") + logger.debug("Injecting mkdocs config directly...") + cfg = MkDocsConfig(config_file_path=None) + cfg.load_file(f) + cfg.load_dict({}) + cfg.validate() + cfg.plugins.on_startup(command="build", dirty=False) + logger.debug(f"Building mkdocs to {tmpdir_write}/ ...") + try: + build.build(cfg, dirty=False) + finally: + cfg.plugins.on_shutdown() + + # Use htmlark to convert the page to a single HTML file with css/js/etc included inline + logger.info(f"Writing documentation to {output_file.name}...") + output_file.write(htmlark.convert_page(f"{tmpdir_write}/index.html", ignore_errors=True)) + + if __name__ == "__main__": orbiter() diff --git a/pyproject.toml b/pyproject.toml index c26102b..5c2e155 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ description = "Astronomer Orbiter can take your legacy workloads and land them s readme = "README.md" authors = [ { name = "Fritz Davenport", email = "fritz@astronomer.io" }, - { name = "CETA Team", email = "ceta@astronomer.io" }, { name = "Astronomer", email = "humans@astronomer.io" }, ] license = { file = "LICENSE" } @@ -79,6 +78,13 @@ dependencies = [ # backport "eval_type_backport; python_version < '3.10'", + + # for 'document' command + "mkdocs", + "mkdocs-material", + "pygments", + "mkdocstrings-python", + "htmlark" ] [project.optional-dependencies] From 194e3e8dc4fd5cb751b7b5f7737f2ab578e9b252 Mon Sep 17 00:00:00 2001 From: fritz-astronomer <80706212+fritz-astronomer@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:32:30 -0500 Subject: [PATCH 2/3] feat: add documentation to docker test --- justfile | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index 846e322..4a9c5a3 100644 --- a/justfile +++ b/justfile @@ -95,15 +95,20 @@ docker-build-binary: just build-binary EOF -docker-run-binary REPO='orbiter-community-translations' RULESET='orbiter_translations.oozie.xml_demo.translation_ruleset': +docker-run-binary REPO='orbiter-community-translations' DEMO="https://raw.githubusercontent.com/astronomer/orbiter-community-translations/refs/heads/main/tests/control_m/demo/workflow/demo.xml" RULESET='orbiter_translations.control_m.xml_demo.translation_ruleset' PLATFORM="linux/amd64": #!/usr/bin/env bash set -euxo pipefail - cat <<"EOF" | docker run --platform linux/amd64 -v `pwd`:/data -w /data -i ubuntu /bin/bash + cat <<"EOF" | docker run --platform {{PLATFORM}} -v `pwd`:/data -w /data -i ubuntu /bin/bash + echo "[SETUP]" && \ echo "setting up certificates for https" && \ apt update && apt install -y ca-certificates && update-ca-certificates --fresh && \ echo "sourcing .env" && \ set -a && source .env && set +a && \ chmod +x ./orbiter-linux-x86_64 && \ + echo "downloading {{DEMO}}" && \ + mkdir -p workflow && pushd workflow && \ + curl -ssLO {{DEMO}} && popd && \ + echo "[INSTALL]" && \ echo "[ORBITER LIST-RULESETS]" && \ ./orbiter-linux-x86_64 list-rulesets && \ mkdir -p workflow && \ @@ -111,14 +116,25 @@ docker-run-binary REPO='orbiter-community-translations' RULESET='orbiter_transla LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 install --repo={{REPO}} && \ echo "[ORBITER TRANSLATE]" && \ LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 translate workflow/ output/ --ruleset {{RULESET}} + echo "[ORBITER DOCUMENT]" && \ + LOG_LEVEL=DEBUG orbiter document --ruleset {{RULESET}} && \ + head translation_ruleset.html EOF -docker-run-python REPO='orbiter-community-translations' RULESET='orbiter_translations.oozie.xml_demo.translation_ruleset': +docker-run-python REPO='orbiter-community-translations' DEMO="https://raw.githubusercontent.com/astronomer/orbiter-community-translations/refs/heads/main/tests/control_m/demo/workflow/demo.xml" RULESET='orbiter_translations.control_m.xml_demo.translation_ruleset' PLATFORM="linux/amd64": #!/usr/bin/env bash set -euxo pipefail - cat <<"EOF" | docker run --platform linux/amd64 -v `pwd`:/data -w /data -i python /bin/bash + cat <<"EOF" | docker run --platform {{PLATFORM}} -v `pwd`:/data -w /data -i python /bin/bash + echo "[SETUP]" && \ echo "sourcing .env" && \ set -a && source .env && set +a && \ + echo "installing cargo (pendulum needs rust)" && \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + export PATH="$HOME/.cargo/bin:$PATH" && \ + echo "downloading {{DEMO}}" && \ + mkdir -p workflow && pushd workflow && \ + curl -ssLO {{DEMO}} && popd && \ + echo "[INSTALL]" && \ echo "installing orbiter" && \ pip install '.' echo "[ORBITER LIST-RULESETS]" && \ @@ -127,5 +143,8 @@ docker-run-python REPO='orbiter-community-translations' RULESET='orbiter_transla echo "[ORBITER INSTALL]" && \ LOG_LEVEL=DEBUG orbiter install --repo={{REPO}} && \ echo "[ORBITER TRANSLATE]" && \ - LOG_LEVEL=DEBUG orbiter translate workflow/ output/ --ruleset {{RULESET}} + LOG_LEVEL=DEBUG orbiter translate workflow/ output/ --ruleset {{RULESET}} && \ + echo "[ORBITER DOCUMENT]" && \ + LOG_LEVEL=DEBUG orbiter document --ruleset {{RULESET}} && \ + head translation_ruleset.html EOF From 4fec3de75f2cf92a1a0efa80904434892637674f Mon Sep 17 00:00:00 2001 From: fritz-astronomer <80706212+fritz-astronomer@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:54:54 -0500 Subject: [PATCH 3/3] fix: small fix for docker-run-binary test --- justfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 4a9c5a3..dcbd2a5 100644 --- a/justfile +++ b/justfile @@ -101,7 +101,7 @@ docker-run-binary REPO='orbiter-community-translations' DEMO="https://raw.githu cat <<"EOF" | docker run --platform {{PLATFORM}} -v `pwd`:/data -w /data -i ubuntu /bin/bash echo "[SETUP]" && \ echo "setting up certificates for https" && \ - apt update && apt install -y ca-certificates && update-ca-certificates --fresh && \ + apt update && apt install -y curl ca-certificates && update-ca-certificates --fresh && \ echo "sourcing .env" && \ set -a && source .env && set +a && \ chmod +x ./orbiter-linux-x86_64 && \ @@ -115,9 +115,9 @@ docker-run-binary REPO='orbiter-community-translations' DEMO="https://raw.githu echo "[ORBITER INSTALL]" && \ LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 install --repo={{REPO}} && \ echo "[ORBITER TRANSLATE]" && \ - LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 translate workflow/ output/ --ruleset {{RULESET}} + LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 translate workflow/ output/ --ruleset {{RULESET}} --no-format && \ echo "[ORBITER DOCUMENT]" && \ - LOG_LEVEL=DEBUG orbiter document --ruleset {{RULESET}} && \ + LOG_LEVEL=DEBUG ./orbiter-linux-x86_64 document --ruleset {{RULESET}} && \ head translation_ruleset.html EOF