Merge pull request #7 from truenas/template-validation

Add logic to render docker compose files

catalog_reader/app_utils.py

@@ -1,3 +1,8 @@
+import contextlib
+import os
+import yaml
+
+
 def get_app_details_base(retrieve_complete_item_keys: bool = True) -> dict:
     return {
         'app_readme': None,
@@ -35,3 +40,21 @@ def get_default_questions_context() -> dict:
         'system.general.config': {'timezone': 'America/Los_Angeles'},
         'unused_ports': [i for i in range(1025, 65535)],
     }
+
+
+def get_app_basic_details(app_path: str) -> dict:
+    # This just retrieves app name and app version from app path
+    with contextlib.suppress(FileNotFoundError, yaml.YAMLError, KeyError):
+        with open(os.path.join(app_path, 'app.yaml'), 'r') as f:
+            app_config = yaml.safe_load(f.read())
+        return {k: app_config[k] for k in ('name', 'train', 'version', 'lib_version')}
+
+    return {}
+
+
+def get_values(values_path: str) -> dict:
+    with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
+        with open(values_path, 'r') as f:
+            return yaml.safe_load(f.read())
+
+    return {}

catalog_templating/render.py

@@ -0,0 +1,91 @@
+import collections
+import importlib
+import os
+import pathlib
+import shutil
+
+from jinja2 import Environment, FileSystemLoader
+
+from apps_validation.exceptions import ValidationError
+from catalog_reader.app_utils import get_app_basic_details
+
+
+def render_templates(app_version_path: str, test_values: dict) -> dict:
+    app_details = get_app_basic_details(app_version_path)
+    if not app_details:
+        raise ValidationError('app_version_path', 'Unable to retrieve app metadata from specified app version path')
+
+    template_path = os.path.join(app_version_path, 'templates')
+    if not pathlib.Path(os.path.join(template_path, 'library')).is_dir():
+        return {}
+
+    template_libs = import_library(os.path.join(template_path, 'library'), app_details)
+    file_loader = FileSystemLoader(template_path)
+    env = Environment(loader=file_loader)
+    rendered_templates = {}
+    for to_render_file in filter(
+        lambda f: f.is_file() and f.name.endswith('.yaml'), pathlib.Path(template_path).iterdir()
+    ):
+        # TODO: Let's look to adding dynamic filter support in the future
+        # env.filters['make_capital'] = lambda st: st.upper()
+        rendered_templates[to_render_file.name] = env.get_template(
+            to_render_file.name
+        ).render(test_values | {'ix_lib': template_libs})
+
+    return rendered_templates
+
+
+def import_library(library_path: str, app_config) -> dict:
+    modules_context = collections.defaultdict(dict)
+    # 2 dirs which we want to import from
+    global_base_lib = os.path.join(library_path, f'base_v{app_config["lib_version"].replace(".", "_")}')
+    app_lib = os.path.join(
+        library_path, app_config['train'], app_config['name'], f'v{app_config["version"].replace(".", "_")}'
+    )
+    if pathlib.Path(global_base_lib).is_dir():
+        modules_context['base'] = import_app_modules(global_base_lib, os.path.basename(global_base_lib))  # base_v1_0_0
+    if pathlib.Path(app_lib).is_dir():
+        modules_context[app_config['train']] = {
+            app_config['name']: import_app_modules(app_lib, os.path.basename(app_lib))  # v1_0_1
+        }
+
+    return modules_context
+
+
+def import_app_modules(modules_path: str, parent_module_name) -> dict:
+    def import_module_context(module_name, file_path):
+        try:
+            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+        except Exception as e:
+            raise Exception(
+                f'Unable to import module {module_name!r} from {file_path!r}: {e!r}.\n\n'
+                'This could be due to various reasons with primary being:\n1) The module is not a valid python module '
+                'which can be imported i.e might have syntax errors\n2) The module has already been imported and '
+                'then has been changed but the version for the module has not been bumped.'
+            )
+        return module
+
+    sub_modules_context = {}
+    try:
+        importlib.sys.path.append(os.path.dirname(modules_path))
+        for sub_modules_file in filter(
+            lambda p: os.path.isfile(os.path.join(modules_path, p)) and p.endswith('.py'), os.listdir(modules_path)
+        ):
+            sub_modules = sub_modules_file.removesuffix('.py')
+            sub_modules_context[sub_modules] = import_module_context(
+                f'{parent_module_name}.{sub_modules}', os.path.join(modules_path, sub_modules_file)
+            )
+    finally:
+        importlib.sys.path.remove(os.path.dirname(modules_path))
+        remove_pycache(modules_path)
+
+    return sub_modules_context
+
+
+def remove_pycache(library_path: str):
+    for modules in filter(
+        lambda p: os.path.exists(os.path.join(library_path, p, '__pycache__')), os.listdir(library_path)
+    ):
+        shutil.rmtree(os.path.join(library_path, modules, '__pycache__'))

catalog_templating/scripts/render_compose.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+import argparse
+import os
+import shutil
+
+from apps_validation.exceptions import ValidationErrors
+from catalog_reader.app_utils import get_values
+from catalog_templating.render import render_templates
+
+
+def render_templates_from_path(app_path: str, values_file: str) -> None:
+    verrors = ValidationErrors()
+    for k, v in (('app_path', app_path), ('values_file', values_file)):
+        if not os.path.exists(v):
+            verrors.add(k, f'{v!r} {k} does not exist')
+
+    verrors.check()
+
+    rendered_data = render_templates(app_path, get_values(values_file))
+    write_template_yaml(app_path, rendered_data)
+
+
+def write_template_yaml(app_path: str, rendered_templates: dict) -> None:
+    rendered_templates_path = os.path.join(app_path, 'templates', 'rendered')
+    shutil.rmtree(rendered_templates_path, ignore_errors=True)
+    os.makedirs(rendered_templates_path)
+
+    for file_name, rendered_template in rendered_templates.items():
+        with open(os.path.join(rendered_templates_path, file_name), 'w') as f:
+            f.write(rendered_template)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(help='sub-command help', dest='action')
+
+    parser_setup = subparsers.add_parser(
+        'render', help='Render TrueNAS catalog app\'s docker compose files'
+    )
+    parser_setup.add_argument('--path', help='Specify path of TrueNAS app version', required=True)
+    parser_setup.add_argument('--values', help='Specify values to be used for rendering the app version', required=True)
+
+    args = parser.parse_args()
+    if args.action == 'render':
+        render_templates_from_path(args.path, args.values)
+    else:
+        parser.print_help()
+
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -1,4 +1,5 @@
 gitpython
+jinja2
 jsonschema==4.10.3
 markdown
 pyyaml

setup.py

@@ -14,15 +14,18 @@
         'apps_validation',
         'apps_validation.*',
         'catalog_reader',
-        'catalog_reader.*'
+        'catalog_reader.*',
+        'catalog_templating',
+        'catalog_templating.*',
     ]),
     license='GNU3',
     platforms='any',
     entry_points={
         'console_scripts': [
-            'apps_catalog_update = catalog_validation.scripts.catalog_update:main',
-            'apps_catalog_validate = catalog_validation.scripts.catalog_validate:main',
+            'apps_catalog_update = apps_validation.scripts.catalog_update:main',
+            'apps_catalog_validate = apps_validation.scripts.catalog_validate:main',
             'apps_dev_charts_validate = apps_validation.scripts.dev_apps_validate:main',  # TODO: Remove apps_prefix
+            'apps_render_app = catalog_templating.scripts.render_compose:main',
         ],
     },
 )