From eece6943a639913bfad7feac80d69b1a7dbcbe81 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Sun, 5 May 2024 17:11:20 +0500 Subject: [PATCH 1/8] Add initial template rendering pyhton package --- catalog_templating/__init__.py | 0 catalog_templating/render.py | 70 ++++++++++++++++++++++++++++++++++ catalog_templating/utils.py | 4 ++ requirements.txt | 1 + 4 files changed, 75 insertions(+) create mode 100644 catalog_templating/__init__.py create mode 100644 catalog_templating/render.py create mode 100644 catalog_templating/utils.py diff --git a/catalog_templating/__init__.py b/catalog_templating/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/catalog_templating/render.py b/catalog_templating/render.py new file mode 100644 index 0000000..4be9c8e --- /dev/null +++ b/catalog_templating/render.py @@ -0,0 +1,70 @@ +import collections +import importlib +import os +import shutil + +from jinja2 import Environment, FileSystemLoader + +from .utils import RE_MODULE_VERSION + + +def render_templates(template_path: str, test_values: dict, lib_version: str): + file_loader = FileSystemLoader(template_path) + env = Environment(loader=file_loader) + # This how filters can be introduced currently not sure how to handle it + env.filters['make_capital'] = lambda st: st.upper() + templates = env.get_template('docker_compose.yaml') + libs = import_library(os.path.join(template_path, 'library'), lib_version) + context = { + **test_values, + 'libs': libs + } + return templates.render(context) + + +def import_library(library_path: str, lib_version: str): + modules_context = collections.defaultdict(dict) + for train_name in os.listdir(library_path): + if not RE_MODULE_VERSION.findall(train_name): + modules_context[train_name] = collections.defaultdict(dict) + for app_name in os.listdir(os.path.join(library_path, train_name)): + modules_context[train_name][app_name] = import_app_modules( + os.path.join(library_path, train_name, app_name, lib_version), lib_version) + else: + base_name = train_name.replace(f'_{RE_MODULE_VERSION.findall(train_name)[0]}', '') + modules_context[base_name] = import_app_modules( + os.path.join(library_path, train_name), train_name) + + return modules_context + + +def import_app_modules(modules_path: str, parent_module_name): + def import_module_context(module_name, file_path): + try: + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + except Exception as e: + raise Exception(f'Some changes might be introduce in this ' + f'package {module_name.split(".")[0]} please update the package versions error: {e}') + return module + sub_modules_context = {} + try: + importlib.sys.path.append(os.path.dirname(modules_path)) + for sub_modules_file in filter(lambda p: os.path.isfile(os.path.join(modules_path, p)), + os.listdir(modules_path)): + sub_modules = sub_modules_file.removesuffix('.py') + sub_modules_context[sub_modules] = import_module_context( + f'{parent_module_name}.{sub_modules}', os.path.join(modules_path, sub_modules_file) + ) + finally: + importlib.sys.path.remove(os.path.dirname(modules_path)) + remove_pycache(modules_path) + + return sub_modules_context + + +def remove_pycache(library_path: str): + for modules in filter(lambda p: os.path.exists(os.path.join(library_path, p, '__pycache__')), + os.listdir(library_path)): + shutil.rmtree(os.path.join(library_path, modules, '__pycache__')) diff --git a/catalog_templating/utils.py b/catalog_templating/utils.py new file mode 100644 index 0000000..c358957 --- /dev/null +++ b/catalog_templating/utils.py @@ -0,0 +1,4 @@ +import re + + +RE_MODULE_VERSION = re.compile(r'\d+_\d+_\d+|\d+$') diff --git a/requirements.txt b/requirements.txt index e0d20b7..ffef472 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ gitpython +jinja2 jsonschema==4.10.3 markdown pyyaml From bbb812fcae7dc0de17edb625ace3f690d33c0472 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Sun, 5 May 2024 22:56:54 +0500 Subject: [PATCH 2/8] Refine rendering logic --- catalog_reader/app_utils.py | 14 ++++++ catalog_templating/render.py | 83 ++++++++++++++++++++++-------------- catalog_templating/utils.py | 4 -- 3 files changed, 66 insertions(+), 35 deletions(-) diff --git a/catalog_reader/app_utils.py b/catalog_reader/app_utils.py index d1dc72e..37012ff 100644 --- a/catalog_reader/app_utils.py +++ b/catalog_reader/app_utils.py @@ -1,3 +1,8 @@ +import contextlib +import os +import yaml + + def get_app_details_base(retrieve_complete_item_keys: bool = True) -> dict: return { 'app_readme': None, @@ -35,3 +40,12 @@ def get_default_questions_context() -> dict: 'system.general.config': {'timezone': 'America/Los_Angeles'}, 'unused_ports': [i for i in range(1025, 65535)], } + + +def get_app_basic_details(app_path: str) -> dict: + # This just retrieves app name and app version from app path + with contextlib.suppress(FileNotFoundError, yaml.YAMLError): + app_config = yaml.safe_load(open(os.path.join(app_path, 'app.yaml'))) + return {k: app_config[k] for k in ('name', 'train', 'version', 'lib_version')} + + return {} diff --git a/catalog_templating/render.py b/catalog_templating/render.py index 4be9c8e..5e40771 100644 --- a/catalog_templating/render.py +++ b/catalog_templating/render.py @@ -1,58 +1,78 @@ import collections import importlib import os +import pathlib import shutil from jinja2 import Environment, FileSystemLoader -from .utils import RE_MODULE_VERSION +from apps_validation.exceptions import ValidationError +from catalog_reader.app_utils import get_app_basic_details -def render_templates(template_path: str, test_values: dict, lib_version: str): +def render_templates(app_version_path: str, test_values: dict) -> dict: + app_details = get_app_basic_details(app_version_path) + if not app_details: + raise ValidationError('app_version_path', 'Unable to retrieve app metadata from specified app version path') + + template_path = os.path.join(app_version_path, 'templates') + if not pathlib.Path(os.path.join(template_path, 'library')).is_dir(): + return {} + + template_libs = import_library(os.path.join(template_path, 'library'), app_details) file_loader = FileSystemLoader(template_path) env = Environment(loader=file_loader) - # This how filters can be introduced currently not sure how to handle it - env.filters['make_capital'] = lambda st: st.upper() - templates = env.get_template('docker_compose.yaml') - libs = import_library(os.path.join(template_path, 'library'), lib_version) - context = { - **test_values, - 'libs': libs - } - return templates.render(context) - - -def import_library(library_path: str, lib_version: str): + rendered_templates = {} + for to_render_file in filter( + lambda f: f.is_file() and f.name.endswith('.yaml'), pathlib.Path(template_path).iterdir() + ): + # TODO: Let's look to adding dynamic filter support in the future + # env.filters['make_capital'] = lambda st: st.upper() + rendered_templates[to_render_file.name] = env.get_template( + to_render_file.name + ).render(test_values | template_libs) + + return rendered_templates + + +def import_library(library_path: str, app_config) -> dict: modules_context = collections.defaultdict(dict) - for train_name in os.listdir(library_path): - if not RE_MODULE_VERSION.findall(train_name): - modules_context[train_name] = collections.defaultdict(dict) - for app_name in os.listdir(os.path.join(library_path, train_name)): - modules_context[train_name][app_name] = import_app_modules( - os.path.join(library_path, train_name, app_name, lib_version), lib_version) - else: - base_name = train_name.replace(f'_{RE_MODULE_VERSION.findall(train_name)[0]}', '') - modules_context[base_name] = import_app_modules( - os.path.join(library_path, train_name), train_name) + # 2 dirs which we want to import from + global_base_lib = os.path.join(library_path, f'base_v{app_config["lib_version"].replace(".", "_")}') + app_lib = os.path.join( + library_path, app_config['train'], app_config['name'], f'v{app_config["version"].replace(".", "_")}' + ) + if pathlib.Path(global_base_lib).is_dir(): + modules_context['base'] = import_app_modules(global_base_lib, os.path.basename(global_base_lib)) # base_v1_0_0 + if pathlib.Path(app_lib).is_dir(): + modules_context[app_config['train']] = { + app_config['name']: import_app_modules(app_lib, os.path.basename(app_lib)) # v1_0_1 + } return modules_context -def import_app_modules(modules_path: str, parent_module_name): +def import_app_modules(modules_path: str, parent_module_name) -> dict: def import_module_context(module_name, file_path): try: spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) except Exception as e: - raise Exception(f'Some changes might be introduce in this ' - f'package {module_name.split(".")[0]} please update the package versions error: {e}') + raise Exception( + f'Unable to import module {module_name!r} from {file_path!r}: {e!r}.\n\n' + 'This could be due to various reasons with primary being:\n1) The module is not a valid python module ' + 'which can be imported i.e might have syntax errors\n2) The module has already been imported and ' + 'then has been changed but the version for the module has not been bumped.' + ) return module + sub_modules_context = {} try: importlib.sys.path.append(os.path.dirname(modules_path)) - for sub_modules_file in filter(lambda p: os.path.isfile(os.path.join(modules_path, p)), - os.listdir(modules_path)): + for sub_modules_file in filter( + lambda p: os.path.isfile(os.path.join(modules_path, p)) and p.endswith('.py'), os.listdir(modules_path) + ): sub_modules = sub_modules_file.removesuffix('.py') sub_modules_context[sub_modules] = import_module_context( f'{parent_module_name}.{sub_modules}', os.path.join(modules_path, sub_modules_file) @@ -65,6 +85,7 @@ def import_module_context(module_name, file_path): def remove_pycache(library_path: str): - for modules in filter(lambda p: os.path.exists(os.path.join(library_path, p, '__pycache__')), - os.listdir(library_path)): + for modules in filter( + lambda p: os.path.exists(os.path.join(library_path, p, '__pycache__')), os.listdir(library_path) + ): shutil.rmtree(os.path.join(library_path, modules, '__pycache__')) diff --git a/catalog_templating/utils.py b/catalog_templating/utils.py index c358957..e69de29 100644 --- a/catalog_templating/utils.py +++ b/catalog_templating/utils.py @@ -1,4 +0,0 @@ -import re - - -RE_MODULE_VERSION = re.compile(r'\d+_\d+_\d+|\d+$') From 4b278820ab9e36ce3178a441f6672cf484dbbbde Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Mon, 6 May 2024 01:49:57 +0500 Subject: [PATCH 3/8] Add catalog_templating to setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 5b6faba..3227dd3 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,8 @@ 'apps_validation.*', 'catalog_reader', 'catalog_reader.*' + 'catalog_templating', + 'catalog_templating.*', ]), license='GNU3', platforms='any', From ee4a71ce81f6554ecfc6ea62bd48ffa89c1647c8 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Mon, 6 May 2024 01:52:08 +0500 Subject: [PATCH 4/8] Fix rendering of compose file --- catalog_templating/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog_templating/render.py b/catalog_templating/render.py index 5e40771..5417b29 100644 --- a/catalog_templating/render.py +++ b/catalog_templating/render.py @@ -30,7 +30,7 @@ def render_templates(app_version_path: str, test_values: dict) -> dict: # env.filters['make_capital'] = lambda st: st.upper() rendered_templates[to_render_file.name] = env.get_template( to_render_file.name - ).render(test_values | template_libs) + ).render(test_values | {'ix_lib': template_libs}) return rendered_templates From 2d8558314c11a59be5d2a3578ae4f201f4adf09f Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Mon, 6 May 2024 01:57:45 +0500 Subject: [PATCH 5/8] Add a util to read provided values file --- catalog_reader/app_utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/catalog_reader/app_utils.py b/catalog_reader/app_utils.py index 37012ff..be0cfb2 100644 --- a/catalog_reader/app_utils.py +++ b/catalog_reader/app_utils.py @@ -44,8 +44,17 @@ def get_default_questions_context() -> dict: def get_app_basic_details(app_path: str) -> dict: # This just retrieves app name and app version from app path - with contextlib.suppress(FileNotFoundError, yaml.YAMLError): - app_config = yaml.safe_load(open(os.path.join(app_path, 'app.yaml'))) + with contextlib.suppress(FileNotFoundError, yaml.YAMLError, KeyError): + with open(os.path.join(app_path, 'app.yaml'), 'r') as f: + app_config = yaml.safe_load(f.read()) return {k: app_config[k] for k in ('name', 'train', 'version', 'lib_version')} return {} + + +def get_values(values_path: str) -> dict: + with contextlib.suppress(FileNotFoundError, yaml.YAMLError): + with open(values_path, 'r') as f: + return yaml.safe_load(f.read()) + + return {} From 8c3347619ce087922c94b38df41d3cbd6f176c1b Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Mon, 6 May 2024 02:14:51 +0500 Subject: [PATCH 6/8] Add a script to render compose files --- .../{utils.py => scripts/__init__.py} | 0 catalog_templating/scripts/render_compose.py | 43 +++++++++++++++++++ setup.py | 1 + 3 files changed, 44 insertions(+) rename catalog_templating/{utils.py => scripts/__init__.py} (100%) create mode 100644 catalog_templating/scripts/render_compose.py diff --git a/catalog_templating/utils.py b/catalog_templating/scripts/__init__.py similarity index 100% rename from catalog_templating/utils.py rename to catalog_templating/scripts/__init__.py diff --git a/catalog_templating/scripts/render_compose.py b/catalog_templating/scripts/render_compose.py new file mode 100644 index 0000000..aadcbbb --- /dev/null +++ b/catalog_templating/scripts/render_compose.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +import argparse +import os +import shutil + +from catalog_reader.app_utils import get_values +from catalog_templating.render import render_templates + + +def render_templates_from_path(app_path: str, values_file: str) -> None: + rendered_data = render_templates(app_path, get_values(values_file)) + write_template_yaml(app_path, rendered_data) + + +def write_template_yaml(app_path: str, rendered_templates: dict) -> None: + rendered_templates_path = os.path.join(app_path, 'templates', 'rendered') + shutil.rmtree(rendered_templates_path, ignore_errors=True) + os.makedirs(rendered_templates_path) + + for file_name, rendered_template in rendered_templates.items(): + with open(os.path.join(rendered_templates_path, file_name), 'w') as f: + f.write(rendered_template) + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(help='sub-command help', dest='action') + + parser_setup = subparsers.add_parser( + 'render', help='Render TrueNAS catalog app\'s docker compose files' + ) + parser_setup.add_argument('--path', help='Specify path of TrueNAS app version', required=True) + parser_setup.add_argument('--values', help='Specify values to be used for rendering the app version', required=True) + + args = parser.parse_args() + if args.action == 'render': + render_templates_from_path(args.path, args.values) + else: + parser.print_help() + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 3227dd3..35a70ea 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ 'apps_catalog_update = catalog_validation.scripts.catalog_update:main', 'apps_catalog_validate = catalog_validation.scripts.catalog_validate:main', 'apps_dev_charts_validate = apps_validation.scripts.dev_apps_validate:main', # TODO: Remove apps_prefix + 'apps_render_app = apps_validation.scripts.render_compose:main', ], }, ) From 22853dba296c9510c3213d3283aa09048fb87855 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Mon, 6 May 2024 02:23:52 +0500 Subject: [PATCH 7/8] Fix setup.py --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 35a70ea..17cbf16 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ 'apps_validation', 'apps_validation.*', 'catalog_reader', - 'catalog_reader.*' + 'catalog_reader.*', 'catalog_templating', 'catalog_templating.*', ]), @@ -22,10 +22,10 @@ platforms='any', entry_points={ 'console_scripts': [ - 'apps_catalog_update = catalog_validation.scripts.catalog_update:main', - 'apps_catalog_validate = catalog_validation.scripts.catalog_validate:main', + 'apps_catalog_update = apps_validation.scripts.catalog_update:main', + 'apps_catalog_validate = apps_validation.scripts.catalog_validate:main', 'apps_dev_charts_validate = apps_validation.scripts.dev_apps_validate:main', # TODO: Remove apps_prefix - 'apps_render_app = apps_validation.scripts.render_compose:main', + 'apps_render_app = catalog_templating.scripts.render_compose:main', ], }, ) From 1250db378bf43735b59ae9a4604595124edb10c2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Mon, 6 May 2024 14:55:10 +0500 Subject: [PATCH 8/8] Add validation for file existence checks in render script --- catalog_templating/scripts/render_compose.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/catalog_templating/scripts/render_compose.py b/catalog_templating/scripts/render_compose.py index aadcbbb..5b1bdc4 100644 --- a/catalog_templating/scripts/render_compose.py +++ b/catalog_templating/scripts/render_compose.py @@ -3,11 +3,19 @@ import os import shutil +from apps_validation.exceptions import ValidationErrors from catalog_reader.app_utils import get_values from catalog_templating.render import render_templates def render_templates_from_path(app_path: str, values_file: str) -> None: + verrors = ValidationErrors() + for k, v in (('app_path', app_path), ('values_file', values_file)): + if not os.path.exists(v): + verrors.add(k, f'{v!r} {k} does not exist') + + verrors.check() + rendered_data = render_templates(app_path, get_values(values_file)) write_template_yaml(app_path, rendered_data)