From 8ee65fbc1302558a5b23be5fdc42e9d3c62341c1 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sat, 30 Dec 2023 22:33:14 +0100 Subject: [PATCH] add jinja2 for cli (#53) * add jinja2 for cli * linting * linting * linting * linting * linting --- api/requirements.txt | 4 +- cli/.editorconfig | 26 ++++ cli/cli_utils/utils.py | 30 ++++ cli/generators/base.py | 6 +- cli/generators/cli.py | 227 ++++++------------------------- cli/requirements.in | 3 +- cli/requirements.txt | 4 +- cli/templates/cli.sh.j2 | 122 +++++++++++++++++ cli/test/test_env_replacement.py | 3 +- cli/test/test_generate.py | 9 +- 10 files changed, 236 insertions(+), 198 deletions(-) create mode 100644 cli/.editorconfig create mode 100644 cli/templates/cli.sh.j2 diff --git a/api/requirements.txt b/api/requirements.txt index c8b3d263..2f1f245c 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -67,7 +67,9 @@ iniconfig==2.0.0 isort==5.12.0 # via datamodel-code-generator jinja2==3.1.2 - # via datamodel-code-generator + # via + # -r ../cli/requirements.in + # datamodel-code-generator jsonschema==4.19.2 # via # openapi-schema-validator diff --git a/cli/.editorconfig b/cli/.editorconfig new file mode 100644 index 00000000..c4ea5ff0 --- /dev/null +++ b/cli/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig helps developers define and maintain consistent +# coding styles between different editors and IDEs +# editorconfig.org + +root = true + +[*] + +# We recommend you to keep these unchanged +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +# Change these settings to your own preference +indent_style = space +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false + +[{*.yml,*.yaml}] +indent_size = 2 + +[*.j2] +indent_size = 2 \ No newline at end of file diff --git a/cli/cli_utils/utils.py b/cli/cli_utils/utils.py index 4e69dd74..e38b7bb6 100644 --- a/cli/cli_utils/utils.py +++ b/cli/cli_utils/utils.py @@ -172,6 +172,29 @@ def replace_environment_variables( return result +def replace_environment_variables_in_dict( + environment: EnvironmentSchema, + haystack: dict[typing.Any, typing.Any | str | float | bool | None], + reverse: bool = False, +) -> dict[typing.Any, typing.Any | str | float | bool | None]: + """ + Replaces the environment variables in the given list. + :param environment: Environment variables + :param haystack: List to replace + :param reverse: Whether to reverse the replacement or not + :return: Replaced list + """ + result: dict[typing.Any, typing.Any | str | float | bool | None] = {} + for item_key, item_value in haystack.items(): + for key, value in environment.__dict__.items(): + if reverse: + value, key = key, value + if isinstance(item_value, str): + item_value = item_value.replace(key, value) + result[item_key] = item_value + return result + + def get_target_environment_variable( target: Target, target_independent_name: str, environment: Optional[EnvironmentSchema] ) -> str: @@ -274,6 +297,13 @@ def replace_environment_variables_in_windfile(environment: EnvironmentSchema, wi if isinstance(action.root, ScriptAction): action.root.script = replace_environment_variable(environment=environment, haystack=action.root.script) action.root.environment = replace_environment_dictionary(environment=environment, env=action.root.environment) + if action.root.parameters: + parameters: Dictionary = Dictionary( + root=replace_environment_variables_in_dict( + environment=environment, haystack=action.root.parameters.root.root + ) + ) + action.root.parameters.root = parameters def combine_docker_config(windfile: WindFile, output_settings: OutputSettings) -> None: diff --git a/cli/generators/base.py b/cli/generators/base.py index 50dcbd94..b24972cb 100644 --- a/cli/generators/base.py +++ b/cli/generators/base.py @@ -27,7 +27,9 @@ class BaseGenerator: final_result: typing.Optional[str] environment: EnvironmentSchema key: typing.Optional[str] + before_results: dict[str, list[Result]] = {} results: dict[str, list[Result]] = {} + after_results: dict[str, list[Result]] = {} needs_lifecycle_parameter: bool = False has_multiple_steps: bool = False needs_subshells: bool = False @@ -49,6 +51,8 @@ def __init__( raise ValueError(f"No environment found for target {input_settings.target.value}") self.environment = env self.results = {} + self.before_results = {} + self.after_results = {} self.key = None self.has_multiple_steps = ( len( @@ -110,7 +114,7 @@ def has_results(self) -> bool: """ Check if there are results in the windfile. """ - if self.results: + if self.before_results or self.after_results: return True for action in self.windfile.actions: if action.root.results and (action.root.platform == self.input_settings.target or not action.root.platform): diff --git a/cli/generators/cli.py b/cli/generators/cli.py index 31b94b3b..0c1684c9 100644 --- a/cli/generators/cli.py +++ b/cli/generators/cli.py @@ -9,8 +9,9 @@ from docker.client import DockerClient # type: ignore from docker.models.containers import Container # type: ignore from docker.types.daemon import CancellableStream # type: ignore +from jinja2 import Environment, FileSystemLoader -from classes.generated.definitions import ScriptAction, Repository, Target, Lifecycle, Result +from classes.generated.definitions import ScriptAction, Repository, Target from classes.generated.windfile import WindFile from classes.input_settings import InputSettings from classes.output_settings import OutputSettings @@ -35,154 +36,6 @@ def __init__( self.functions = [] super().__init__(windfile, input_settings, output_settings, metadata) - def add_prefix(self) -> None: - """ - Add the prefix to the bash script. - E.g. the shebang, some output settings, etc. - """ - self.result.append("#!/usr/bin/env bash") - self.result.append("set -e") - - # actions could run in a different directory, so we need to store to the initial directory - if self.has_multiple_steps: - self.result.append(f"export {self.initial_directory_variable}=$(pwd)") - - # to work with jenkins and bamboo, we need a way to access the repository url, as this is not possible - # in a scripted jenkins pipeline, we set it as an environment variable - self.add_repository_urls_to_environment() - - if self.windfile.environment: - for env_var in self.windfile.environment.root.root: - self.result.append(f'export {env_var}="' f'{self.windfile.environment.root.root[env_var]}"') - - def add_postfix(self) -> None: - """ - Add the postfix to the bash script. - E.g. some output settings, the callable functions etc. - """ - self.result.append("\nmain () {") - # to enable sourcing the script, we need to skip execution if we do so - # for that, we check if the first parameter is sourcing, which is not ever given to the script elsewhere - if self.needs_lifecycle_parameter: - self.add_line(indentation=2, line='local _current_lifecycle="${1}"') - if self.needs_subshells: - self.add_line(indentation=2, line='if [[ "${1}" == "aeolus_sourcing" ]]; then') - self.add_line(indentation=4, line="return 0 # just source to use the methods in the subshell, no execution") - self.add_line(indentation=2, line="fi") - self.add_line(indentation=2, line="local _script_name") - self.add_line(indentation=2, line="_script_name=${BASH_SOURCE[0]:-$0}") - if self.has_always_actions(): - self.add_line(indentation=2, line="trap final_aeolus_post_action EXIT") - for function in self.functions: - parameter: str = "" - if self.needs_lifecycle_parameter: - parameter = ' ${{_current_lifecycle}}"' - if self.needs_subshells: - self.add_line( - indentation=2, - line=f'bash -c "source ${{_script_name}} aeolus_sourcing;{function}{parameter}"', - ) - else: - self.add_line(indentation=2, line=f"{function}{parameter}") - if self.has_multiple_steps: - self.add_line(indentation=2, line=f'cd "${{{self.initial_directory_variable}}}"') - self.result.append("}\n") - self.result.append('main "${@}"') - - def handle_always_steps(self, steps: list[str]) -> None: - """ - Translate a step into a CI post action. - :param steps: to call always - :return: CI action - """ - self.result.append("") - self.result.append("final_aeolus_post_action () " + "{") - self.add_line(indentation=2, line="set +e # from now on, we don't exit on errors") - self.add_line(indentation=2, line="echo '⚙️ executing final_aeolus_post_action'") - self.add_line(indentation=2, line=f'cd "${{{self.initial_directory_variable}}}"') - for step in steps: - parameter: str = "" - if self.needs_lifecycle_parameter: - parameter = ' ${{_current_lifecycle}}"' - self.add_line(indentation=2, line=f"{step}{parameter}") - if len(steps) > 1: - self.add_line(indentation=2, line=f'cd "${{{self.initial_directory_variable}}}"') - self.result.append("}") - - def add_lifecycle_guards(self, name: str, exclusions: Optional[List[Lifecycle]], indentations: int = 2) -> None: - """ - Add lifecycle guards to the given action. - :param name: name of the action - :param exclusions: list of lifecycle exclusions - :param indentations: number of indentations - """ - if exclusions is not None: - # we don't need the local variable if there are no exclusions - self.add_line(indentation=indentations, line='local _current_lifecycle="${1}"') - - for exclusion in exclusions: - self.add_line( - indentation=indentations, line=f'if [[ "${{_current_lifecycle}}" == "{exclusion.name}" ]]; then' - ) - indentations += 2 - self.add_line( - indentation=indentations, line="echo '⚠️ " f"{name} is excluded during {exclusion.name}'" - ) - self.add_line(indentation=indentations, line="return 0") - indentations -= 2 - self.add_line(indentation=indentations, line="fi") - - def handle_result_list(self, indentation: int, results: List[Result], workdir: Optional[str]) -> None: - """ - Process the results of a step. - https://askubuntu.com/a/889746 - https://stackoverflow.com/a/8088439 - :param indentation: indentation level - :param results: list of results - :param workdir: workdir of the step - """ - self.add_line(indentation=indentation, line=f'cd "${{{self.initial_directory_variable}}}"') - self.add_line(indentation=indentation, line=f"mkdir -p {self.windfile.metadata.moveResultsTo}") - self.add_line(indentation=indentation, line="shopt -s extglob") - for result in results: - source_path: str = result.path - if workdir: - source_path = f"{workdir}/{result.path}" - self.add_line(indentation=indentation, line=f'local _sources="{source_path}"') - self.add_line(indentation=indentation, line="local _directory") - self.add_line(indentation=indentation, line='_directory=$(dirname "${_sources}")') - if result.ignore: - self.add_line(indentation=indentation, line=f'_sources=$(echo "${{_sources}}"/!({result.ignore}))') - self.add_line( - indentation=indentation, line=f'mkdir -p {self.windfile.metadata.moveResultsTo}/"${{_directory}}"' - ) - self.add_line( - indentation=indentation, - line=f'cp -a "${{_sources}}" {self.windfile.metadata.moveResultsTo}/"${{_directory}}"', - ) - - def handle_before_results(self, step: ScriptAction) -> None: - """ - Process the results of a step. - :param step: object to process - """ - if not step.results: - return - before: List[Result] = [result for result in step.results if result.before] - if before: - self.handle_result_list(indentation=2, results=before, workdir=step.workdir) - - def handle_after_results(self, step: ScriptAction) -> None: - """ - Process the results of a step. - :param step: object to process - """ - if not step.results: - return - after: List[Result] = [result for result in step.results if not result.before] - if after: - self.handle_result_list(indentation=2, results=after, workdir=step.workdir) - def handle_step(self, name: str, step: ScriptAction, call: bool) -> None: """ Translate a step into a CI action. @@ -210,26 +63,15 @@ def handle_step(self, name: str, step: ScriptAction, call: bool) -> None: while f"{valid_funtion_name}_{number}" in self.functions: number += 1 valid_funtion_name += f"_{number}" - step.name = valid_funtion_name if call: self.functions.append(valid_funtion_name) - self.result.append("") - self.result.append(f"{valid_funtion_name} () " + "{") - self.add_lifecycle_guards(name=name, exclusions=step.excludeDuring, indentations=2) + step.name = valid_funtion_name - self.add_line(indentation=2, line="echo '⚙️ executing " f"{name}'") - if self.windfile.metadata.moveResultsTo: - self.handle_before_results(step=step) - if step.workdir: - self.add_line(indentation=2, line=f'cd "{step.workdir}"') - self.add_environment(step=step) - self.add_parameters(step=step) - for line in step.script.split("\n"): - if line: - self.add_line(indentation=2, line=line) - if self.windfile.metadata.moveResultsTo: - self.handle_after_results(step=step) - self.result.append("}") + if step.results: + if self.windfile.metadata.moveResultsTo: + self.before_results[step.name] = [result for result in step.results if result.before] + if self.windfile.metadata.moveResultsTo: + self.after_results[step.name] = [result for result in step.results if result.before] return None def add_environment(self, step: ScriptAction) -> None: @@ -287,15 +129,6 @@ def check(self, content: str) -> bool: logger.error("❌ ", stdout, self.output_settings.emoji) return has_passed - def handle_clone(self, name: str, repository: Repository) -> None: - """ - Handles the clone step. - :param name: Name of the step - :param repository: Repository - """ - directory: str = repository.path - self.result.append(f"# the repository {name} is expected to be mounted into the container at /{directory}") - def determine_docker_image(self) -> str: """ Determine the docker image to use. @@ -356,26 +189,46 @@ def run(self, job_id: str) -> None: os.unlink(temp.name) return + def generate_using_jinja2(self) -> str: + """ + Generate the bash script to be used as a local CI system with jinja2. + """ + # Load the template from the file system + env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))) + template = env.get_template("cli.sh.j2") + + # Prepare your data + data = { + "has_multiple_steps": self.has_multiple_steps, + "initial_directory_variable": self.initial_directory_variable, + "environment": self.windfile.environment.root.root if self.windfile.environment else {}, + "needs_lifecycle_parameter": self.needs_lifecycle_parameter, + "needs_subshells": self.needs_subshells, + "has_always_actions": self.has_always_actions(), + "functions": self.functions, + "steps": [action.root for action in self.windfile.actions], + "always_steps": [action.root for action in self.windfile.actions if action.root.runAlways], + "metadata": self.windfile.metadata, + "before_results": self.before_results, + "after_results": self.after_results, + } + + # Render the template with your data + rendered_script = template.render(data) + + return rendered_script + def generate(self) -> str: """ Generate the bash script to be used as a local CI system. We don't clone the repository here, because we don't want to handle the credentials in the CI system. :return: bash script """ + self.result = [] utils.replace_environment_variables_in_windfile(environment=self.environment, windfile=self.windfile) - self.add_prefix() - if self.windfile.repositories: - for name in self.windfile.repositories: - repository: Repository = self.windfile.repositories[name] - self.handle_clone(name, repository) + self.add_repository_urls_to_environment() for step in self.windfile.actions: if isinstance(step.root, ScriptAction): self.handle_step(name=step.root.name, step=step.root, call=not step.root.runAlways) - if self.has_always_actions(): - always_actions: list[str] = [] - for step in self.windfile.actions: - if isinstance(step.root, ScriptAction) and step.root.runAlways: - always_actions.append(step.root.name) - self.handle_always_steps(steps=always_actions) - self.add_postfix() + self.result.append(self.generate_using_jinja2()) return super().generate() diff --git a/cli/requirements.in b/cli/requirements.in index b5f33fe5..f48cd2f6 100644 --- a/cli/requirements.in +++ b/cli/requirements.in @@ -14,4 +14,5 @@ coverage gitpython types-requests urllib3 -python-jenkins \ No newline at end of file +python-jenkins +Jinja2 \ No newline at end of file diff --git a/cli/requirements.txt b/cli/requirements.txt index c154eb8a..ba3943c3 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -53,7 +53,9 @@ iniconfig==2.0.0 isort==5.12.0 # via datamodel-code-generator jinja2==3.1.2 - # via datamodel-code-generator + # via + # -r requirements.in + # datamodel-code-generator jsonschema==4.17.3 # via # jsonschema-spec diff --git a/cli/templates/cli.sh.j2 b/cli/templates/cli.sh.j2 new file mode 100644 index 00000000..bffdac49 --- /dev/null +++ b/cli/templates/cli.sh.j2 @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +{% set initial_directory = "\"${%s}\"" % initial_directory_variable -%} +set -e +{% if has_multiple_steps -%} + export {{ initial_directory_variable }}=${PWD} +{%- endif -%} + +{# Handling environment variables from windfile -#} +{%- for env_var in environment -%} + export {{ env_var }}="{{ environment[env_var] }}" +{%- endfor -%} + +{# Additional functions based on steps -#} +{%- for step in steps %} +{{ step.name }} () { + echo '⚙️ executing {{ step.name }}' + {% if step.workdir -%} + cd "{{ step.workdir }}" + {% endif %} + {%- if step.environment -%} + {%- for env_var, env_value in step.environment.items -%} + export {{ env_var }}="{{ env_value }}" + {%- endfor -%} + {% endif %} + {%- if step.parameters -%} + {%- for param_var, param_value in step.parameters.items -%} + export {{ param_var }}="{{ param_value }}" + {% endfor %} + {%- endif -%} + {%- if step.needs_lifecycle_parameter -%} + local _current_lifecycle="${1}" + {% endif %} + {%- if metadata.moveResultsTo %} + cd {{ initial_directory }} + mkdir -p {{ metadata.moveResultsTo }} + shopt -s extglob + {%- for result in before_results[step.name] %} + {%- set source_path = result.path -%} + {%- if step.workdir -%} + {%- set source_path = step.workdir + "/" + result.path -%} + {%- endif -%} + local _sources="{{ source_path }}" + local _directory + _directory=$(dirname "${_sources}") + {%- if result.ignore -%} + _sources=$(echo ${_sources}/!({{ result.ignore | join(' ') }})) + {%- endif -%} + cp -a ${_sources} {{ metadata.moveResultsTo }}/${_directory} + {%- endfor -%} + {%- endif -%} + {{ step.script }} + {%- if metadata.moveResultsTo %} + cd {{ initial_directory }} + mkdir -p {{ metadata.moveResultsTo }} + shopt -s extglob + {%- for result in after_results[step.name] %} + {%- set source_path = result.path -%} + {%- if step.workdir -%} + {%- set source_path = step.workdir + "/" + result.path -%} + {%- endif -%} + local _sources="{{ source_path }}" + local _directory + _directory=$(dirname "${_sources}") + {%- if result.ignore -%} + _sources=$(echo ${_sources}/!({{ result.ignore | join(' ') }})) + {%- endif -%} + cp -a ${_sources} {{ metadata.moveResultsTo }}/${_directory} + {%- endfor -%} + {%- endif %} +} +{% endfor -%} + +{%- if has_always_actions -%} +function final_aeolus_post_action () { + set +e # from now on, we don't exit on errors + echo '⚙️ executing final_aeolus_post_action' + cd {{ initial_directory }} + {%- for step in always_steps -%} + {%- if needs_lifecycle_parameter %} + {{ step.name }} "${_current_lifecycle}" + {%- else %} + {{ step.name }} + {%- endif -%} + {% endfor %} +} +{% endif %} +main () { + if [[ "${1}" == "aeolus_sourcing" ]]; then + return 0 # just source to use the methods in the subshell, no execution + fi +{%- if needs_lifecycle_parameter -%} + local _current_lifecycle="${1}" +{% endif %} +{%- if needs_subshells %} + local _script_name + _script_name=${BASH_SOURCE[0]:-$0} +{%- endif -%} +{% if has_always_actions %} + trap final_aeolus_post_action EXIT +{% endif %} +{%- for function in functions %} +{%- if needs_lifecycle_parameter -%} +{%- if needs_subshells -%} + cd {{ initial_directory }} + bash -c "source ${_script_name} aeolus_sourcing; {{ function }} \"${_current_lifecycle}\"" +{%- else %} + cd {{ initial_directory }} + {{ function }} "${_current_lifecycle}" + {%- endif -%} + {%- else -%} + {%- if needs_subshells %} + cd {{ initial_directory }} + bash -c "source ${_script_name} aeolus_sourcing; {{ function }}" + {%- else %} + cd {{ initial_directory }} + {{ function }} + {%- endif -%} + {%- endif -%} +{% endfor %} +} + +main "${@}" diff --git a/cli/test/test_env_replacement.py b/cli/test/test_env_replacement.py index 4958b348..54848996 100644 --- a/cli/test/test_env_replacement.py +++ b/cli/test/test_env_replacement.py @@ -102,6 +102,7 @@ def generate_and_check_if_repository_variables_are_set( metadata=metadata, ) result: str = gen.generate() + if windfile is None: self.fail("Windfile is None") @@ -132,5 +133,5 @@ def check_if_all_env_variables_are_replaced(self, result: str, env_vars: Environ allowed: List[str] = [env_vars.__dict__[e] for e in env_vars.__dict__.keys()] forbidden_with_none: List[Optional[str]] = [e if e not in allowed else None for e in env_vars.__dict__.keys()] forbidden: List[str] = [e for e in forbidden_with_none if e is not None] - self.assertFalse(all(e not in result for e in forbidden)) + self.assertTrue(all(e not in result for e in forbidden)) self.assertTrue(any(e in result for e in allowed)) diff --git a/cli/test/test_generate.py b/cli/test/test_generate.py index 860be378..a90fe2fd 100644 --- a/cli/test/test_generate.py +++ b/cli/test/test_generate.py @@ -50,10 +50,8 @@ def test_generate_valid_cli_script(self) -> None: result: str = cli.generate() self.assertTrue(result.count("#!/usr/bin/env bash") == 1) self.assertTrue("set -e" in result) - # one echo for execution, one echo in the actual action - self.assertTrue(result.count("internal-action") == 1) - # one call to the function and the function itself - self.assertTrue(result.count("internalaction") == 2) + # one echo for execution, one call to the function and the function itself + self.assertTrue(result.count("internalaction") == 3) self.assertTrue(result.count("{") == result.count("}")) self.assertTrue(cli.check(content=result)) @@ -132,8 +130,7 @@ def test_generate_cli_script_with_workdir(self) -> None: # we change into the workdir, so we need to change back, and to be sure to always # be in the correct directory, we need to do this after every action self.assertTrue(result.count('cd "/aeolus"') == 1) - self.assertIn("export AEOLUS_INITIAL_DIRECTORY=$(pwd)", result) - print(result) + self.assertIn("export AEOLUS_INITIAL_DIRECTORY=${PWD}", result) self.assertTrue(result.count('cd "${AEOLUS_INITIAL_DIRECTORY}"') == len(windfile.actions)) def test_generate_jenkinsfile_with_workdir(self) -> None: