add jinja2 for cli (#53)

* add jinja2 for cli * linting * linting * linting * linting * linting
ls1intum · Dec 30, 2023 · 8ee65fb · 8ee65fb
1 parent 4c3e82d
commit 8ee65fb
Show file tree

Hide file tree

Showing 10 changed files with 236 additions and 198 deletions.
diff --git a/api/requirements.txt b/api/requirements.txt
@@ -67,7 +67,9 @@ iniconfig==2.0.0
 isort==5.12.0
     # via datamodel-code-generator
 jinja2==3.1.2
-    # via datamodel-code-generator
+    # via
+    #   -r ../cli/requirements.in
+    #   datamodel-code-generator
 jsonschema==4.19.2
     # via
     #   openapi-schema-validator

diff --git a/cli/.editorconfig b/cli/.editorconfig
@@ -0,0 +1,26 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+[*]
+
+# We recommend you to keep these unchanged
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# Change these settings to your own preference
+indent_style = space
+indent_size = 4
+
+[*.md]
+trim_trailing_whitespace = false
+
+[{*.yml,*.yaml}]
+indent_size = 2
+
+[*.j2]
+indent_size = 2
diff --git a/cli/cli_utils/utils.py b/cli/cli_utils/utils.py
@@ -172,6 +172,29 @@ def replace_environment_variables(
     return result
 
 
+def replace_environment_variables_in_dict(
+    environment: EnvironmentSchema,
+    haystack: dict[typing.Any, typing.Any | str | float | bool | None],
+    reverse: bool = False,
+) -> dict[typing.Any, typing.Any | str | float | bool | None]:
+    """
+    Replaces the environment variables in the given list.
+    :param environment: Environment variables
+    :param haystack: List to replace
+    :param reverse: Whether to reverse the replacement or not
+    :return: Replaced list
+    """
+    result: dict[typing.Any, typing.Any | str | float | bool | None] = {}
+    for item_key, item_value in haystack.items():
+        for key, value in environment.__dict__.items():
+            if reverse:
+                value, key = key, value
+            if isinstance(item_value, str):
+                item_value = item_value.replace(key, value)
+        result[item_key] = item_value
+    return result
+
+
 def get_target_environment_variable(
     target: Target, target_independent_name: str, environment: Optional[EnvironmentSchema]
 ) -> str:
@@ -274,6 +297,13 @@ def replace_environment_variables_in_windfile(environment: EnvironmentSchema, wi
         if isinstance(action.root, ScriptAction):
             action.root.script = replace_environment_variable(environment=environment, haystack=action.root.script)
         action.root.environment = replace_environment_dictionary(environment=environment, env=action.root.environment)
+        if action.root.parameters:
+            parameters: Dictionary = Dictionary(
+                root=replace_environment_variables_in_dict(
+                    environment=environment, haystack=action.root.parameters.root.root
+                )
+            )
+            action.root.parameters.root = parameters
 
 
 def combine_docker_config(windfile: WindFile, output_settings: OutputSettings) -> None:

diff --git a/cli/generators/base.py b/cli/generators/base.py
@@ -27,7 +27,9 @@ class BaseGenerator:
     final_result: typing.Optional[str]
     environment: EnvironmentSchema
     key: typing.Optional[str]
+    before_results: dict[str, list[Result]] = {}
     results: dict[str, list[Result]] = {}
+    after_results: dict[str, list[Result]] = {}
     needs_lifecycle_parameter: bool = False
     has_multiple_steps: bool = False
     needs_subshells: bool = False
@@ -49,6 +51,8 @@ def __init__(
             raise ValueError(f"No environment found for target {input_settings.target.value}")
         self.environment = env
         self.results = {}
+        self.before_results = {}
+        self.after_results = {}
         self.key = None
         self.has_multiple_steps = (
             len(
@@ -110,7 +114,7 @@ def has_results(self) -> bool:
         """
         Check if there are results in the windfile.
         """
-        if self.results:
+        if self.before_results or self.after_results:
             return True
         for action in self.windfile.actions:
             if action.root.results and (action.root.platform == self.input_settings.target or not action.root.platform):

diff --git a/cli/generators/cli.py b/cli/generators/cli.py
@@ -9,8 +9,9 @@
 from docker.client import DockerClient  # type: ignore
 from docker.models.containers import Container  # type: ignore
 from docker.types.daemon import CancellableStream  # type: ignore
+from jinja2 import Environment, FileSystemLoader
 
-from classes.generated.definitions import ScriptAction, Repository, Target, Lifecycle, Result
+from classes.generated.definitions import ScriptAction, Repository, Target
 from classes.generated.windfile import WindFile
 from classes.input_settings import InputSettings
 from classes.output_settings import OutputSettings
@@ -35,154 +36,6 @@ def __init__(
         self.functions = []
         super().__init__(windfile, input_settings, output_settings, metadata)
 
-    def add_prefix(self) -> None:
-        """
-        Add the prefix to the bash script.
-        E.g. the shebang, some output settings, etc.
-        """
-        self.result.append("#!/usr/bin/env bash")
-        self.result.append("set -e")
-
-        # actions could run in a different directory, so we need to store to the initial directory
-        if self.has_multiple_steps:
-            self.result.append(f"export {self.initial_directory_variable}=$(pwd)")
-
-        # to work with jenkins and bamboo, we need a way to access the repository url, as this is not possible
-        # in a scripted jenkins pipeline, we set it as an environment variable
-        self.add_repository_urls_to_environment()
-
-        if self.windfile.environment:
-            for env_var in self.windfile.environment.root.root:
-                self.result.append(f'export {env_var}="' f'{self.windfile.environment.root.root[env_var]}"')
-
-    def add_postfix(self) -> None:
-        """
-        Add the postfix to the bash script.
-        E.g. some output settings, the callable functions etc.
-        """
-        self.result.append("\nmain () {")
-        # to enable sourcing the script, we need to skip execution if we do so
-        # for that, we check if the first parameter is sourcing, which is not ever given to the script elsewhere
-        if self.needs_lifecycle_parameter:
-            self.add_line(indentation=2, line='local _current_lifecycle="${1}"')
-        if self.needs_subshells:
-            self.add_line(indentation=2, line='if [[ "${1}" == "aeolus_sourcing" ]]; then')
-            self.add_line(indentation=4, line="return 0 # just source to use the methods in the subshell, no execution")
-            self.add_line(indentation=2, line="fi")
-            self.add_line(indentation=2, line="local _script_name")
-            self.add_line(indentation=2, line="_script_name=${BASH_SOURCE[0]:-$0}")
-        if self.has_always_actions():
-            self.add_line(indentation=2, line="trap final_aeolus_post_action EXIT")
-        for function in self.functions:
-            parameter: str = ""
-            if self.needs_lifecycle_parameter:
-                parameter = ' ${{_current_lifecycle}}"'
-            if self.needs_subshells:
-                self.add_line(
-                    indentation=2,
-                    line=f'bash -c "source ${{_script_name}} aeolus_sourcing;{function}{parameter}"',
-                )
-            else:
-                self.add_line(indentation=2, line=f"{function}{parameter}")
-            if self.has_multiple_steps:
-                self.add_line(indentation=2, line=f'cd "${{{self.initial_directory_variable}}}"')
-        self.result.append("}\n")
-        self.result.append('main "${@}"')
-
-    def handle_always_steps(self, steps: list[str]) -> None:
-        """
-        Translate a step into a CI post action.
-        :param steps: to call always
-        :return: CI action
-        """
-        self.result.append("")
-        self.result.append("final_aeolus_post_action () " + "{")
-        self.add_line(indentation=2, line="set +e # from now on, we don't exit on errors")
-        self.add_line(indentation=2, line="echo '⚙️ executing final_aeolus_post_action'")
-        self.add_line(indentation=2, line=f'cd "${{{self.initial_directory_variable}}}"')
-        for step in steps:
-            parameter: str = ""
-            if self.needs_lifecycle_parameter:
-                parameter = ' ${{_current_lifecycle}}"'
-            self.add_line(indentation=2, line=f"{step}{parameter}")
-            if len(steps) > 1:
-                self.add_line(indentation=2, line=f'cd "${{{self.initial_directory_variable}}}"')
-        self.result.append("}")
-
-    def add_lifecycle_guards(self, name: str, exclusions: Optional[List[Lifecycle]], indentations: int = 2) -> None:
-        """
-        Add lifecycle guards to the given action.
-        :param name: name of the action
-        :param exclusions: list of lifecycle exclusions
-        :param indentations: number of indentations
-        """
-        if exclusions is not None:
-            # we don't need the local variable if there are no exclusions
-            self.add_line(indentation=indentations, line='local _current_lifecycle="${1}"')
-
-            for exclusion in exclusions:
-                self.add_line(
-                    indentation=indentations, line=f'if [[ "${{_current_lifecycle}}" == "{exclusion.name}" ]]; then'
-                )
-                indentations += 2
-                self.add_line(
-                    indentation=indentations, line="echo '⚠️  " f"{name} is excluded during {exclusion.name}'"
-                )
-                self.add_line(indentation=indentations, line="return 0")
-                indentations -= 2
-                self.add_line(indentation=indentations, line="fi")
-
-    def handle_result_list(self, indentation: int, results: List[Result], workdir: Optional[str]) -> None:
-        """
-        Process the results of a step.
-        https://askubuntu.com/a/889746
-        https://stackoverflow.com/a/8088439
-        :param indentation: indentation level
-        :param results: list of results
-        :param workdir: workdir of the step
-        """
-        self.add_line(indentation=indentation, line=f'cd "${{{self.initial_directory_variable}}}"')
-        self.add_line(indentation=indentation, line=f"mkdir -p {self.windfile.metadata.moveResultsTo}")
-        self.add_line(indentation=indentation, line="shopt -s extglob")
-        for result in results:
-            source_path: str = result.path
-            if workdir:
-                source_path = f"{workdir}/{result.path}"
-            self.add_line(indentation=indentation, line=f'local _sources="{source_path}"')
-            self.add_line(indentation=indentation, line="local _directory")
-            self.add_line(indentation=indentation, line='_directory=$(dirname "${_sources}")')
-            if result.ignore:
-                self.add_line(indentation=indentation, line=f'_sources=$(echo "${{_sources}}"/!({result.ignore}))')
-            self.add_line(
-                indentation=indentation, line=f'mkdir -p {self.windfile.metadata.moveResultsTo}/"${{_directory}}"'
-            )
-            self.add_line(
-                indentation=indentation,
-                line=f'cp -a "${{_sources}}" {self.windfile.metadata.moveResultsTo}/"${{_directory}}"',
-            )
-
-    def handle_before_results(self, step: ScriptAction) -> None:
-        """
-        Process the results of a step.
-        :param step: object to process
-        """
-        if not step.results:
-            return
-        before: List[Result] = [result for result in step.results if result.before]
-        if before:
-            self.handle_result_list(indentation=2, results=before, workdir=step.workdir)
-
-    def handle_after_results(self, step: ScriptAction) -> None:
-        """
-        Process the results of a step.
-        :param step: object to process
-        """
-        if not step.results:
-            return
-        after: List[Result] = [result for result in step.results if not result.before]
-        if after:
-            self.handle_result_list(indentation=2, results=after, workdir=step.workdir)
-
     def handle_step(self, name: str, step: ScriptAction, call: bool) -> None:
         """
         Translate a step into a CI action.
@@ -210,26 +63,15 @@ def handle_step(self, name: str, step: ScriptAction, call: bool) -> None:
             while f"{valid_funtion_name}_{number}" in self.functions:
                 number += 1
             valid_funtion_name += f"_{number}"
-            step.name = valid_funtion_name
         if call:
             self.functions.append(valid_funtion_name)
-        self.result.append("")
-        self.result.append(f"{valid_funtion_name} () " + "{")
-        self.add_lifecycle_guards(name=name, exclusions=step.excludeDuring, indentations=2)
+        step.name = valid_funtion_name
 
-        self.add_line(indentation=2, line="echo '⚙️ executing " f"{name}'")
-        if self.windfile.metadata.moveResultsTo:
-            self.handle_before_results(step=step)
-        if step.workdir:
-            self.add_line(indentation=2, line=f'cd "{step.workdir}"')
-        self.add_environment(step=step)
-        self.add_parameters(step=step)
-        for line in step.script.split("\n"):
-            if line:
-                self.add_line(indentation=2, line=line)
-        if self.windfile.metadata.moveResultsTo:
-            self.handle_after_results(step=step)
-        self.result.append("}")
+        if step.results:
+            if self.windfile.metadata.moveResultsTo:
+                self.before_results[step.name] = [result for result in step.results if result.before]
+            if self.windfile.metadata.moveResultsTo:
+                self.after_results[step.name] = [result for result in step.results if result.before]
         return None
 
     def add_environment(self, step: ScriptAction) -> None:
@@ -287,15 +129,6 @@ def check(self, content: str) -> bool:
                     logger.error("❌ ", stdout, self.output_settings.emoji)
             return has_passed
 
-    def handle_clone(self, name: str, repository: Repository) -> None:
-        """
-        Handles the clone step.
-        :param name: Name of the step
-        :param repository: Repository
-        """
-        directory: str = repository.path
-        self.result.append(f"# the repository {name} is expected to be mounted into the container at /{directory}")
-
     def determine_docker_image(self) -> str:
         """
         Determine the docker image to use.
@@ -356,26 +189,46 @@ def run(self, job_id: str) -> None:
             os.unlink(temp.name)
         return
 
+    def generate_using_jinja2(self) -> str:
+        """
+        Generate the bash script to be used as a local CI system with jinja2.
+        """
+        # Load the template from the file system
+        env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates")))
+        template = env.get_template("cli.sh.j2")
+
+        # Prepare your data
+        data = {
+            "has_multiple_steps": self.has_multiple_steps,
+            "initial_directory_variable": self.initial_directory_variable,
+            "environment": self.windfile.environment.root.root if self.windfile.environment else {},
+            "needs_lifecycle_parameter": self.needs_lifecycle_parameter,
+            "needs_subshells": self.needs_subshells,
+            "has_always_actions": self.has_always_actions(),
+            "functions": self.functions,
+            "steps": [action.root for action in self.windfile.actions],
+            "always_steps": [action.root for action in self.windfile.actions if action.root.runAlways],
+            "metadata": self.windfile.metadata,
+            "before_results": self.before_results,
+            "after_results": self.after_results,
+        }
+
+        # Render the template with your data
+        rendered_script = template.render(data)
+
+        return rendered_script
+
     def generate(self) -> str:
         """
         Generate the bash script to be used as a local CI system. We don't clone the repository here, because
         we don't want to handle the credentials in the CI system.
         :return: bash script
         """
+        self.result = []
         utils.replace_environment_variables_in_windfile(environment=self.environment, windfile=self.windfile)
-        self.add_prefix()
-        if self.windfile.repositories:
-            for name in self.windfile.repositories:
-                repository: Repository = self.windfile.repositories[name]
-                self.handle_clone(name, repository)
+        self.add_repository_urls_to_environment()
         for step in self.windfile.actions:
             if isinstance(step.root, ScriptAction):
                 self.handle_step(name=step.root.name, step=step.root, call=not step.root.runAlways)
-        if self.has_always_actions():
-            always_actions: list[str] = []
-            for step in self.windfile.actions:
-                if isinstance(step.root, ScriptAction) and step.root.runAlways:
-                    always_actions.append(step.root.name)
-            self.handle_always_steps(steps=always_actions)
-        self.add_postfix()
+        self.result.append(self.generate_using_jinja2())
         return super().generate()
diff --git a/cli/requirements.in b/cli/requirements.in
@@ -14,4 +14,5 @@ coverage
 gitpython
 types-requests
 urllib3
-python-jenkins
+python-jenkins
+Jinja2