From 66c7fcf78427a415c7e631b490b57c8ab8f0b2cd Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 09:35:30 +0100 Subject: [PATCH 1/7] cache template --- cli/generators/cli.py | 9 ++++++--- cli/templates/cli.sh.j2 | 14 +++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cli/generators/cli.py b/cli/generators/cli.py index 0c1684c9..de65079c 100644 --- a/cli/generators/cli.py +++ b/cli/generators/cli.py @@ -29,6 +29,8 @@ class CliGenerator(BaseGenerator): initial_directory_variable: str = "AEOLUS_INITIAL_DIRECTORY" + template: Optional[typing.Any] = None + def __init__( self, windfile: WindFile, input_settings: InputSettings, output_settings: OutputSettings, metadata: PassMetadata ): @@ -194,8 +196,9 @@ def generate_using_jinja2(self) -> str: Generate the bash script to be used as a local CI system with jinja2. """ # Load the template from the file system - env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))) - template = env.get_template("cli.sh.j2") + if not self.template: + env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))) + self.template = env.get_template("cli.sh.j2") # Prepare your data data = { @@ -214,7 +217,7 @@ def generate_using_jinja2(self) -> str: } # Render the template with your data - rendered_script = template.render(data) + rendered_script = self.template.render(data) return rendered_script diff --git a/cli/templates/cli.sh.j2 b/cli/templates/cli.sh.j2 index bffdac49..8737b5e0 100644 --- a/cli/templates/cli.sh.j2 +++ b/cli/templates/cli.sh.j2 @@ -5,12 +5,12 @@ set -e export {{ initial_directory_variable }}=${PWD} {%- endif -%} -{# Handling environment variables from windfile -#} -{%- for env_var in environment -%} - export {{ env_var }}="{{ environment[env_var] }}" -{%- endfor -%} +{%- if environment -%} +{% for env_var in environment %} +export {{ env_var }}="{{ environment[env_var] }}" +{% endfor %} +{%- endif -%} -{# Additional functions based on steps -#} {%- for step in steps %} {{ step.name }} () { echo '⚙️ executing {{ step.name }}' @@ -70,8 +70,8 @@ set -e } {% endfor -%} -{%- if has_always_actions -%} -function final_aeolus_post_action () { +{%- if has_always_actions %} +final_aeolus_post_action () { set +e # from now on, we don't exit on errors echo '⚙️ executing final_aeolus_post_action' cd {{ initial_directory }} From 65d17f5051993cf724b9b929f0078b9e9c89aea4 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 17:22:02 +0100 Subject: [PATCH 2/7] add jinja for jenkins --- .github/workflows/deploy-test.yaml | 10 + cli/.editorconfig | 5 +- cli/generators/bamboo.py | 14 +- cli/generators/base.py | 15 +- cli/generators/cli.py | 35 +-- cli/generators/jenkins.py | 345 ++++------------------------- cli/templates/Jenkinsfile.j2 | 161 ++++++++++++++ 7 files changed, 227 insertions(+), 358 deletions(-) create mode 100644 cli/templates/Jenkinsfile.j2 diff --git a/.github/workflows/deploy-test.yaml b/.github/workflows/deploy-test.yaml index 4a0c44a9..7c8dd1f2 100644 --- a/.github/workflows/deploy-test.yaml +++ b/.github/workflows/deploy-test.yaml @@ -1,6 +1,16 @@ name: Deploy to Aeolus Test on: + workflow_dispatch: + inputs: + docker-tag: + description: 'Docker tag to deploy (e.g. 1.0.0 or latest, default: nightly)' + required: true + default: 'nightly' + branch-name: + description: 'Branch name to deploy (default: develop)' + required: true + default: 'develop' pull_request: types: [labeled] diff --git a/cli/.editorconfig b/cli/.editorconfig index c4ea5ff0..a385a5e0 100644 --- a/cli/.editorconfig +++ b/cli/.editorconfig @@ -23,4 +23,7 @@ trim_trailing_whitespace = false indent_size = 2 [*.j2] -indent_size = 2 \ No newline at end of file +indent_size = 2 + +[Jenkinsfile] +indent_size = 2 diff --git a/cli/generators/bamboo.py b/cli/generators/bamboo.py index 51916330..7c22bf56 100644 --- a/cli/generators/bamboo.py +++ b/cli/generators/bamboo.py @@ -16,10 +16,6 @@ from typing import List, Any, Optional import requests -from docker.client import DockerClient # type: ignore -from docker.errors import DockerException # type: ignore -from docker.models.containers import Container # type: ignore -from docker.types.daemon import CancellableStream # type: ignore import cli_utils from classes.generated.definitions import Target @@ -29,6 +25,10 @@ from classes.pass_metadata import PassMetadata from cli_utils import logger, utils from generators.base import BaseGenerator +from docker.client import DockerClient # type: ignore +from docker.errors import DockerException # type: ignore +from docker.models.containers import Container # type: ignore +from docker.types.daemon import CancellableStream # type: ignore def docker_available() -> bool: @@ -111,7 +111,7 @@ def generate_in_api(self, payload: str) -> Optional[str]: if response.status_code == 200: logger.info("🔨", "Bamboo YAML Spec file generated", self.output_settings.emoji) body: dict[str, str] = response.json() - self.result.append(body["result"]) + self.result = body["result"] return body["key"] logger.error("❌", "Bamboo YAML Spec file generation failed", self.output_settings.emoji) raise ValueError("Bamboo YAML Spec file generation failed") @@ -142,7 +142,7 @@ def generate_in_java(self, base64_str: str) -> None: raise ValueError("Bamboo YAML Spec file generation failed") logger.info("🔨", "Bamboo YAML Spec file generated", self.output_settings.emoji) result_logs: str = process.stdout - self.result.append(result_logs) + self.result = result_logs def generate_in_docker(self, base64_str: str) -> None: """ @@ -182,7 +182,7 @@ def generate_in_docker(self, base64_str: str) -> None: logger.info("🔨", "Bamboo YAML Spec file generated", self.output_settings.emoji) result_logs: str = container.logs(stdout=True, stderr=False).decode("utf-8") container.remove() - self.result.append(result_logs) + self.result = result_logs def check(self, content: str) -> bool: raise NotImplementedError("check_syntax() not implemented") diff --git a/cli/generators/base.py b/cli/generators/base.py index b24972cb..5895ed9f 100644 --- a/cli/generators/base.py +++ b/cli/generators/base.py @@ -23,7 +23,7 @@ class BaseGenerator: input_settings: InputSettings output_settings: OutputSettings metadata: PassMetadata - result: typing.List[str] + result: str final_result: typing.Optional[str] environment: EnvironmentSchema key: typing.Optional[str] @@ -41,7 +41,7 @@ def __init__( self.input_settings = input_settings self.output_settings = output_settings self.metadata = metadata - self.result = [] + self.result = "" if input_settings.target is None: raise ValueError("No target specified") env: typing.Optional[EnvironmentSchema] = utils.get_ci_environment( @@ -67,14 +67,6 @@ def __init__( self.needs_lifecycle_parameter = self.__needs_lifecycle_parameter() self.needs_subshells = self.has_multiple_steps - def add_line(self, indentation: int, line: str) -> None: - """ - Add a line to the result. - :param indentation: indentation level - :param line: line to add - """ - self.result.append(" " * indentation + line) - def add_repository_urls_to_environment(self) -> None: """ Some systems don't offer a way to access repository urls in the CI file. @@ -160,8 +152,7 @@ def generate(self) -> str: """ Generate the CI file. """ - self.final_result = "\n".join(self.result) - return self.final_result + return self.result def run(self, job_id: str) -> None: """ diff --git a/cli/generators/cli.py b/cli/generators/cli.py index de65079c..e6cc7915 100644 --- a/cli/generators/cli.py +++ b/cli/generators/cli.py @@ -3,13 +3,9 @@ import re import subprocess import tempfile -import typing from typing import List, Optional -from docker.client import DockerClient # type: ignore -from docker.models.containers import Container # type: ignore -from docker.types.daemon import CancellableStream # type: ignore -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader, Template from classes.generated.definitions import ScriptAction, Repository, Target from classes.generated.windfile import WindFile @@ -18,6 +14,9 @@ from classes.pass_metadata import PassMetadata from cli_utils import logger, utils from generators.base import BaseGenerator +from docker.client import DockerClient # type: ignore +from docker.models.containers import Container # type: ignore +from docker.types.daemon import CancellableStream # type: ignore class CliGenerator(BaseGenerator): @@ -29,7 +28,7 @@ class CliGenerator(BaseGenerator): initial_directory_variable: str = "AEOLUS_INITIAL_DIRECTORY" - template: Optional[typing.Any] = None + template: Optional[Template] = None def __init__( self, windfile: WindFile, input_settings: InputSettings, output_settings: OutputSettings, metadata: PassMetadata @@ -76,30 +75,6 @@ def handle_step(self, name: str, step: ScriptAction, call: bool) -> None: self.after_results[step.name] = [result for result in step.results if result.before] return None - def add_environment(self, step: ScriptAction) -> None: - """ - Add environment variables to the step. - :param step: to add environment variables to - """ - if step.environment: - for env_var in step.environment.root.root: - env_value: typing.Any = step.environment.root.root[env_var] - if isinstance(env_value, List): - env_value = " ".join(env_value) - self.result.append(f'export {env_var}="' f'{env_value}"') - - def add_parameters(self, step: ScriptAction) -> None: - """ - Add parameters to the step. - :param step: to add parameters to - """ - if step.parameters is not None: - for parameter in step.parameters.root.root: - value: typing.Any = step.parameters.root.root[parameter] - if isinstance(value, List): - value = " ".join(value) - self.add_line(indentation=2, line=f'{parameter}="' f'{value}"') - def check(self, content: str) -> bool: """ Check the generated bash file for syntax errors. diff --git a/cli/generators/jenkins.py b/cli/generators/jenkins.py index 98b97d6f..b33356dd 100644 --- a/cli/generators/jenkins.py +++ b/cli/generators/jenkins.py @@ -3,13 +3,14 @@ Jenkins generator. Generates a jenkins pipeline to be used in the Jenkins CI system. The generated pipeline is a scripted pipeline. """ -import typing -from typing import Optional, List, Any +import os +from typing import Optional, List from xml.dom.minidom import Document, parseString, Element import jenkins # type: ignore +from jinja2 import Environment, FileSystemLoader, Template -from classes.generated.definitions import ScriptAction, Target, Repository, Docker +from classes.generated.definitions import Target from classes.generated.windfile import WindFile from classes.input_settings import InputSettings from classes.output_settings import OutputSettings @@ -24,310 +25,24 @@ class JenkinsGenerator(BaseGenerator): to be used in the Jenkins CI system. """ + template: Optional[Template] = None + def __init__( self, windfile: WindFile, input_settings: InputSettings, output_settings: OutputSettings, metadata: PassMetadata ): input_settings.target = Target.jenkins super().__init__(windfile, input_settings, output_settings, metadata) - def add_docker_config(self, config: Optional[Docker], indentation: int) -> None: - """ - Add the docker configuration to the pipeline or stage - :param config: Docker configuration to add - :param indentation: indentation level - """ - if config is None: - return - tag: str = config.tag if config.tag is not None else "latest" - complete_image: str = config.image + ":" + tag if ":" not in config.image else config.image - self.add_line(indentation=indentation, line="agent {") - self.add_line(indentation=indentation, line=" docker {") - self.add_line(indentation=indentation, line=" image '" + complete_image + "'") - args: Optional[str] = None - if config.volumes is not None: - args = "-v " + ":".join(config.volumes) - if config.parameters is not None: - if args is None: - args = "" - args += " " + " ".join(config.parameters) - if args is not None: - self.add_line(indentation=indentation, line=" args '" + args + "'") - self.add_line(indentation=indentation, line=" }") - self.add_line(indentation=indentation, line="}") - - def add_lifecycle_parameter(self, indentation: int) -> None: - self.add_line(indentation=indentation, line="parameters {") - # we set it to working_time by default, as this is the most common case, and we want to avoid - # that the job does not execute stages only meant to be executed during evaluation (e.g. hidden tests) - indentation += 2 - self.add_line( - indentation=indentation, - line="string(name: 'current_lifecycle', defaultValue: 'working_time', description: 'The current stage')", - ) - indentation -= 2 - self.add_line(indentation=indentation, line="}") - def add_prefix(self) -> None: """ Add the prefix to the pipeline, e.g. the agent, the environment, etc. """ - indentation: int = 0 - self.add_line(indentation=indentation, line="pipeline {") - indentation += 2 - if self.windfile.metadata.docker is None: - self.add_line(indentation=indentation, line="agent any") - else: - self.add_docker_config(config=self.windfile.metadata.docker, indentation=indentation) # to respect the exclusion during different parts of the lifecycle # we need a parameter that holds the current lifecycle - if self.needs_lifecycle_parameter: - self.add_lifecycle_parameter(indentation=indentation) # to work with jenkins and bamboo, we need a way to access the repository url, as this is not possible # in a scripted jenkins pipeline, we set it as an environment variable self.add_repository_urls_to_environment() - if self.windfile.environment: - self.add_line(indentation=indentation, line="environment {") - indentation += 2 - for env_var in self.windfile.environment.root.root: - self.add_line( - indentation=indentation, line=f"{env_var} = '{self.windfile.environment.root.root[env_var]}'" - ) - indentation -= 2 - self.add_line(indentation=indentation, line="}") - assert indentation == 2 # we need to be at the same level as in the beginning, otherwise something is wrong - self.add_line(indentation=indentation, line="stages {") - - def add_postfix(self) -> None: - """ - Add the postfix to the pipeline - """ - self.result.append("}") - - def handle_always_step(self, name: str, step: ScriptAction, indentation: int = 4) -> None: - """ - Translate a step into a CI post action. - :param name: Name of the step to handle - :param step: to translate - :param indentation: indentation level - :return: CI action - """ - original_type: Optional[str] = self.metadata.get_meta_for_action(name).get("original_type") - - self.add_script( - wrapper="always", - name=name, - original_type=original_type, - script=step.script, - indentation=indentation, - workdir=step.workdir, - ) - self.handle_step_results(workdir=step.workdir, step=step) - if self.windfile.metadata.resultHook: - script: str = "sendTestResults " - if self.windfile.metadata.resultHookCredentials: - script += f"credentialsId: '{self.windfile.metadata.resultHookCredentials}', " - script += f"notificationUrl: '{self.windfile.metadata.resultHook}'" - self.add_script( - wrapper="post", - name=name, - original_type="custom", - script=script, - indentation=indentation, - workdir=None, - ) - - # pylint: disable=too-many-arguments - def add_script( - self, - wrapper: str, - name: str, - original_type: Optional[str], - script: str, - indentation: int, - workdir: Optional[str], - ) -> None: - """ - Add a script to the pipeline. - :param wrapper: wrapper to use, e.g. steps, post, always etc. - :param name: Name of the step to handle - :param original_type: original type of the action - :param script: Script to add - :param indentation: indentation level - :param workdir: workdir to use - """ - self.add_line(indentation=indentation, line=f"{wrapper} " + "{") - indentation += 2 - self.add_line(indentation=indentation, line=f"echo '⚙️ executing {name}'") - if workdir: - self.add_line(indentation=indentation, line=f"dir('{workdir}') " + "{") - indentation += 2 - was_script_or_file: bool = original_type in ("file", "script") - if was_script_or_file: - self.add_line(indentation=indentation, line="sh '''") - for line in script.split("\n"): - if line: - self.add_line(indentation=indentation, line=line) - if was_script_or_file: - self.add_line(indentation=indentation, line="'''") - if workdir: - indentation -= 2 - self.add_line(indentation=indentation, line="}") - indentation -= 2 - self.add_line(indentation=indentation, line="}") - - def add_results( - self, - indentation: int, - ) -> None: - """ - Add a script to the pipeline. - :param indentation: indentation level - """ - self.add_line(indentation=indentation, line="always {") - indentation += 2 - self.add_line(indentation=indentation, line="echo '⚙️ publishing results'") - for workdir, entries in self.results.items(): - for result in entries: - full_path: str = workdir + "/" + result.path - ignore: str = f"exclude: {result.ignore}" if result.ignore else "" - if result.type == "junit": - self.add_line(indentation=indentation, line=f"junit '{full_path}'") - else: - self.add_line( - indentation=indentation, - line=f"archiveArtifacts: '{full_path}', fingerprint: true, allowEmptyArchive: true, {ignore}", - ) - indentation -= 2 - self.add_line(indentation=indentation, line="}") - - def handle_step(self, name: str, step: ScriptAction, call: bool) -> None: - """ - Translate a step into a CI action. - :param name: Name of the step to handle - :param step: to translate - :param call: whether to call the step or not - :return: CI action - """ - original_type: Optional[str] = self.metadata.get_meta_for_action(name).get("original_type") - if original_type == "platform": - if step.platform == Target.jenkins.name: - logger.info( - "🔨", - "Platform action detected. Should be executed now...", - self.output_settings.emoji, - ) - # TODO implement # pylint: disable=fixme - return None - logger.info( - "🔨", - "Unfitting platform action detected. Skipping...", - self.output_settings.emoji, - ) - return None - self.add_line(indentation=4, line=f"stage('{name}') " + "{") - self.add_docker_config(config=step.docker, indentation=6) - self.handle_step_results(workdir=step.workdir, step=step) - - if step.excludeDuring is not None: - self.add_line(indentation=6, line="when {") - self.add_line(indentation=8, line="anyOf {") - for exclusion in step.excludeDuring: - self.add_line(indentation=10, line=f"expression {{ params.current_lifecycle != '{exclusion.name}' }}") - self.add_line(indentation=8, line="}") - self.add_line(indentation=6, line="}") - self.add_environment_variables(step=step) - self.add_script( - wrapper="steps", - name=name, - original_type=original_type, - script=step.script, - indentation=6, - workdir=step.workdir, - ) - self.add_line(indentation=4, line="}") - return None - - def handle_step_results(self, workdir: Optional[str], step: ScriptAction) -> None: - """ - Process the results of a step. - :param workdir: working directory of the step - :param step: object to process - """ - key: str = workdir if workdir else "." - if step.results: - for result in step.results: - self.add_result(workdir=key, result=result) - - def add_environment_variables(self, step: ScriptAction) -> None: - """ - Add environment variables and parameters to the step. - :param step: Step to add environment variables to - """ - if step.environment is not None or step.parameters is not None: - self.add_line(indentation=6, line="environment {") - if step.parameters is not None: - for param in step.parameters.root.root: - param_value: typing.Any = step.parameters.root.root[param] - if isinstance(param_value, List): - param_value = " ".join(param_value) - self.add_line(indentation=8, line=f'{param} = "{param_value}"') - if step.environment is not None: - for env_var in step.environment.root.root: - value: typing.Any = step.environment.root.root[env_var] - if isinstance(value, List): - value = " ".join(value) - self.add_line(indentation=8, line=f'{env_var} = "{value}"') - self.add_line(indentation=6, line="}") - - def handle_clone(self, name: str, repository: Repository, indentation: int) -> None: - """ - Handles the clone step. - :param name: Name of the repository to clone - :param repository: Repository ot checkout - :param indentation: indentation level - """ - original_indentation: int = indentation - self.add_line(indentation=indentation, line=f"stage('{name}') " + "{") - indentation += 2 - self.add_line(indentation=indentation, line="steps {") - indentation += 2 - self.add_line(indentation=indentation, line=f"echo '🖨️ cloning {name}'") - if repository.path != ".": - self.add_line(indentation=indentation, line=f"dir('{repository.path}') " + "{") - indentation += 2 - self.add_line(indentation=indentation, line="checkout([$class: 'GitSCM',") - indentation += 2 - self.add_line(indentation=indentation, line="branches: [[name: '" + repository.branch + "']],") - self.add_line(indentation=indentation, line="doGenerateSubmoduleConfigurations: false,") - self.add_line(indentation=indentation, line="extensions: [],") - self.add_line(indentation=indentation, line="submoduleCfg: [],") - self.add_line(indentation=indentation, line="userRemoteConfigs: [[") - indentation += 2 - if self.windfile.metadata.gitCredentials and isinstance(self.windfile.metadata.gitCredentials, str): - self.add_line( - indentation=indentation, line="credentialsId: '" + self.windfile.metadata.gitCredentials + "'," - ) - self.add_line(indentation=indentation, line=f"name: '{name}',") - url: str = repository.url - if self.metadata.get(scope="repositories", key=name, subkey="url") is not None: - repository_metadata: dict[str, Any] = self.metadata.get(scope="repositories") - if repository_metadata is not None and name in repository_metadata and "url" in repository_metadata[name]: - cached_value: str = repository_metadata[name]["url"] - url = "${" + cached_value + "}" - self.add_line(indentation=indentation, line=f"url: '{url}'") - indentation -= 2 - self.add_line(indentation=indentation, line="]]") - indentation -= 2 - self.add_line(indentation=indentation, line="])") - indentation -= 2 - self.add_line(indentation=indentation, line="}") - if repository.path != ".": - indentation -= 2 - self.add_line(indentation=indentation, line="}") - indentation -= 2 - self.add_line(indentation=indentation, line="}") - assert indentation == original_indentation def run(self, job_id: str) -> None: """ @@ -391,6 +106,35 @@ def publish(self) -> None: server.upsert_job(job_name, config_xml.toxml()) + def generate_using_jinja2(self) -> str: + """ + Generate the bash script to be used as a local CI system with jinja2. + """ + if not self.template: + env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))) + self.template = env.get_template("Jenkinsfile.j2") + + data = { + "docker": self.windfile.metadata.docker, + "environment": self.windfile.environment.root.root if self.windfile.environment else {}, + "needs_lifecycle_parameter": self.needs_lifecycle_parameter, + "repositories": self.windfile.repositories if self.windfile.repositories else {}, + "has_always_actions": self.has_always_actions(), + "steps": [action.root for action in self.windfile.actions if not action.root.runAlways], + "always_steps": [action.root for action in self.windfile.actions if action.root.runAlways], + "metadata": self.windfile.metadata, + "before_results": self.before_results, + "after_results": self.after_results, + "repo_metadata": self.metadata.get(scope="repositories"), + "has_results": self.has_results(), + "results": self.results, + } + + # Render the template with your data + rendered_script = self.template.render(data) + + return rendered_script + def generate(self) -> str: """ Generate the bash script to be used as a local CI system. @@ -398,25 +142,10 @@ def generate(self) -> str: """ utils.replace_environment_variables_in_windfile(environment=self.environment, windfile=self.windfile) self.add_prefix() - if self.windfile.repositories: - for name in self.windfile.repositories: - repository: Repository = self.windfile.repositories[name] - self.handle_clone(name=name, repository=repository, indentation=4) - for step in self.windfile.actions: - if isinstance(step.root, ScriptAction) and not step.root.runAlways: - self.handle_step(name=step.root.name, step=step.root, call=True) - self.add_line(indentation=2, line="}") - if self.has_always_actions() or self.has_results(): - self.add_line(indentation=2, line="post {") - for post_step in self.windfile.actions: - if isinstance(post_step.root, ScriptAction) and post_step.root.runAlways: - self.handle_always_step(name=post_step.root.name, step=post_step.root) - if self.has_results(): - self.add_results(indentation=4) - self.add_line(indentation=2, line="}") - self.add_postfix() + if self.output_settings.ci_credentials is not None: self.publish() + self.result = self.generate_using_jinja2() return super().generate() def check(self, content: str) -> bool: diff --git a/cli/templates/Jenkinsfile.j2 b/cli/templates/Jenkinsfile.j2 new file mode 100644 index 00000000..9a92ee1a --- /dev/null +++ b/cli/templates/Jenkinsfile.j2 @@ -0,0 +1,161 @@ +pipeline { +{%- if docker %} + agent { + docker { + {%- set dockerImage = "%s" % docker.image -%} + {%- if docker.tag -%} + {% set dockerImage = "{}:{}".format(dockerImage, docker.tag) %} + {%- endif %} + image '{{ dockerImage }}' + {%- if docker.parameters -%} + {%- set dockerParameters = docker.parameters | join(' ') -%} + {%- endif %} + {%- if docker.volumes -%} + {%- set dockerParameters = '-v ' + (docker.volumes | join(' ')) + ' ' + dockerParameters -%} + {%- endif %} + {%- if dockerParameters %} + args '{{ dockerParameters }}' + } + {%- endif %} + } +{%- else %} + agent any +{%- endif -%} + +{%- if needs_lifecycle_parameter %} + parameters { + string(name: 'current_lifecycle', defaultValue: 'working_time', description: 'The current stage') + } +{%- endif %} + +{%- if environment %} + environment { + {%- for env_var, env_value in environment.items() %} + {{ env_var }} = '{{ env_value }}' + {%- endfor %} + } +{%- endif %} + +stages { + {%- for name, repository in repositories.items() %} + stage('{{ name }}') { + steps { + echo '🖨️ cloning aeolus' + dir('{{ repository.path }}') { + checkout([$class: 'GitSCM', + branches: [[name: '{{ repository.branch }}']], + doGenerateSubmoduleConfigurations: false, + extensions: [], + submoduleCfg: [], + userRemoteConfigs: [[ + {%- if metadata.gitCredentials %} + credentialsId: '{{ metadata.gitCredentials }}', + {%- endif %} + name: '{{ name }}', + {%- set url = '${%s}' % repo_metadata[name]["url"] %} + url: '{{ url }}' + ]] + ]) + } + } + } + {%- endfor %} + {%- for step in steps %} + stage('{{ step.name }}') { + {%- if step.docker %} + agent { + docker { + {%- set dockerImage = "%s" % step.docker.image -%} + {%- if docker.tag -%} + {% set dockerImage = "{}:{}".format(dockerImage, step.docker.tag) %} + {%- endif %} + image '{{ dockerImage }}' + {%- if docker.parameters %} + args '{{ docker.parameters | join(' ') }}' + {%- endif %} + } + } + {%- if step.excludeDuring %} + when { + anyOf { + {%- for exclude in step.excludeDuring %} + expression { params.current_lifecycle != '{{ exclude | replace('Lifecycle.', '') }}' } + {%- endfor %} + } + } + {%- endif %} + {%- if step.environment %} + environment { + {%- for env_var, env_value in step.environment.items %} + {{ env_var }} = '{{ env_value }}' + {%- endfor %} + } + {%- endif %} + {%- endif %} + steps { + echo '⚙️ executing {{ step.name }}' + {%- if step.workdir %} + dir('{{ step.workdir }}') { + sh ''' + {{ step.script }} + ''' + } + {%- else %} + sh ''' + {{ step.script }} + ''' + {%- endif %} + } + } + {%- endfor %} + } + + {%- if has_always_actions %} + post { + {%- for action in always_steps %} + always { + echo '⚙️ executing {{ action.name }}' + {%- if action.workdir %} + dir('{{ action.workdir }}') { + sh ''' + {{ action.script }} + ''' + } + {%- else %} + sh ''' + {{ action.script }} + ''' + {%- endif %} + } + {%- endfor %} + {%- if metadata.resultHook %} + + {%- endif %} + {%- if has_results %} + always { + echo '📊 publishing results' + {%- for workdir, entries in results.items() %} + dir('{{ workdir }}') { + {%- for result in entries %} + {%- if result.type == 'junit' %} + junit './{{ result.path }}' + {%- else %} + archiveArtifacts: './{{result.path}}', fingerprint: true, allowEmptyArchive: true, + {%- endif %} + {%- endfor %} + } + {%- endfor %} + } + {%- if metadata.resultHook %} + always { + {%- if metadata.resultHookCredentials %} + sendTestResults credentialsId: '{{ metadata.resultHookCredentials }}', notificationUrl: '{{ metadata.resultHook }}' + {%- else %} + sendTestResults notificationUrl: '{{ metadata.resultHook }}' + {%- endif %} + } + {%- endif %} + {%- endif %} + } + {%- endif %} +} From a4f9f5a3a93ff973b2ff1a1f816dd7c9c0de7de1 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 17:24:34 +0100 Subject: [PATCH 3/7] fix tests --- cli/generators/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/generators/cli.py b/cli/generators/cli.py index e6cc7915..e96cac4d 100644 --- a/cli/generators/cli.py +++ b/cli/generators/cli.py @@ -202,11 +202,11 @@ def generate(self) -> str: we don't want to handle the credentials in the CI system. :return: bash script """ - self.result = [] + self.result = "" utils.replace_environment_variables_in_windfile(environment=self.environment, windfile=self.windfile) self.add_repository_urls_to_environment() for step in self.windfile.actions: if isinstance(step.root, ScriptAction): self.handle_step(name=step.root.name, step=step.root, call=not step.root.runAlways) - self.result.append(self.generate_using_jinja2()) + self.result = self.generate_using_jinja2() return super().generate() From 560762cb40a10a93265d943860cb47de2ad156d7 Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 17:28:29 +0100 Subject: [PATCH 4/7] linting --- cli/generators/bamboo.py | 8 ++++---- cli/generators/cli.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/generators/bamboo.py b/cli/generators/bamboo.py index 7c22bf56..a7a4aa2f 100644 --- a/cli/generators/bamboo.py +++ b/cli/generators/bamboo.py @@ -16,6 +16,10 @@ from typing import List, Any, Optional import requests +from docker.client import DockerClient # type: ignore +from docker.errors import DockerException # type: ignore +from docker.models.containers import Container # type: ignore +from docker.types.daemon import CancellableStream # type: ignore import cli_utils from classes.generated.definitions import Target @@ -25,10 +29,6 @@ from classes.pass_metadata import PassMetadata from cli_utils import logger, utils from generators.base import BaseGenerator -from docker.client import DockerClient # type: ignore -from docker.errors import DockerException # type: ignore -from docker.models.containers import Container # type: ignore -from docker.types.daemon import CancellableStream # type: ignore def docker_available() -> bool: diff --git a/cli/generators/cli.py b/cli/generators/cli.py index e96cac4d..7226fadf 100644 --- a/cli/generators/cli.py +++ b/cli/generators/cli.py @@ -6,6 +6,9 @@ from typing import List, Optional from jinja2 import Environment, FileSystemLoader, Template +from docker.client import DockerClient # type: ignore +from docker.models.containers import Container # type: ignore +from docker.types.daemon import CancellableStream # type: ignore from classes.generated.definitions import ScriptAction, Repository, Target from classes.generated.windfile import WindFile @@ -14,9 +17,6 @@ from classes.pass_metadata import PassMetadata from cli_utils import logger, utils from generators.base import BaseGenerator -from docker.client import DockerClient # type: ignore -from docker.models.containers import Container # type: ignore -from docker.types.daemon import CancellableStream # type: ignore class CliGenerator(BaseGenerator): From 9b5ef20b9bbff9c79efad70a3c386ed0f89b749c Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 17:31:44 +0100 Subject: [PATCH 5/7] linting --- cli/generators/cli.py | 3 ++- cli/generators/jenkins.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/generators/cli.py b/cli/generators/cli.py index 7226fadf..149519f6 100644 --- a/cli/generators/cli.py +++ b/cli/generators/cli.py @@ -3,6 +3,7 @@ import re import subprocess import tempfile +import typing from typing import List, Optional from jinja2 import Environment, FileSystemLoader, Template @@ -176,7 +177,7 @@ def generate_using_jinja2(self) -> str: self.template = env.get_template("cli.sh.j2") # Prepare your data - data = { + data: dict[str, typing.Any] = { "has_multiple_steps": self.has_multiple_steps, "initial_directory_variable": self.initial_directory_variable, "environment": self.windfile.environment.root.root if self.windfile.environment else {}, diff --git a/cli/generators/jenkins.py b/cli/generators/jenkins.py index b33356dd..63699df1 100644 --- a/cli/generators/jenkins.py +++ b/cli/generators/jenkins.py @@ -4,6 +4,7 @@ The generated pipeline is a scripted pipeline. """ import os +import typing from typing import Optional, List from xml.dom.minidom import Document, parseString, Element @@ -114,11 +115,11 @@ def generate_using_jinja2(self) -> str: env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))) self.template = env.get_template("Jenkinsfile.j2") - data = { + data: dict[str, typing.Any] = { "docker": self.windfile.metadata.docker, - "environment": self.windfile.environment.root.root if self.windfile.environment else {}, + "environment": self.windfile.environment.root.root if self.windfile.environment else None, "needs_lifecycle_parameter": self.needs_lifecycle_parameter, - "repositories": self.windfile.repositories if self.windfile.repositories else {}, + "repositories": self.windfile.repositories if self.windfile.repositories else None, "has_always_actions": self.has_always_actions(), "steps": [action.root for action in self.windfile.actions if not action.root.runAlways], "always_steps": [action.root for action in self.windfile.actions if action.root.runAlways], From c7d9a15d403e4288bb50deee08a71ac978aac2dd Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 17:33:26 +0100 Subject: [PATCH 6/7] linting --- cli/templates/Jenkinsfile.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/templates/Jenkinsfile.j2 b/cli/templates/Jenkinsfile.j2 index 9a92ee1a..376ffe9a 100644 --- a/cli/templates/Jenkinsfile.j2 +++ b/cli/templates/Jenkinsfile.j2 @@ -37,6 +37,7 @@ pipeline { {%- endif %} stages { + {%- if repositories %} {%- for name, repository in repositories.items() %} stage('{{ name }}') { steps { @@ -59,7 +60,8 @@ stages { } } } - {%- endfor %} + {%- endfor -%} + {%- endif %} {%- for step in steps %} stage('{{ step.name }}') { {%- if step.docker %} From 5f52a55b3d4ceba8bfdadcf8601d4ca41c5475ed Mon Sep 17 00:00:00 2001 From: Andreas Resch Date: Sun, 31 Dec 2023 23:25:42 +0100 Subject: [PATCH 7/7] fix template --- cli/templates/Jenkinsfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/templates/Jenkinsfile.j2 b/cli/templates/Jenkinsfile.j2 index 376ffe9a..3b3f6d11 100644 --- a/cli/templates/Jenkinsfile.j2 +++ b/cli/templates/Jenkinsfile.j2 @@ -36,7 +36,7 @@ pipeline { } {%- endif %} -stages { + stages { {%- if repositories %} {%- for name, repository in repositories.items() %} stage('{{ name }}') {