From ff1c7dee24a94268320f5e3c27c5328dc52675c6 Mon Sep 17 00:00:00 2001 From: Diego Quintana Date: Tue, 5 Sep 2023 09:52:34 +0200 Subject: [PATCH] chore: remove git pulls at dags and update docker images at DockerOperator calls (et/somenergia-jardiner!8) * remove legacy git pull, change to harbor, change image * remove git code updates from alarms --- Makefile | 3 ++ containers/app/Dockerfile | 10 +++-- dags/jardineria_dag.py | 23 ++-------- dags/plant_production_datasets_dag.py | 43 ++----------------- ...rter_interinverter_relative_temperature.py | 25 ++--------- ...plantmonitor_alert_inverter_temperature.py | 24 ++--------- .../plantmonitor_alert_inverter_zero_power.py | 24 ++--------- dags/plantmonitor_alert_meter_no_readings.py | 23 ++-------- dags/plantmonitor_alert_meter_zero_energy.py | 23 ++-------- 9 files changed, 31 insertions(+), 167 deletions(-) diff --git a/Makefile b/Makefile index 70d4bec6..fdc8226b 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ sh: ## run a shell in the container app.build: ## build image using docker build @docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) build app --progress=plain +app.push: ## push image using docker push + @docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) push app + app_dev.build: ## build image for development using docker build @docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) build app-dev --progress=plain diff --git a/containers/app/Dockerfile b/containers/app/Dockerfile index aadaaf6b..014cacef 100644 --- a/containers/app/Dockerfile +++ b/containers/app/Dockerfile @@ -120,7 +120,7 @@ FROM python:${PYTHON_VERSION}-slim AS app # refresh global arguments ARG WORKDIR ARG USERNAME -ARG USER_UID=1010 +ARG USER_UID ARG USER_GID ARG DBT_PROJECT_DIR_NAME @@ -139,12 +139,17 @@ ENV PATH=/opt/pipx/bin:${WORKDIR}/.venv/bin:$PATH \ RUN groupadd --gid $USER_GID "${USERNAME}" \ && useradd --uid $USER_UID --gid $USER_GID -m "${USERNAME}" +RUN apt-get update \ + && apt-get install -y vim + # ------------------------------- app specific ------------------------------- # WORKDIR ${WORKDIR} +# RUN chown -R ${USER_UID}:${USER_GID} ${WORKDIR} +# copy /app/.venv from app-pre COPY --from=app-pre --chown=${USER_UID}:${USER_GID} ${WORKDIR} ${WORKDIR} USER ${USERNAME} @@ -202,7 +207,6 @@ COPY --chown=${USER_UID}:${USER_GID} ./${DBT_PROJECT_DIR_NAME}/dbt_project.yml . RUN cd ${WORKDIR}/${DBT_PROJECT_DIR_NAME}/ && dbt deps --profiles-dir /home/${USERNAME}/.dbt - USER ${USERNAME} # ---------------------------------------------------------------------------- # @@ -242,7 +246,7 @@ RUN groupadd --gid ${USER_GID} "${USERNAME}" \ # --------------------------- install dependencies --------------------------- # RUN apt-get update \ - && apt-get install -y git sudo iputils-ping wget + && apt-get install -y git vim iputils-ping wget # ------------------------------- app specific ------------------------------- # diff --git a/dags/jardineria_dag.py b/dags/jardineria_dag.py index 5cd03df7..71ed3714 100644 --- a/dags/jardineria_dag.py +++ b/dags/jardineria_dag.py @@ -5,10 +5,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -59,19 +55,12 @@ def get_random_moll(): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, task_name="jardineria", repo_name=repo_name - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - notify_alarms_task = DockerOperator( api_version="auto", task_id="jardineria", - docker_conn_id="somenergia_registry", - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command=( @@ -89,9 +78,3 @@ def get_random_moll(): trigger_rule="none_failed", ) - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> notify_alarms_task - task_branch_pull_ssh >> task_update_image - task_update_image >> notify_alarms_task diff --git a/dags/plant_production_datasets_dag.py b/dags/plant_production_datasets_dag.py index 2f34695f..abf37082 100644 --- a/dags/plant_production_datasets_dag.py +++ b/dags/plant_production_datasets_dag.py @@ -6,10 +6,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -68,13 +64,6 @@ def dbapi_to_dict(dbapi: str): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, task_name="dbt_deps_task", repo_name=repo_name - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - dbapi = Variable.get("plantmonitor_db") dbapi_dict = dbapi_to_dict(dbapi) @@ -87,31 +76,13 @@ def dbapi_to_dict(dbapi: str): "DBNAME": dbapi_dict["database"], } - dbt_deps_task = DockerOperator( - api_version="auto", - task_id="dbt_deps_task", - docker_conn_id="somenergia_registry", - environment=environment, - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name - ), - working_dir=f"/repos/{repo_name}/dbt_jardiner", - command="dbt deps --profiles-dir config", - docker_url=sampled_moll, - mounts=[mount_nfs], - mount_tmp_dir=False, - auto_remove=True, - retrieve_output=True, - trigger_rule="none_failed", - ) - dbt_transformation_task = DockerOperator( api_version="auto", task_id="dbt_transformation_task", - docker_conn_id="somenergia_registry", environment=environment, - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}/dbt_jardiner", command="dbt run --profiles-dir config --target prod --select plant_production_hourly+", @@ -122,11 +93,3 @@ def dbapi_to_dict(dbapi: str): retrieve_output=True, trigger_rule="none_failed", ) - - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> dbt_deps_task - task_branch_pull_ssh >> task_update_image - task_update_image >> dbt_deps_task - dbt_deps_task >> dbt_transformation_task diff --git a/dags/plantmonitor_alert_inverter_interinverter_relative_temperature.py b/dags/plantmonitor_alert_inverter_interinverter_relative_temperature.py index d61d823c..5abf2c04 100644 --- a/dags/plantmonitor_alert_inverter_interinverter_relative_temperature.py +++ b/dags/plantmonitor_alert_inverter_interinverter_relative_temperature.py @@ -5,10 +5,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -59,21 +55,12 @@ def get_random_moll(): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, - task_name="alert_inverter_interinverter_relative_temperature", - repo_name=repo_name, - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - notify_alarms_task = DockerOperator( api_version="auto", task_id="alert_inverter_interinverter_relative_temperature", - docker_conn_id="somenergia_registry", - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\ @@ -86,9 +73,3 @@ def get_random_moll(): trigger_rule="none_failed", ) - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> notify_alarms_task - task_branch_pull_ssh >> task_update_image - task_update_image >> notify_alarms_task diff --git a/dags/plantmonitor_alert_inverter_temperature.py b/dags/plantmonitor_alert_inverter_temperature.py index c876525c..e4ab4458 100644 --- a/dags/plantmonitor_alert_inverter_temperature.py +++ b/dags/plantmonitor_alert_inverter_temperature.py @@ -5,10 +5,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -59,19 +55,12 @@ def get_random_moll(): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, task_name="alert_inverter_temperature", repo_name=repo_name - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - notify_alarms_task = DockerOperator( api_version="auto", task_id="alert_inverter_temperature", - docker_conn_id="somenergia_registry", - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\ @@ -83,10 +72,3 @@ def get_random_moll(): retrieve_output=True, trigger_rule="none_failed", ) - - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> notify_alarms_task - task_branch_pull_ssh >> task_update_image - task_update_image >> notify_alarms_task diff --git a/dags/plantmonitor_alert_inverter_zero_power.py b/dags/plantmonitor_alert_inverter_zero_power.py index 3557ad98..17193011 100644 --- a/dags/plantmonitor_alert_inverter_zero_power.py +++ b/dags/plantmonitor_alert_inverter_zero_power.py @@ -5,10 +5,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -59,19 +55,12 @@ def get_random_moll(): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, task_name="alert_inverter_zero_power", repo_name=repo_name - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - notify_alarms_task = DockerOperator( api_version="auto", task_id="alert_inverter_zero_power", - docker_conn_id="somenergia_registry", - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\ @@ -83,10 +72,3 @@ def get_random_moll(): retrieve_output=True, trigger_rule="none_failed", ) - - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> notify_alarms_task - task_branch_pull_ssh >> task_update_image - task_update_image >> notify_alarms_task diff --git a/dags/plantmonitor_alert_meter_no_readings.py b/dags/plantmonitor_alert_meter_no_readings.py index 9a2c167e..d782efd2 100644 --- a/dags/plantmonitor_alert_meter_no_readings.py +++ b/dags/plantmonitor_alert_meter_no_readings.py @@ -5,10 +5,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -59,19 +55,12 @@ def get_random_moll(): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, task_name="alert_meter_no_readings", repo_name=repo_name - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - notify_alarms_task = DockerOperator( api_version="auto", task_id="alert_meter_no_readings", - docker_conn_id="somenergia_registry", - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\ @@ -84,9 +73,3 @@ def get_random_moll(): trigger_rule="none_failed", ) - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> notify_alarms_task - task_branch_pull_ssh >> task_update_image - task_update_image >> notify_alarms_task diff --git a/dags/plantmonitor_alert_meter_zero_energy.py b/dags/plantmonitor_alert_meter_zero_energy.py index b4e5f0d3..3d6814d8 100644 --- a/dags/plantmonitor_alert_meter_zero_energy.py +++ b/dags/plantmonitor_alert_meter_zero_energy.py @@ -5,10 +5,6 @@ from airflow.models import Variable from airflow.providers.docker.operators.docker import DockerOperator from docker.types import DriverConfig, Mount -from util_tasks.t_branch_pull_ssh import build_branch_pull_ssh_task -from util_tasks.t_check_repo import build_check_repo_task -from util_tasks.t_git_clone_ssh import build_git_clone_ssh_task -from util_tasks.t_update_docker_image import build_update_image_task my_email = Variable.get("fail_email") addr = Variable.get("repo_server_url") @@ -59,19 +55,12 @@ def get_random_moll(): sampled_moll = get_random_moll() - task_check_repo = build_check_repo_task(dag=dag, repo_name=repo_name) - task_git_clone = build_git_clone_ssh_task(dag=dag, repo_name=repo_name) - task_branch_pull_ssh = build_branch_pull_ssh_task( - dag=dag, task_name="alert_meter_zero_energy", repo_name=repo_name - ) - task_update_image = build_update_image_task(dag=dag, repo_name=repo_name) - notify_alarms_task = DockerOperator( api_version="auto", task_id="alert_meter_zero_energy", - docker_conn_id="somenergia_registry", - image="{}/{}-requirements:latest".format( - "{{ conn.somenergia_registry.host }}", repo_name + docker_conn_id="somenergia_harbor_dades_registry", + image="{}/{}-app:latest".format( + "{{ conn.somenergia_harbor_dades_registry.host }}", repo_name ), working_dir=f"/repos/{repo_name}", command='python3 -m scripts.notify_alert "{{ var.value.plantmonitor_db }}"\ @@ -84,9 +73,3 @@ def get_random_moll(): trigger_rule="none_failed", ) - task_check_repo >> task_git_clone - task_check_repo >> task_branch_pull_ssh - task_git_clone >> task_update_image - task_branch_pull_ssh >> notify_alarms_task - task_branch_pull_ssh >> task_update_image - task_update_image >> notify_alarms_task