chore: netejar codi legacy de dags i dbt (et/somenergia-jardiner!142)

* fix: update dbt build selector

* fix: update dbt build selector

* fix: update selector from type to config.materialized

oops

* ci: update dbt build with commented params

* ci: use dbt target pre instead of test

reverts on previou change, it is breaking CI

* ci: temporary disable dbtstates during merge requests

* fix: add sqlfluff-lint to skipped pre-commit tasks

* ci: fix typo

* ci: fix typo

* ci: allow running dbt build without states

* Revert "ci: let pre-commit lint sql files during jobs"

This reverts commit f79db3e1ddb34f1c20fbf915e2f45703989c8d33.

* ci: let pre-commit lint sql files during jobs

* chore: rename cicd dbt target from pre to test

* chore: delete deprecated alert notify python script

* chore: remove legacy alarms and notify

* add elementary on dag dbt runs

* chore: standardize dag tags

.gitlab-ci.yml

@@ -21,6 +21,7 @@ variables:
   SOMENERGIA_REGISTRY: $SOM_HARBOR_DADES_URL
   DBT_PROJECT_DIR_NAME: dbt_jardiner
   SOM_PROJECT_NAME: somenergia-jardiner
+  DBT_SKIP_STATE: "false"
   IMAGE_NAME_PREFIX: $SOM_HARBOR_DADES_URL/$CI_PROJECT_NAME
   IMAGE_NAME_APP: $IMAGE_NAME_PREFIX-app
   IMAGE_NAME_APP_PRE_RELEASE: $IMAGE_NAME_APP:pre-release
@@ -135,7 +136,8 @@ dbt-build:
     DBUSER: $SOM_JARDINER_DB_USER
     DBPASSWORD: $SOM_JARDINER_DB_PASSWORD
     DBNAME: $SOM_JARDINER_DB_DBNAME
-    DBT_MODELS_SELECTOR: "--models state:modified+"
+    DBT_MODELS_SELECTOR: "--models state:modified+ --defer"
+    DBT_STATE: "--state state/prod"
     DBT_MANIFEST_ARTIFACT_URL: "https://$CI_SERVER_HOST/api/v4/projects/$CI_PROJECT_ID/jobs/artifacts/${CI_DEFAULT_BRANCH}/download?job=pages&job_token=$CI_JOB_TOKEN"
   image: ${SOM_HARBOR_DADES_URL}/${SOM_PROJECT_NAME}-dbt-docs:latest
   script:
@@ -146,31 +148,27 @@ dbt-build:
     - curl --location --output /tmp/artifacts.zip ${DBT_MANIFEST_ARTIFACT_URL}
     - unzip -o /tmp/artifacts.zip -d /tmp/artifacts
     - cp /tmp/artifacts/public/dbt_docs/manifest.json ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR_NAME}/state/prod/manifest.json
-    - dbt build --target ${DBT_TARGET_NAME} --store-failures --threads 4 ${DBT_MODELS_SELECTOR} --state state/prod
+    - dbt build --target ${DBT_TARGET_NAME} --store-failures --threads 4 ${DBT_MODELS_SELECTOR} ${DBT_STATE}
   tags:
     - somenergia-et
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
       variables:
         DBT_TARGET_NAME: prod
-        DBT_MODELS_SELECTOR: "--models state:modified+"
       changes:
         paths: *dbt-build-changes-paths
       allow_failure: false
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
       variables:
-        DBT_MODELS_SELECTOR: "--models state:modified+ tag:dset_responses_fresh"
+        DBT_MODELS_SELECTOR: "--select tag:dset_responses_fresh state:modified+ +state:modified+,config.materialized:incremental +state:modified+,config.materialized:table --defer"
         DBT_TARGET_NAME: pre
         DBT_FAIL_FAST: "True"
       changes:
         compare_to: "refs/heads/main"
         paths: *dbt-build-changes-paths
       allow_failure: false
-  artifacts:
-    paths:
-      - ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR_NAME}/target
 
 release-app-image:
   stage: release

dags/dag_jardiner_dbt_snapshot.py

@@ -71,7 +71,7 @@ def dbapi_to_dict(dbapi: str) -> dict:
     start_date=datetime(2024, 1, 28),
     schedule_interval="30 22 * * *",
     catchup=False,
-    tags=["jardiner", "dbt", "snapshot"],
+    tags=["project:jardiner", "dbt", "dbt-snapshot"],
     max_active_runs=1,
     default_args=args,
     doc_md=__doc__,

dags/dag_simel_mhcil_nas_mirror_from_minio.py

@@ -49,13 +49,13 @@ def get_random_moll():
     schedule_interval="@daily",
     catchup=False,
     tags=[
-        "curve:MHCIL",
-        "scope:Jardiner",
-        "scope:Mercat",
-        "ingesta:historic",
-        "ingesta:minio",
-        "source:SIMEL",
-        "source:NAS",
+        "ingesta",
+        "project:jardiner",
+        "project:mercat",
+        "source:minio",
+        "source:mhcil",
+        "source:simel",
+        "source:nas",
     ],
     default_args=args,
 ) as dag:

dags/dbt_test_quality_dset.py

@@ -69,7 +69,13 @@ def dbapi_to_dict(dbapi: str):
     start_date=datetime(2023, 12, 1, 0, 0, 0),
     schedule_interval="0 3 * * *",
     catchup=False,
-    tags=["Plantmonitor", "Jardiner", "test", "dbt", "data_quality"],
+    tags=[
+        "project:plantmonitor",
+        "project:jardiner",
+        "dbt-test",
+        "dbt",
+        "data-quality",
+    ],
     default_args=args,
     doc_md=__doc__,
 ) as dag:

dags/dset_materialize_dag.py

@@ -67,7 +67,13 @@ def dbapi_to_dict(dbapi: str):
     start_date=datetime(2023, 1, 10),
     schedule_interval="3/10 * * * *",
     catchup=False,
-    tags=["scope:Plantmonitor", "scope:Jardiner", "dbt", "source:DSET"],
+    tags=[
+        "project:plantmonitor",
+        "project:jardiner",
+        "dbt",
+        "dbt-run",
+        "source:dset-api",
+    ],
     max_active_runs=1,
     default_args=args,
     doc_md=__doc__,

dags/generate_gitlab_ci_pages.py

@@ -29,7 +29,12 @@
     start_date=dt.datetime(2023, 9, 27),
     catchup=False,
     dagrun_timeout=dt.timedelta(minutes=10),
-    tags=["jardiner", "gitlab-ci", "pages"],
+    tags=[
+        "project:jardiner",
+        "gitlab-ci",
+        "gitlab-pages",
+        "experimental",
+    ],
     max_active_runs=1,
 ) as dag:
     TOKEN = Variable.get("GITLAB_CI_TOKEN_JARDINER_PAGES")