ci: update dbt test should run in pre and prod depending on context (…

…et/somenergia-jardiner!33)

* chore: lint test not passing before fixing it. Related to #76

* fix: missing dbt deps in task

* change: avoid merging arrays in yaml

* fix: extending arrays properly

extending arrays in yaml is clunky, see yaml/yaml#35

* new: add dbt build instead of dbt test, parametrized by context

during merge requests, use target pre
on main, use target prod

* new: make dbt run when there are changes in dbt models

.gitlab-ci.yml

@@ -67,6 +67,13 @@ pre-commit:
   - jardiner/*
   - .gitlab-ci.yml
 
+
+.dbt-build-changes-paths: &dbt-build-changes-paths
+  - dbt_jardiner/**/*.sql
+  - dbt_jardiner/**/*.yaml
+  - .gitlab-ci.yml
+
+
 .docker-login-before-script:
   &docker-login-before-script ## login to the Docker registry, needed to push and pull from private registries
   - echo "Logging in to Docker Hub"
@@ -116,10 +123,10 @@ build-app-image-pre-release:
   tags:
     - somenergia-et
 
-dbt-test-prod:
+dbt-build:
   stage: test
   variables:
-    DBT_TARGET_NAME: prod
+    DBT_TARGET_NAME: pre
     DBHOST: $SOM_JARDINER_DB_HOST
     DBPORT: $SOM_JARDINER_DB_PORT
     DBUSER: $SOM_JARDINER_DB_USER
@@ -129,20 +136,24 @@ dbt-test-prod:
   script:
     - cd ${CI_PROJECT_DIR}/${DBT_PROJECT_DIR_NAME}
     - dbt deps --target ${DBT_TARGET_NAME}
-    - dbt test --target ${DBT_TARGET_NAME}
+    - dbt build --target ${DBT_TARGET_NAME} --store-failures
   tags:
     - somenergia-et
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
       when: always
+      variables:
+        DBT_TARGET_NAME: prod
       changes:
-        paths: *app-build-changes-paths
+        paths: *dbt-build-changes-paths
       allow_failure: false
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always
+      variables:
+        DBT_TARGET_NAME: pre
       changes:
         compare_to: "refs/heads/main"
-        paths: *app-build-changes-paths
+        paths: *dbt-build-changes-paths
       allow_failure: false
   artifacts:
     paths:

dbt_jardiner/tests/test_repeated_device_uuid_errors.sql

@@ -1,15 +1,19 @@
 -- tests pairs device_uuid, plant, device, device_type, device_parent as unique
-with tb as (
-    select distinct device_uuid, plant, device, device_type, device_parent
-    from {{ ref('raw_gestio_actius__signal_denormalized') }}
-    order by plant, device),
+with
+    tb as (
+        select distinct device_uuid, plant, device, device_type, device_parent
+        from {{ ref("raw_gestio_actius__signal_denormalized") }}
+        order by plant, device
+    ),
+
 tb2 as (
-    select tb.device_uuid, count(*) as duplicates
-    from tb
-    group by device_uuid
-    having count(*)> 1
+      select tb.device_uuid, count(*) as duplicates
+      from tb
+      group by tb.device_uuid
+      having count(*) > 1
     )
-select r.*, duplicates
-from {{ ref('raw_gestio_actius__signal_denormalized') }} r
-left join tb2 on tb2.device_uuid = r.device_uuid
-where duplicates is not null
+
+select r.*, tb2.duplicates
+from {{ ref("raw_gestio_actius__signal_denormalized") }} as r
+left join tb2 on r.device_uuid = tb2.device_uuid
+where tb2.duplicates is not null