new: add mhcil transformations in dbt to datamarts (et/somenergia-jar…

…diner!38)

Merge branch 'feature/mhcil-transformations-dbt-to-dms' into 'main'

dbt_jardiner/dbt_project.yml

@@ -40,16 +40,25 @@ models:
           node_color: "#d17f4c"
         dset:
           +tags: dset
+        simel:
+          +grants:
+            select: ['mercat_electric']
       intermediate:
         +docs:
           node_color: "#d05808"
         dset:
           +tags: dset
+        simel:
+          +grants:
+            select: ['mercat_electric']
       marts:
         +docs:
           node_color: "goldenrod"
         dset:
           +tags: dset
+        simel:
+          +grants:
+            select: ['mercat_electric']
       observability:
         +docs:
           node_color: "#c3c557"

dbt_jardiner/models/jardiner/intermediate/simel/int_simel_mhcil__pen_best_ranked_incremental.sql

@@ -0,0 +1,47 @@
+{{
+    config(
+        enabled=false,
+        materialized="incremental",
+        incremental_strategy="append",
+        unique_key="surrogate_key",
+    )
+}}
+
+
+{% if not is_incremental() %}
+
+  select {{ dbt_utils.star(from=ref("int_simel_mhcil__pen_best_ranked_view")) }}
+  from {{ ref("int_simel_mhcil__pen_best_ranked_view") }}
+
+{% else %}
+
+  with existing as (
+    select
+      surrogate_key,
+      total_rank,
+      ranked_at
+    from {{ this }}
+  ),
+
+  incoming as (
+    select {{ dbt_utils.star(from=ref("int_simel_mhcil__pen_best_ranked_view")) }}
+    from {{ ref("int_simel_mhcil__pen_best_ranked_view") }}
+  ),
+
+  new_rankings as (
+    select
+      incoming.surrogate_key,
+      incoming.total_rank,
+      incoming.ranked_at
+    from existing
+      left join incoming
+        on existing.surrogate_key = incoming.surrogate_key
+    where
+      existing.total_rank < incoming.total_rank
+      and existing.ranked_at < incoming.ranked_at
+  )
+
+  select *
+  from new_rankings
+
+{% endif %}

dbt_jardiner/models/jardiner/intermediate/simel/int_simel_mhcil__pen_best_ranked_view.sql

@@ -0,0 +1,28 @@
+{{ config(materialized="view") }}
+
+with ranked as (
+  select
+    *,
+    extract(epoch from start_at) + file_version + firmeza_rank + maturity_rank as total_rank
+  from {{ ref("int_simel_mhcil__pen_extended") }}
+),
+
+ordered as (
+  select
+    *,
+    row_number() over (
+      partition by cil, start_at
+      order by total_rank
+    ) as row_order
+  from ranked
+),
+
+best_ranked as (
+  select
+    *,
+    now() as ranked_at,
+    {{ dbt_utils.generate_surrogate_key(["cil", "start_at"]) }} as surrogate_key
+  from ordered where row_order = 1
+)
+
+select * from best_ranked

dbt_jardiner/models/jardiner/intermediate/simel/int_simel_mhcil__pen_extended.sql

@@ -0,0 +1,59 @@
+{{ config(materialized="view") }}
+
+
+with timestamp_crafted as (
+  select
+    *,
+    make_timestamp(
+      cast("year" as int), cast("month" as int), cast("day" as int), cast("hour" as int), 0, 0
+    ) as measurement_timestamp
+  from {{ ref("raw_airbyte_simel__mhcil_pen") }}
+),
+
+timestamp_localized as (
+  select
+    *,
+    case
+      when is_summer
+        then measurement_timestamp at time zone 'CEST'
+      else measurement_timestamp at time zone 'CET'
+    end as measurement_timestamptz
+  from timestamp_crafted
+),
+
+extended_mhcil as (
+  select
+    cil,
+    measurement_timestamptz as start_at,
+    hour,
+    is_summer,
+    energy_kwh,
+    reactive_energy_2_kvarh,
+    reactive_energy_3_kvarh,
+    measurement_type,
+    _ab_source_file_url as file_name,
+    _airbyte_normalized_at as ingested_at,
+    cast(_ab_source_file_last_modified as timestamptz) as file_last_modified_at,
+    cast(split_part(_ab_source_file_url, '.', 2) as int) as file_version,
+    cast(split_part(_ab_source_file_url, '_', 1) as varchar(5)) as file_type,  -- fecha a la que corresponden los datos
+    cast(split_part(_ab_source_file_url, '_', 2) as varchar(2)) as release_period,  -- tipo de fichero según simel
+    cast(split_part(_ab_source_file_url, '_', 3) as varchar(4)) as member_code,  -- periodo de publicación
+    cast(split_part(_ab_source_file_url, '_', 4) as varchar(2)) as file_receiver_type,  -- código de participante
+    to_date(right(split_part(_ab_source_file_url, '.', 1), 8), 'YYYYMMDD') as file_date  -- tipo de receptor de fichero
+  from timestamp_localized
+),
+
+ranked as (
+  select
+    exmhcil.*,
+    cast(firmeza_ranking.rank as int) as firmeza_rank,
+    cast(maturity_ranking.rank as int) as maturity_rank
+  from extended_mhcil as exmhcil
+    left join {{ ref("seed_simel_mhcil_rankings__firmeza") }} as firmeza_ranking
+      on exmhcil.measurement_type = firmeza_ranking.name
+    left join {{ ref("seed_simel_mhcil_rankings__maturity") }} as maturity_ranking
+      on exmhcil.release_period = maturity_ranking.name
+)
+
+select *
+from ranked

dbt_jardiner/models/jardiner/intermediate/simel/_simel__models.yaml → dbt_jardiner/models/jardiner/raw/simel/_simel__models.yaml

@@ -2,7 +2,7 @@
 version: 2
 
 models:
-    - name: raw_airbyte_simel__simel_mhcil_nas
+    - name: raw_airbyte_simel__mhcil_pen
       columns:
           - name: cil
             description: A - Código CIL (25 characters)

dbt_jardiner/models/jardiner/raw/simel/_simel__sources.yaml

@@ -0,0 +1,10 @@
+---
+version: 2
+
+sources:
+  - name: airbyte_simel
+    schema: airbyte_imported
+    tables:
+      - name: simel_mhcil_nas
+        description: Dades des de SIMEL sincronitzades des de minio.somenergia.coop. Només conté dades de sistema peninsular (PEN)
+

dbt_jardiner/seeds/jardiner/_jardiner__seeds.yaml

@@ -0,0 +1,9 @@
+version: 2
+
+seeds:
+  - name: seed_simel_file_rankings__firmeza
+    description: |
+      Seed que genera un ranking dels fitxers segons la seva firmeza i el document SIMEL_Ficheros_Intercambio_Informacion_v40_Oct2022.pdf
+  - name: seed_simel_file_rankings__maturity
+    description: |
+      Seed que genera un ranking dels fitxers segons la seva maduresa i el document SIMEL_Ficheros_Intercambio_Informacion_v40_Oct2022.pdf

dbt_jardiner/seeds/jardiner/seed_simel_mhcil_rankings__firmeza.csv

@@ -0,0 +1,5 @@
+id,name,rank,description
+1,R,4,Real con reparto por potencia
+2,L,3,Estimación con reparto por potencia
+3,E,2,Real con reparto por energía individualizada de CIL
+4,M,1,Estimación con reparto por energía individualizada de CI

dbt_jardiner/seeds/jardiner/seed_simel_mhcil_rankings__maturity.csv

@@ -0,0 +1,6 @@
+id,name,rank,description
+1,HD,1,valores de cierre diario
+2,H2,2,valores de cierre de mes m-1
+3,H3,3,valores de cierre de mes m-3
+4,HP,4,valores de cierre provisional
+5,HC,5,valores de cierre definitivo

dbt_jardiner/tests/dset/test_dset_signals_receiver_last_hour.sql

@@ -1,4 +1,4 @@
-{{ config(error_if=">500") }}
+{{ config(error_if=">1000") }}
 {# error limit is set on half the number of signal uuids available #}
 
 with