Revert "new: update seeds new plants (makassar, valteina and fontivso…

…lar) (et/somenergia-jardiner!26)"

This reverts commit b416d3a

.gitignore

@@ -1,11 +1,5 @@
 # handmade ignores
 
-# duckdb
-*.duckdb
-
-# dbt internals
-dbt_jardiner/pre/
-
 # tmux shenanigans
 tmux_jardiner
 ..lock
@@ -236,4 +230,4 @@ fabric.properties
 .idea/httpRequests
 
 # Android studio 3.1+ serialized cache file
-.idea/caches/build_file_checksums.ser
+.idea/caches/build_file_checksums.ser

.sqlfluff

@@ -1,11 +1,13 @@
 [sqlfluff]
-exclude_rules = layout.indent, layout.cte_bracket, layout.select_targets, LT05
+exclude_rules = layout.indent, layout.cte_bracket, layout.select_targets
 # set max_line_length to whatever you set in sqlfmt
 max_line_length = 120
+# don't check for line length, let sqlfmt do that
+exclude_rules = LT05
 
 [sqlfluff:rules]
 capitalisation_policy = lower
 extended_capitalisation_policy = lower
 
 [sqlfluff:rules:convention.terminator]
-multiline_newline = True
+multiline_newline = True

dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_long_hourly.sql

@@ -4,35 +4,22 @@
 We therefore set it to 0 between midnight and 3 since all plants are fotovoltaic #}
 {# TODO we should pass the inverter_energy to incremental instead of accumulative
 since this is ugly as fuck #}
-
-{#
-    This model is followed by a pivot which wants to distinguish between meter_exported_energy and inverter_exported_energy
-
-    case when the content of the google sheets is fragile. We should limit the amount of metrics
-    to what we need to uniformize at the dropdown of
-    [sheet](https://docs.google.com/spreadsheets/d/1ybUXREO8cMaLMlV4Kt2iYyoNg2msirnbTTiYL2PBY2M).
-
-#}
 with
     dset_key_metrics as (
         select date_trunc('hour', ts) as start_hour, plant, device_type, metric as split_metric, signal_value
         from {{ ref("int_dset_responses__values_incremental") }}
-        where metric in ('energia_activa_exportada', 'irradiancia')
+        where metric in ('inverter_energy', 'irradiance', 'exported_energy')
     )
 select
     start_hour,
     plant,
+    case when split_metric = 'irradiance' then 'irradiation' else split_metric end as metric,
     case
-        when split_metric = 'irradiancia' then 'irradiation'
-        when split_metric = 'energia_activa_exportada' then device_type || '_exported_energy'
-        else split_metric
-    end as metric,
-    case
-        when split_metric = 'energia_activa_exportada' and device_type = 'inverter'
+        when split_metric = 'inverter_energy' and device_type = 'inverter'
         then (extract(hour from start_hour) > 3)::integer * (max(signal_value) - min(signal_value))  {# we have random-ish resets before 3 #}
-        when split_metric = 'irradiancia' and device_type in ('sensor', 'module', 'inverter')
+        when split_metric = 'irradiance' and device_type in ('sensor', 'module', 'inverter')
         then avg(signal_value)
-        when split_metric = 'energia_activa_exportada' and device_type = 'meter'
+        when split_metric = 'exported_energy' and device_type = 'meter'
         then max(signal_value) - min(signal_value)
         else null
     end as metric_value

dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__with_signal_metadata.sql

@@ -6,8 +6,7 @@
 e.g. signal_value * (case when signal_unit = 'kwh' then 1000 else 1)#}
 
 SELECT
-    metadata.plant_uuid,
-	  metadata.plant,
+    metadata.plant,
     metadata.signal,
     metadata.metric,
     metadata.device,
@@ -28,5 +27,5 @@ SELECT
 	valors.queried_at,
     valors.ts,
     valors.signal_value as signal_value
-FROM {{ ref('raw_gestio_actius__signal_denormalized') }} AS metadata
-LEFT JOIN {{ ref('int_dset_responses__deduplicated') }} AS valors USING(signal_uuid)
+FROM {{ ref('seed_signals__with_devices') }} AS metadata
+LEFT JOIN {{ ref('int_dset_responses__deduplicated') }} AS valors USING(signal_uuid)

dbt_jardiner/models/jardiner/intermediate/dset/int_signal_device_relation__distinct_devices.sql

@@ -1,5 +1,5 @@
 {{ config(materialized="table") }}
 
-select distinct plant, plant_uuid, device, device_uuid, device_type, device_parent
-from {{ ref('raw_gestio_actius__signal_denormalized') }}
+select distinct plant, device, device_uuid, device_type, device_parent
+from {{ ref("seed_signals__with_devices") }}
 order by plant, device

dbt_jardiner/models/jardiner/intermediate/int_plants_overview_instant.sql

@@ -23,19 +23,19 @@ with pot_instantanea_planta as (
 )
 select
     p.plant_name as nom_planta,
-    -- p.municipality as municipi,
-    p.province as provincia,
-    p.technology as tecnologia,
-    p.peak_power_kw as potencia_pic_kw,
-    p.nominal_power_kw as potencia_nominal_kw,
+    --municipality as municipi,
+    province as provincia,
+    technology as tecnologia,
+    peak_power_kw as potencia_pic_kw,
+    nominal_power_kw as potencia_nominal_kw,
     i.ultim_registre_pot_instantanea,
     i.pot_instantantanea_planta_kw,
     ir.ts as ultim_registre_irradiacio,
     ir.signal_value as irradiacio,
     ppd.dia,
     ppd.energia_exportada_comptador_kwh,
     ppd.energia_esperada_solargis_kwh
-from {{ ref('raw_gestio_actius_plant_parameters') }} p
+from {{ ref('seed_plants__parameters') }} p
 left join pot_instantanea_planta i
     on i.plant = p.plant_name
 left join plant_production_daily_previous_day ppd on ppd.nom_planta = p.plant_name

dbt_jardiner/models/jardiner/intermediate/obt/obt_hourly_incremental.sql

@@ -5,18 +5,17 @@
 #}
 
 with plants as (
-    select distinct plant, plant_uuid from {{ ref('int_signal_device_relation__distinct_devices') }}
+    select distinct plant from {{ ref('int_signal_device_relation__distinct_devices') }}
 )
 select
     spine.start_hour,
-    plants.plant_uuid,
+    plant_metadata.plant_id as plant_id,
     plants.plant as plant_name,
-    plant_metadata.plant_id as plantmonitor_plant_id,
     plant_metadata.peak_power_kw::float as peak_power_kw,
     plant_metadata.technology as technology,
     dset.irradiation as dset_irradiation_wh,
-    dset.inverter_exported_energy as dset_inverter_energy_kwh,
-    dset.meter_exported_energy as dset_meter_instant_exported_energy_kwh,
+    dset.inverter_energy as dset_inverter_energy_kwh,
+    dset.exported_energy as dset_meter_instant_exported_energy_kwh,
     NULL::integer as dset_meter_exported_energy_kwh,
     NULL::integer as dset_meter_imported_energy_kwh,
     forecast.forecastdate as forecast_date,
@@ -26,14 +25,14 @@ select
     sr.energy_output_kwh as satellite_energy_output_kwh,
     omie.price as omie_price_eur_mwh,
     {# exported_energy should be in wh we pass it to kwh. Also the /1000 is GSTC[W/m2] #}
-    (dset.meter_exported_energy*1000 / plant_metadata.peak_power_kw::float) / (NULLIF(sr.tilted_irradiation_wh_m2, 0.0) / 1000.0) as pr_hourly,
+    (dset.exported_energy*1000 / plant_metadata.peak_power_kw::float) / (NULLIF(sr.tilted_irradiation_wh_m2, 0.0) / 1000.0) as pr_hourly,
     spine.start_hour between solar_events.sunrise_real and solar_events.sunset_real as is_daylight_real,
     spine.start_hour between solar_events.sunrise_generous and solar_events.sunset_generous as is_daylight_generous,
     round(meter_registry.export_energy_wh/1000,2) as erp_meter_exported_energy_kwh,
     round(meter_registry.import_energy_wh/1000,2) as erp_meter_imported_energy_kwh
 from {{ ref('spine_hourly') }} as spine
 left join plants ON TRUE
-left join {{ ref('raw_gestio_actius_plant_parameters') }} plant_metadata on plants.plant_uuid = plant_metadata.plant_uuid
+left join {{ ref('seed_plants__parameters') }} plant_metadata on plants.plant = plant_metadata.plant_name
 left join {{ ref('int_dset_metrics_wide_hourly') }} dset using(start_hour, plant)
 left join {{ ref('int_energy_forecasts__best_from_plantmonitordb') }} forecast using(start_hour, plant_id)
 left join {{ ref('int_satellite_readings__hourly') }} sr

dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__denormalized.sql

@@ -10,7 +10,7 @@ select
     module_temperature_dc,
     energy_output_kwh
 from {{ ref('raw_solargis_satellite_readings__temp_and_pv_energy') }} sg
-left join {{ref('raw_gestio_actius_plant_parameters')}} p on p.plant_id = sg.plant_id
+left join {{ref('seed_plants__parameters')}} p on p.plant_id = sg.plant_id
 
 -- SolarGis PVOUT (aquí photovoltaic_energy_output_wh) retorna l'energia en kwh però plantmonitor per error ho registra com a wh sense fer cap transformació.
 -- Entenem que al redash s'està corregint a mà abans de mostrar el valor.

dbt_jardiner/models/jardiner/marts/dm_dset_responses__check_if_we_receive_data_for_signals.sql

@@ -14,6 +14,6 @@ select
     signals.device_type,
     signals.device_uuid,
     coalesce(rebut_from_dset, false) as rebut_from_dset
-from {{ ref('raw_gestio_actius__signal_denormalized') }} as signals
+from {{ ref("seed_signals__with_devices") }} as signals
 left join valors on signals.signal_uuid = valors.signal_uuid
 order by plant, signal

dbt_jardiner/models/jardiner/marts/dm_plants.sql

@@ -1,7 +1,7 @@
 {{ config(materialize='view') }}
 
 select
-    plant_name as nom_planta,
+    plant as nom_planta,
     municipality as municipi,
     province as provincia,
     latitude as latitud,

dbt_jardiner/models/jardiner/raw/gestio_actius/_raw_gestio_actius__models.yaml

@@ -1,57 +1,8 @@
 version: 2
 
 models:
+
   - name: raw_gestio_actius_production_target
     description: >
       Conté la taula incremental que es crea mitjançant el snapshot (al schema lake)
-      que llegueix la source del drive importada per airbyte (al schema airbyte_imported)
-  - name: raw_gestio_actius__signals__denormalized
-    description: >
-      Taula amb la llista de devices, els seus uuids i si tenen un parent.
-
-      Conté la relació entre signal_description de l'API de DSET (un UUID
-      imputat per nosaltres en el document de maping que se'ls envia) amb un device_uuid
-      també imputat per nosaltres que es correspon amb la columna homònima de la taula
-      d'aparell que correspongui ja sigui inverter, string, sensor, plant, etc.
-      Aquesta taula equival al 'seed' data/seed_signals__with_devices.csv que es carrega
-      a la taula utilitzant l'script scripts/file_to_sql.py.
-      Exemple del maping enviat per [Llanillos](https://docs.google.com/spreadsheets/d/1op_WHvGZNyDdkBD7EOXK-CPU5-aGjK3r/edit#gid=629299947)
-    columns:
-      - name: signal_id
-        description: id intern de la db
-      - name: plant_uuid
-        description: uuid4 de la planta
-      - name: plant
-        description: nom de la planta
-      - name: signal
-        description: nom del senyal
-      - name: metric
-        description: nom de la metrica
-      - name: device
-        description: nom del device
-      - name: device_type
-        description: tipus de device
-        tests:
-          - accepted_values:
-              values:
-                ["meter", "inverter", "sensor", "plant", "string", "module"]
-      - name: device_parent
-        description: device parent del device. e.g. un string té l'inversor1 com a pare
-      - name: signal_uuid
-        description:
-          UUID del senyal present a la columns signal_UUID del maping. Es assignada manualment amb
-          cada nou mapeig de GA.
-        tests:
-          - unique
-      - name: device_uuid
-        description:
-          Imputat per nosaltres que es correspon amb la columna homònima de la
-          taula d'aparell que correspongui ja sigui inverter, string, sensor, plant, etc.
-    tests:
-      - dbt_utils.unique_combination_of_columns:
-          config:
-            severity: error
-          combination_of_columns:
-            - plant_uuid
-            - device_uuid
-            - signal_uuid
+      que llegueix la source del drive importada per airbyte (al schema airbyte_imported)

dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_plant_parameters.sql

@@ -1,35 +1,34 @@
-{{ config(materialized="view") }}
+{{ config(materialized='view') }}
 
-with
-    current_data as (
-        select
-            plant_id::numeric,
-            plant_uuid::uuid,
-            case
-                when planta = 'Vallehermoso' then 'Alcolea'
-                when planta = 'Tahal' then 'Terborg'
-                else planta
-            end as plant_name,
-            latitud::numeric as latitude,
-            longitut::numeric as longitude,
-            municipi as municipality,
-            provincia as province,
-            tecnologia as technology,
-            data_connexio::date as connection_date,
-            potencia_nominal_kw::numeric as nominal_power_kw,
-            potencia_pic_kw::numeric as peak_power_kw,
-            "owner",
-            n_strings_plant::numeric,
-            n_modules_string::numeric,
-            n_strings_inverter::numeric,
-            esquema_unifilar,
-            layout,
-            data_actualitzacio::date as gestio_actius_updated_at,
-            dbt_updated_at::date as dbt_updated_at,
-            dbt_valid_from::date as dbt_valid_from,
-            coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to
-        from {{ ref("snapshot_plant_parameters") }} as pl
-    )
+with current_data as (
+    SELECT
+        plant_id::numeric,
+        case
+            when planta = 'Vallehermoso' then 'Alcolea'
+            when planta = 'Tahal' then 'Terborg'
+            else planta
+        end as plant,
+        latitud::numeric as latitude,
+        longitut::numeric as longitude,
+        municipi as municipality,
+        provincia as province,
+        tecnologia as technology,
+        data_connexio::date as connection_date,
+        potencia_nominal_kw::numeric as nominal_power_kw,
+        potencia_pic_kw::numeric as peak_power_kw,
+        owner,
+        n_strings_plant::numeric,
+        n_modules_string::numeric,
+        n_strings_inverter::numeric,
+        is_tilted::boolean,
+        esquema_unifilar,
+        layout,
+        data_actualitzacio::date as gestio_actius_updated_at,
+        dbt_updated_at::date as dbt_updated_at,
+        dbt_valid_from::date as dbt_valid_from,
+        coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to
+    FROM {{ ref('snapshot_plant_parameters') }} as pl
+)
 select *
 from current_data
 where dbt_valid_from::date <= current_date and current_date < dbt_valid_to::date

dbt_jardiner/models/jardiner/raw/misc/_raw_misc__models.yaml

@@ -0,0 +1,32 @@
+version: 2
+
+models:
+  - name: raw_devices__with_signals
+    description: >
+      Taula amb la llista de devices, els seus uuids i si tenen un parent.
+    columns:
+      - name: plant
+        description: nom de la planta
+      - name: signal
+        description: nom del senyal
+      - name: metric
+        description: nom de la metrica
+      - name: device
+        description: nom del device
+      - name: device_type
+        description: tipus de device
+      - name: device_parent
+        description: uuid del pare de un aparell, si en té
+      - name: signal_uuid
+        description: uuid4 del senyal
+        tests:
+          - unique
+      - name: device_uuid
+        description: uuid4 del device
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          config:
+            severity: error
+          combination_of_columns:
+            - device_uuid
+            - signal_uuid