From 578eb9ffa3a8ce2031685af819f249f6ce6bc60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pol=20Mons=C3=B3=20Purt=C3=AD?= Date: Tue, 2 Apr 2024 09:58:10 +0200 Subject: [PATCH] Chore/sqlfluffing (et/somenergia-jardiner!141) * :warning: some sqlfluff fix changed column order, restored some columns ordering, might require full refresh on prod * add flake8 config until I figure out which linter we use * fixed all linting errors * manually fixed models * manually fixed models * fixable issues * fixable issues * fixable issues * fixable issues --- .flake8 | 3 + .../analyses/dset_gaps_preview_20240110.sql | 82 +++++---- dbt_jardiner/analyses/plants_description.sql | 79 ++++----- .../dset/int_dset__last_registries.sql | 12 +- .../int_dset__power_from_instant_energy.sql | 10 +- .../dset/int_dset_energy_inverter__5m.sql | 41 +++-- .../dset/int_dset_meter__readings.sql | 6 +- .../int_dset_meter__readings_wide_hourly.sql | 3 +- .../dset/int_dset_metrics_long_hourly.sql | 85 ++++++--- .../dset/int_dset_metrics_wide_hourly.sql | 6 +- .../dset/int_dset_responses__deduplicated.sql | 8 +- .../dset/int_dset_responses__materialized.sql | 56 +++--- .../int_dset_responses__spined_metadata.sql | 7 +- ...int_dset_responses__values_incremental.sql | 4 +- .../erp/int_erp_meter_registry__hourly.sql | 16 +- ...gnal_device_relation__distinct_devices.sql | 9 +- .../int_plants_overview_instant.sql | 8 +- ...gy_forecasts__best_from_plantmonitordb.sql | 30 ++-- .../obt/obt_hourly_incremental.sql | 63 +++++-- .../raw_airbyte_simel__simel_mhcil_nas.sql | 42 ++--- ...nt_plantmonitordb_solarevent__generous.sql | 4 +- .../int_satellite_readings__denormalized.sql | 26 +-- .../int_satellite_readings__hourly.sql | 34 ++-- ..._dashboard_overview__produccio_cartera.sql | 46 +++-- .../marts/dm_dashboard_overview_instant.sql | 33 +++- .../marts/dm_dashboard_overview_monthly.sql | 24 ++- ...m_dset__all_signals_received_per_plant.sql | 25 +-- ..._energy_inverter__monthly__reset_daily.sql | 42 +++-- .../marts/dm_plant_production_daily.sql | 77 +++++---- .../marts/dm_plant_production_monthly.sql | 67 +++++-- .../models/jardiner/marts/dm_plants.sql | 42 ++--- ...et_energy_inverter__daily__reset_daily.sql | 116 +++++++------ ...__check_if_we_receive_data_for_signals.sql | 43 +++-- .../obs_dset_responses__expected_signals.sql | 14 +- ...et_responses__instant_power_comparison.sql | 10 +- ...obs_dset_responses__unexpected_signals.sql | 5 +- .../raw/dset/raw_dset_meter__readings.sql | 10 +- .../dset/raw_dset_responses__api_response.sql | 87 ++++++++-- ...aw_dset_responses__last_signal_reading.sql | 55 +++--- ...raw_gestio_actius__signal_denormalized.sql | 12 +- .../raw_gestio_actius_plant_parameters.sql | 5 +- .../raw_gestio_actius_production_target.sql | 8 +- ...asts__denormalized_from_plantmonitordb.sql | 39 ++--- ...orical_price__with_row_number_per_date.sql | 20 ++- .../plantmonitor_erp/raw_meterregistry.sql | 10 +- .../raw_plantmonitor_plants.sql | 10 +- ...aw_plantmonitordb_solarevent__generous.sql | 17 +- ..._solargis_satellite_readings__solargis.sql | 15 +- ...satellite_readings__temp_and_pv_energy.sql | 18 +- .../jardiner/raw/utils/spine_hourly.sql | 11 +- .../raw/utils/spine_monthly__full_year.sql | 14 +- .../models/legacy/alarms/alarm_historical.sql | 24 +-- .../alarms/alarm_inverter_temperature.sql | 25 +-- .../alarm_inverter_zero_power_at_daylight.sql | 57 +++--- ..._inverter_zero_power_at_daylight_daily.sql | 29 ++-- .../legacy/alarms/alarm_meter_no_energy.sql | 16 +- ...alarm_meter_no_energy_plant_thresholds.sql | 19 +- .../legacy/alarms/alarm_meter_no_readings.sql | 27 +-- .../alarms/stage/alarm_current_alarmed.sql | 4 +- .../stage/alarm_historical_one_day_ago.sql | 18 +- .../stage/alarm_historical_two_days_ago.sql | 4 +- ...ter_interinverter_relative_temperature.sql | 28 +-- .../alerts/alert_inverter_temperature.sql | 42 ++--- .../alert_inverter_zero_power_at_daylight.sql | 90 ++++++---- .../legacy/alerts/alert_meter_no_readings.sql | 10 +- .../legacy/alerts/alert_meter_zero_energy.sql | 87 ++++++---- ...alert_string_low_intensity_at_daylight.sql | 73 ++++---- .../models/legacy/alerts/alerts_historic.sql | 28 +-- ...alert_inverterregistry_clean_last_hour.sql | 4 +- .../stage/alert_meter_newest_reading.sql | 53 +++--- .../stage/alert_meter_zero_energy_by_now.sql | 101 ++++++----- .../average_yearly_production_plant.sql | 44 +++-- .../datasets/plant_production_daily.sql | 123 +++++++++---- .../datasets/plant_production_hourly.sql | 163 +++++++++++------- .../datasets/plant_production_monthly.sql | 129 ++++++++++---- .../tecnics/inverter_intensity_energes.sql | 31 ++-- .../tecnics/inverter_intensity_ercam.sql | 31 ++-- .../tecnics/inverter_intensity_exiom.sql | 31 ++-- .../tecnics/inverterregistry_energes.sql | 2 +- .../tecnics/inverterregistry_ercam.sql | 2 +- .../tecnics/inverterregistry_exiom.sql | 2 +- .../tecnics/meterregistry_energetica.sql | 4 +- .../sensorirradiationregistry_energes.sql | 27 +-- .../sensorirradiationregistry_ercam.sql | 27 +-- .../operational/derived/view_availability.sql | 25 +-- .../derived/view_clean_irradiation.sql | 46 +++-- .../derived/view_cos_phi_monthly.sql | 35 ++-- .../derived/view_expected_power.sql | 67 +++---- .../operational/derived/view_hd_daily.sql | 17 +- .../operational/derived/view_ht_daily.sql | 24 +-- .../operational/derived/view_ht_monthly.sql | 24 +-- .../derived/view_inverter_metrics_daily.sql | 43 +++-- .../view_plant_export_energy_monthly.sql | 56 +++--- .../operational/derived/view_pr_hourly.sql | 37 ++-- .../operational/derived/view_q_monthly.sql | 31 ++-- .../derived/view_satellite_irradiation.sql | 13 +- .../derived/view_target_energy.sql | 28 +-- ...asts__denormalized_from_plantmonitordb.sql | 40 ++--- .../external_data/forecasts_daily.sql | 8 +- .../external_data/forecasts_hourly.sql | 38 ++-- .../plantmonitordb_solarevent__generous.sql | 17 +- .../satellite_readings_daily.sql | 17 +- .../inverter/inverterregistry_clean.sql | 48 +++--- .../inverter/inverterregistry_clean_daily.sql | 66 +++---- .../inverter/inverterregistry_multisource.sql | 54 +++--- .../inverter/inverterregistry_raw.sql | 30 ++-- .../operational/inverter/inverters_raw.sql | 14 +- .../inverter/lake_modbusreadings_inverter.sql | 2 +- .../lake_modbusreadings_inverter_concat.sql | 20 +-- .../lake_modbusreadings_inverter_standard.sql | 29 ++-- .../stg/inverterregistry_gapfilled.sql | 30 ++-- .../inverter/stg/inverterregistry_sample.sql | 10 +- .../lake_modbusreadings_inverter_pivoted.sql | 16 +- .../meter/meter_noenergy_alarm.sql | 2 +- .../operational/meter/meter_registry.sql | 19 +- .../meter/meter_registry_daily.sql | 12 +- .../meter/meter_registry_hourly.sql | 27 +-- .../operational/meter/meter_registry_raw.sql | 22 +-- .../legacy/operational/meter/meters_raw.sql | 12 +- .../meter/meters_with_thresholds.sql | 12 +- .../modbus/lake_modbusreadings_raw.sql | 21 ++- .../lake_modbusreadings_selected_newest.sql | 42 +++-- .../stg/lake_modbusreadings_invalid.sql | 5 +- .../stg/lake_modbusreadings_selected_raw.sql | 38 ++-- .../stg/lake_modbusreadings_stringbox_raw.sql | 11 +- .../modbus/stg/lake_modbusreadings_valid.sql | 7 +- ...io_actius_plant_parameters__raw_legacy.sql | 65 +++---- .../operational/plant/plantparameters_raw.sql | 30 ++-- .../operational/plant/som_plants_extended.sql | 17 +- .../operational/plant/som_plants_raw.sql | 12 +- ...atellite_readings__denormalized_legacy.sql | 21 +-- .../satellite_readings__hourly_legacy.sql | 33 ++-- ...te_readings__temp_and_pv_energy_legacy.sql | 18 +- .../stg/lake_modbusreadings_stringboxes.sql | 48 +++--- ...ake_modbusreadings_stringboxes_invalid.sql | 5 +- ...ke_modbusreadings_stringboxes_standard.sql | 8 +- .../string/stringregistry_denormalized.sql | 24 +-- .../string/stringregistry_latest_hour.sql | 2 +- .../string/stringregistry_multisource.sql | 22 +-- .../operational/string/stringregistry_raw.sql | 10 +- .../legacy/operational/string/strings_raw.sql | 10 +- .../operational/tecnics/plants_energes.sql | 15 +- .../operational/tecnics/plants_energetica.sql | 15 +- .../operational/tecnics/plants_ercam.sql | 15 +- .../operational/tecnics/plants_exiom.sql | 15 +- .../legacy/operational/view_target_energy.sql | 32 ++-- .../models/legacy/utils/som_big_plants.sql | 10 +- .../models/legacy/utils/som_plants.sql | 17 +- .../models/legacy/utils/spine_days.sql | 3 +- .../legacy/utils/spine_days_localized.sql | 3 +- .../models/legacy/utils/spine_hours.sql | 3 +- .../legacy/utils/spine_hours_localized.sql | 3 +- .../utils/spine_hours_localized_until_now.sql | 5 +- .../legacy/utils/spine_plant_meter_daily.sql | 9 +- .../legacy/utils/spine_plant_meter_hourly.sql | 14 +- .../legacy/utils/spine_plant_meter_today.sql | 8 +- .../spine_plant_meter_until_last_hour.sql | 8 +- .../snapshots/snapshot_plant_parameters.sql | 2 +- .../snapshots/snapshot_production_target.sql | 2 +- .../snapshot_signal_denormalized.sql | 2 +- ...t_active_power__missing_phase_last_day.sql | 55 +++--- ...set_gaps_per_day_and_signal_last_month.sql | 160 ++++++++--------- ...have_references_in_signal_denormalized.sql | 8 +- ...set_raw_uuids_not_empty_or_null_3_days.sql | 36 ++-- .../test_dset_signals_receiver_all_time.sql | 53 +++--- .../test_dset_signals_receiver_last_hour.sql | 47 ++--- .../tests/dset/test_gapfill_5min_nrows.sql | 23 +-- .../dset/test_repeated_device_uuid_errors.sql | 21 ++- ...gnal_is_correctly_converted_to_numeric.sql | 3 +- .../test_solargis_temperature_registry.sql | 15 +- dbt_jardiner/tests/dset/test_spine_hourly.sql | 26 ++- 171 files changed, 2858 insertions(+), 2074 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..10090e87 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 95 +max-complexity = 10 \ No newline at end of file diff --git a/dbt_jardiner/analyses/dset_gaps_preview_20240110.sql b/dbt_jardiner/analyses/dset_gaps_preview_20240110.sql index 8980df64..ba6107ff 100644 --- a/dbt_jardiner/analyses/dset_gaps_preview_20240110.sql +++ b/dbt_jardiner/analyses/dset_gaps_preview_20240110.sql @@ -1,39 +1,59 @@ with - som_uuids as ( - select distinct group_name, signal_code, signal_uuid - from {{ ref("int_dset_responses__materialized") }} - where ts >= '2024-01-10' and ts < '2024-01-11' - order by signal_uuid desc), +som_uuids as ( + select distinct + group_name, + signal_code, + signal_uuid + from {{ ref("int_dset_responses__materialized") }} + where ts >= '2024-01-10' and ts < '2024-01-11' + order by signal_uuid desc +), - dset_uuids as ( - select a.gru_codi, a.gru_nom, a.sen_codi, a.sen_descripcio, a.esperats_frequencia, a.trobats_senyal, b.signal_uuid - from analytics.se_forats_hornsby_dades_dia_10 as a - left join som_uuids as b - on a.sen_codi = b.signal_code - and a.gru_nom = b.group_name), +dset_uuids as ( + select + a.gru_codi, + a.gru_nom, + a.sen_codi, + a.sen_descripcio, + a.esperats_frequencia, + a.trobats_senyal, + b.signal_uuid + from analytics.se_forats_hornsby_dades_dia_10 as a + left join som_uuids as b + on + a.sen_codi = b.signal_code + and a.gru_nom = b.group_name +), - som_count as ( - select - signal_uuid, count(*) as cnt - from {{ ref("int_dset_responses__materialized") }} - where ts >= '2024-01-10' and ts < '2024-01-11' - and signal_value is not null - group by signal_uuid - order by signal_uuid desc, cnt desc), +som_count as ( + select + signal_uuid, + count(*) as cnt + from {{ ref("int_dset_responses__materialized") }} + where + ts >= '2024-01-10' and ts < '2024-01-11' + and signal_value is not null + group by signal_uuid + order by signal_uuid desc, cnt desc +), - summary as ( - select b.*, a.cnt as som_trobats_senyal from som_count as a - left join dset_uuids as b - on a.signal_uuid = b.signal_uuid - order by b.trobats_senyal desc, a.cnt desc - ), +summary as ( + select + b.*, + a.cnt as som_trobats_senyal + from som_count as a + left join dset_uuids as b + on a.signal_uuid = b.signal_uuid + order by b.trobats_senyal desc, a.cnt desc +), - final as ( - select *, - esperats_frequencia - trobats_senyal as n_forats_dset, - 288 - som_trobats_senyal as n_forats_som - from summary - ) +final as ( + select + *, + esperats_frequencia - trobats_senyal as n_forats_dset, + 288 - som_trobats_senyal as n_forats_som + from summary +) select * from final order by gru_codi, gru_nom, sen_codi diff --git a/dbt_jardiner/analyses/plants_description.sql b/dbt_jardiner/analyses/plants_description.sql index 4a71524e..b59367f7 100644 --- a/dbt_jardiner/analyses/plants_description.sql +++ b/dbt_jardiner/analyses/plants_description.sql @@ -1,40 +1,41 @@ -- to generate the seed for plants_descriptions.csv -SELECT - p.id AS plant_id, - p.name AS plant_name, - p2.latitude AS latitude, - p2.longitude AS longitude, - p3.connection_date AS connection_date, - FALSE AS is_tilted, - p3.peak_power_w / 1000 AS peak_power_kw, - p3.nominal_power_w / 1000 AS nominal_power_kw, - CASE - WHEN p.name = 'Torregrossa' THEN 'GAS' - WHEN p.name = 'Valteina' THEN 'HIDRO' - ELSE 'FV' - END AS technology, - p3.target_monthly_energy_wh / 100 AS target_monthly_energy_gwh, - CASE - WHEN - p.name IN ('Florida', 'Alcolea', 'Matallana') THEN 'energes' - WHEN - p.name IN ('Valteina') THEN 'energetica' - WHEN - p.name IN ('Fontivsolar') THEN 'ercam' - WHEN - p.name IN ('Llanillos', 'Asomada') THEN 'exiom' - END AS propietari -FROM - public.plant AS p -LEFT JOIN - public.plantparameters AS p3 - ON - p.id = p3.plant -LEFT JOIN - public.plantlocation AS p2 - ON - p.id = p2.plant -WHERE - "description" != 'SomRenovables' -ORDER BY - p.id; +select + p.id as plant_id, + p.name as plant_name, + p2.latitude as latitude, + p2.longitude as longitude, + p3.connection_date as connection_date, + false as is_tilted, + p3.peak_power_w / 1000 as peak_power_kw, + p3.nominal_power_w / 1000 as nominal_power_kw, + case + when p.name = 'Torregrossa' then 'GAS' + when p.name = 'Valteina' then 'HIDRO' + else 'FV' + end as technology, + p3.target_monthly_energy_wh / 100 as target_monthly_energy_gwh, + case + when + p.name in ('Florida', 'Alcolea', 'Matallana') then 'energes' + when + p.name in ('Valteina') then 'energetica' + when + p.name in ('Fontivsolar') then 'ercam' + when + p.name in ('Llanillos', 'Asomada') then 'exiom' + end as propietari +from + public.plant as p + left join + public.plantparameters as p3 + on + p.id = p3.plant + left join + public.plantlocation as p2 + on + p.id = p2.plant +where + plant.description != 'SomRenovables' +order by + p.id +; diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__last_registries.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__last_registries.sql index 797fadd7..11e626d8 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__last_registries.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__last_registries.sql @@ -2,11 +2,15 @@ with ts_last_registry as ( - select signal_uuid, plant_uuid, max(ts) as ultim_registre - from {{ ref('int_dset_responses__values_incremental') }} - group by plant_uuid, signal_uuid + select + signal_uuid, + plant_uuid, + max(ts) as ultim_registre + from {{ ref('int_dset_responses__values_incremental') }} + group by plant_uuid, signal_uuid ) + select m.* from ts_last_registry as tslr -left join {{ ref('int_dset_responses__values_incremental') }} as m + left join {{ ref('int_dset_responses__values_incremental') }} as m on tslr.signal_uuid = m.signal_uuid and tslr.ultim_registre = m.ts diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__power_from_instant_energy.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__power_from_instant_energy.sql index c8d7fa16..26110156 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__power_from_instant_energy.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset__power_from_instant_energy.sql @@ -14,6 +14,7 @@ with instant_energy as ( and metric_name = 'energia_activa_exportada_instantania' and signal_value is not null ), + instant_energy_delta as ( select ts, @@ -23,26 +24,31 @@ instant_energy_delta as ( lead(ts) over plant_window as previous_ts, lead(meter_energy_kwh) over plant_window as previous_energy, ts - lead(ts) over plant_window as delta_ts, - meter_energy_kwh - lead(meter_energy_kwh) over plant_window as delta_meter_energy_kwh + meter_energy_kwh + - lead(meter_energy_kwh) over plant_window as delta_meter_energy_kwh from instant_energy window plant_window as (partition by plant_uuid order by ts desc) order by ts desc ), + inferred_meter_power as ( select ts, plant_name, plant_uuid, meter_energy_kwh, - 60 / extract(minute from delta_ts) * delta_meter_energy_kwh as inferred_meter_power_kw, previous_energy, delta_meter_energy_kwh, previous_ts, delta_ts, + 60 + / extract(minute from delta_ts) + * delta_meter_energy_kwh as inferred_meter_power_kw, 60 / extract(minute from delta_ts) as dt_hour_normalized from instant_energy_delta order by ts desc ) + select plant_uuid, ts, diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_energy_inverter__5m.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_energy_inverter__5m.sql index 7e26e230..5b775427 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_energy_inverter__5m.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_energy_inverter__5m.sql @@ -1,26 +1,25 @@ {{ config(materialized="view") }} -with - inverters_energy as ( - select - ts, - plant_uuid, - plant_name, - device_uuid, - device_name, - metric_name, - signal_uuid, - case - when signal_unit = 'MWh' then signal_value * 1000 - when signal_unit = 'kWh' then signal_value - else null - end as inverter_energy_kwh - from {{ ref("int_dset_responses__values_incremental") }} - where - device_type in ('inverter') - and metric_name - in ('energia_activa_exportada', 'energia_activa_exportada_total') - ) +with inverters_energy as ( + select + ts, + plant_uuid, + plant_name, + device_uuid, + device_name, + metric_name, + signal_uuid, + case + when signal_unit = 'MWh' then signal_value * 1000 + when signal_unit = 'kWh' then signal_value + end as inverter_energy_kwh + from {{ ref("int_dset_responses__values_incremental") }} + where + device_type in ('inverter') + and metric_name + in ('energia_activa_exportada', 'energia_activa_exportada_total') +) + select * from inverters_energy order by ts desc diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings.sql index 2119ff4e..00178fdb 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings.sql @@ -3,6 +3,7 @@ with quarterhourly_spine as ( select generate_series('2023-12-01', now(), '15 minutes') as start_ts ), + meter_metadata as ( select metadata.plant_uuid::uuid, @@ -22,6 +23,7 @@ meter_metadata as ( and metadata.is_enabled and metadata.metric_name ilike 'energia_%' ), + raw_meter_readings as ( select meter_readings.start_ts, @@ -33,11 +35,12 @@ raw_meter_readings as ( from {{ ref("raw_dset_meter__readings") }} as meter_readings where meter_readings.signal_uuid is not null ), + meter_readings_with_metadata as ( select + meter_metadata.*, quarterhourly_spine.start_ts, raw_meter_readings.end_ts, - meter_metadata.*, raw_meter_readings.signal_value, raw_meter_readings.signal_unit, raw_meter_readings.queried_at, @@ -46,4 +49,5 @@ meter_readings_with_metadata as ( left join meter_metadata on true left join raw_meter_readings using (signal_uuid, start_ts) ) + select * from meter_readings_with_metadata diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings_wide_hourly.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings_wide_hourly.sql index 4e9f2453..cd08102c 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings_wide_hourly.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_meter__readings_wide_hourly.sql @@ -2,9 +2,9 @@ with meter_readings_hourly as ( select - date_trunc('hour', start_ts) as start_hour, plant_uuid, plant_name, + date_trunc('hour', start_ts) as start_hour, sum(meter_exported_energy) as meter_exported_energy, sum(meter_imported_energy) as meter_imported_energy, sum(meter_reactive_energy_q1) as meter_reactive_energy_q1, @@ -16,4 +16,5 @@ with meter_readings_hourly as ( group by date_trunc('hour', start_ts), plant_uuid, plant_name order by date_trunc('hour', start_ts) desc ) + select * from meter_readings_hourly diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_long_hourly.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_long_hourly.sql index 002877f8..df46f564 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_long_hourly.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_long_hourly.sql @@ -14,32 +14,67 @@ since this is ugly as fuck #} #} with - dset_key_metrics as ( - select date_trunc('hour', ts) as start_hour, plant_name, plant_uuid, device_type, metric_name as split_metric, signal_value - from {{ ref("int_dset_responses__values_incremental") }} - where metric_name in ('energia_activa_exportada', 'energia_activa_exportada_instantania', 'energia_activa_importada', 'irradiancia') - ) -select - start_hour, +dset_key_metrics as ( + select plant_name, plant_uuid, - case - when split_metric = 'irradiancia' then 'irradiation' {# from W/m^2 to Wh/m^2 because split_metric has hourly granularity #} - when split_metric = 'energia_activa_exportada' then device_type || '_exported_energy' - when split_metric = 'energia_activa_importada' then device_type || '_imported_energy' - when split_metric = 'energia_activa_exportada_instantania' then device_type || '_instant_exported_energy' - else split_metric - end as metric_name, - case - when device_type = 'inverter' and split_metric = 'energia_activa_exportada' - then (extract(hour from start_hour) > 3)::integer * (max(signal_value) - min(signal_value)) {# we have random-ish resets before 3 #} - when device_type in ('sensor', 'module', 'inverter') and split_metric = 'irradiancia' - then avg(signal_value) - when device_type = 'meter' and (split_metric = 'energia_activa_exportada' or split_metric = 'energia_activa_importada') - then sum(signal_value) - when device_type = 'meter' and (split_metric = 'energia_activa_exportada_instantania') - then max(signal_value)-min(signal_value) - else null - end as metric_value + device_type, + metric_name as split_metric, + signal_value, + date_trunc('hour', ts) as start_hour + from {{ ref("int_dset_responses__values_incremental") }} + where + metric_name in ( + 'energia_activa_exportada', + 'energia_activa_exportada_instantania', + 'energia_activa_importada', + 'irradiancia' + ) +) + +select + start_hour, + plant_name, + plant_uuid, + case + when + split_metric = 'irradiancia' + then 'irradiation' + {# from W/m^2 to Wh/m^2 because split_metric has hourly granularity #} + when + split_metric = 'energia_activa_exportada' + then device_type || '_exported_energy' + when + split_metric = 'energia_activa_importada' + then device_type || '_imported_energy' + when + split_metric = 'energia_activa_exportada_instantania' + then device_type || '_instant_exported_energy' + else split_metric + end as metric_name, + case + when + device_type = 'inverter' + and split_metric = 'energia_activa_exportada' + then + (extract(hour from start_hour) > 3)::integer + * (max(signal_value) - min(signal_value)) + {# we have random-ish resets before 3 #} + when + device_type in ('sensor', 'module', 'inverter') + and split_metric = 'irradiancia' + then avg(signal_value) + when + device_type = 'meter' + and ( + split_metric = 'energia_activa_exportada' + or split_metric = 'energia_activa_importada' + ) + then sum(signal_value) + when + device_type = 'meter' + and (split_metric = 'energia_activa_exportada_instantania') + then max(signal_value) - min(signal_value) + end as metric_value from dset_key_metrics group by start_hour, plant_name, plant_uuid, device_type, split_metric diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_wide_hourly.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_wide_hourly.sql index 55dd24a0..ad8f9105 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_wide_hourly.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_metrics_wide_hourly.sql @@ -1,9 +1,9 @@ {{ config(materialized="view") }} select - start_hour, - plant_name, - plant_uuid, + start_hour, + plant_name, + plant_uuid, {{ pivot( column="metric_name", diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__deduplicated.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__deduplicated.sql index 5b652497..f4b66ddd 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__deduplicated.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__deduplicated.sql @@ -1,10 +1,10 @@ {{ config(materialize="view") }} with - ordered as ( - select *, row_number() over (partition by ts, signal_uuid order by queried_at desc) as row_order - from {{ ref("int_dset_responses__validate_uuids") }} - ) +ordered as ( + select *, row_number() over (partition by ts, signal_uuid order by queried_at desc) as row_order + from {{ ref("int_dset_responses__validate_uuids") }} +) select {{ dbt_utils.star(from=ref("int_dset_responses__validate_uuids")) }} from ordered where row_order = 1 diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__materialized.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__materialized.sql index 57ffb3b8..128d345d 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__materialized.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__materialized.sql @@ -10,36 +10,38 @@ with - normalized_jsonb as ( +normalized_jsonb as ( - select - group_name, - queried_at, - ts, - signal_code, - signal_device_type, - signal_device_uuid, - signal_frequency, - signal_id, - signal_is_virtual, - signal_last_ts, - signal_last_value, - signal_type, - signal_tz, - signal_unit, - signal_uuid, - signal_uuid_raw, - signal_value, - now() as materialized_at - from {{ ref("int_dset_responses__deduplicated") }} - ) + select + group_name, + queried_at, + ts, + signal_code, + signal_device_type, + signal_device_uuid, + signal_frequency, + signal_id, + signal_is_virtual, + signal_last_ts, + signal_last_value, + signal_type, + signal_tz, + signal_unit, + signal_uuid, + signal_uuid_raw, + signal_value, + now() as materialized_at + from {{ ref("int_dset_responses__deduplicated") }} +) select * from normalized_jsonb {% if is_incremental() -%} -where queried_at > coalesce((select max(queried_at) from {{ this }}), '1900-01-01') - and ts > now() - interval '2 days' - -- dedupliquem 2 dies enrera (ho diu el predicate) i per això no podem garantir que tot l'anterior sigui unic - -- i per tant ho descartem aqui, en la selecció del incremental - -- Si per algun motiu deixem de materialitzar durant 48 hores, caldrà fer un full-refresh! + where + queried_at + > coalesce((select max(queried_at) from {{ this }}), '1900-01-01') + and ts > now() - interval '2 days' +-- dedupliquem 2 dies enrera (ho diu el predicate) i per això no podem garantir que tot l'anterior sigui unic +-- i per tant ho descartem aqui, en la selecció del incremental +-- Si per algun motiu deixem de materialitzar durant 48 hores, caldrà fer un full-refresh! {%- endif %} diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql index 8e76ea3b..d423237e 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql @@ -21,8 +21,11 @@ spined_expected_signals as ( spina5m {# dm_plants is the SSOT of the plants names #} left join {{ ref("int_gda_plants__plants_catalog") }} as plants on true - left join {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata using (plant_uuid) + left join + {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata + using (plant_uuid) ), + dset_from_december_2023 as ( select * from {{ ref("int_dset_responses__materialized") }} where @@ -60,7 +63,7 @@ spined_dset as ( valors.ts is not null as from_dset from spined_expected_signals as spined left join dset_from_december_2023 as valors - using (ts, signal_uuid) + on spined.ts = valors.ts and spined.signal_uuid = valors.signal_uuid order by spined.ts desc, spined.plant_name asc ) diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__values_incremental.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__values_incremental.sql index fe879463..55c07376 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__values_incremental.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__values_incremental.sql @@ -17,7 +17,9 @@ from {{ ref("int_dset_responses__spined_metadata") }} #} -- noqa: disable=LT02 {% if is_incremental() -%} -where materialized_at > coalesce((select max(materialized_at) from {{ this }}), '1900-01-01') +where + materialized_at + > coalesce((select max(materialized_at) from {{ this }}), '1900-01-01') and ts > now() - interval '2 days' {# dedupliquem 2 dies enrera (ho diu el predicate) i per això no podem garantir que tot l'anterior sigui unic diff --git a/dbt_jardiner/models/jardiner/intermediate/erp/int_erp_meter_registry__hourly.sql b/dbt_jardiner/models/jardiner/intermediate/erp/int_erp_meter_registry__hourly.sql index a1bb2846..a1f1931f 100644 --- a/dbt_jardiner/models/jardiner/intermediate/erp/int_erp_meter_registry__hourly.sql +++ b/dbt_jardiner/models/jardiner/intermediate/erp/int_erp_meter_registry__hourly.sql @@ -3,18 +3,27 @@ with meter_registry_hourly_raw as ( select - date_trunc('hour', mr.start_hour) as start_hour, mr.meter_id, mr.meter_name, mr.plant_id, p.plant_uuid, p.plant_name, p.plant_codename, + date_trunc('hour', mr.start_hour) as start_hour, round(avg(mr.export_energy_wh), 2) as export_energy_wh, round(avg(mr.import_energy_wh), 2) as import_energy_wh from {{ ref('raw_meterregistry') }} as mr - left join {{ ref("raw_plantmonitor_plants") }} as p using (plant_id) - group by date_trunc('hour', mr.start_hour), p.plant_uuid, mr.plant_id, p.plant_name, p.plant_codename, mr.meter_id, mr.meter_name + left join + {{ ref("raw_plantmonitor_plants") }} as p + on mr.plant_id = p.plant_id + group by + date_trunc('hour', mr.start_hour), + p.plant_uuid, + mr.plant_id, + p.plant_name, + p.plant_codename, + mr.meter_id, + mr.meter_name ) select @@ -22,6 +31,5 @@ select case when export_energy_wh > 0 then 1 when export_energy_wh = 0 then 0 - else null end as has_energy from meter_registry_hourly_raw diff --git a/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql b/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql index 427ab1a3..edf44f45 100644 --- a/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql +++ b/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql @@ -1,5 +1,12 @@ {{ config(materialized="table") }} -select distinct plant_name, plant_uuid, device_name, device_uuid, device_type, device_parent, device_parent_uuid +select distinct + plant_name, + plant_uuid, + device_name, + device_uuid, + device_type, + device_parent, + device_parent_uuid from {{ ref('raw_gestio_actius__signal_denormalized') }} order by plant_name, device_name diff --git a/dbt_jardiner/models/jardiner/intermediate/int_plants_overview_instant.sql b/dbt_jardiner/models/jardiner/intermediate/int_plants_overview_instant.sql index 4b89a9c1..c379a9ec 100644 --- a/dbt_jardiner/models/jardiner/intermediate/int_plants_overview_instant.sql +++ b/dbt_jardiner/models/jardiner/intermediate/int_plants_overview_instant.sql @@ -80,13 +80,17 @@ select p.technology, p.peak_power_kw, p.nominal_power_kw, - coalesce(i.instant_power_plant_kw_registered_at, ip.ts) as instant_power_plant_kw_registered_at, - coalesce(i.instant_power_plant_kw, ip.inferred_meter_power_kw) as instant_power_plant_kw, ir.ts as irradiance_w_m2_last_registered_at, ir.signal_value as irradiance_w_m2, ppd.day, ppd.meter_exported_energy_kwh, ppd.solargis_meter_expected_energy_kwh, + coalesce( + i.instant_power_plant_kw_registered_at, ip.ts + ) as instant_power_plant_kw_registered_at, + coalesce( + i.instant_power_plant_kw, ip.inferred_meter_power_kw + ) as instant_power_plant_kw, case when i.instant_power_plant_kw is not null then 'directa' else 'inferida' diff --git a/dbt_jardiner/models/jardiner/intermediate/meteologica/int_energy_forecasts__best_from_plantmonitordb.sql b/dbt_jardiner/models/jardiner/intermediate/meteologica/int_energy_forecasts__best_from_plantmonitordb.sql index 7bfb2317..270255fb 100644 --- a/dbt_jardiner/models/jardiner/intermediate/meteologica/int_energy_forecasts__best_from_plantmonitordb.sql +++ b/dbt_jardiner/models/jardiner/intermediate/meteologica/int_energy_forecasts__best_from_plantmonitordb.sql @@ -12,16 +12,22 @@ #} with - forecasts_denormalized as ( - select * - from {{ ref("raw_energy_forecasts__denormalized_from_plantmonitordb") }} - inner join {{ ref("raw_plantmonitor_plants") }} p using (plant_id) - ) +forecasts_denormalized as ( + select + forecasts.plant_id, + forecasts.time, + forecasts.forecastdate, + plants.plant_uuid, + forecasts.energy_kwh + from {{ ref("raw_energy_forecasts__denormalized_from_plantmonitordb") }} as forecasts + inner join {{ ref("raw_plantmonitor_plants") }} as plants using (plant_id) +) + select distinct - on (plant_id, "time") - fd.forecastdate, - fd."time" - interval '1 hour' as start_hour, - plant_uuid, - fd.energy_kwh -from forecasts_denormalized fd -order by plant_id, "time" desc, forecastdate desc +on (fd.plant_id, fd."time") + fd.forecastdate, + fd.plant_uuid, + fd.energy_kwh, + fd."time" - interval '1 hour' as start_hour +from forecasts_denormalized as fd +order by fd.plant_id asc, fd."time" desc, fd.forecastdate desc diff --git a/dbt_jardiner/models/jardiner/intermediate/obt/obt_hourly_incremental.sql b/dbt_jardiner/models/jardiner/intermediate/obt/obt_hourly_incremental.sql index d36621b8..16a0bfb0 100644 --- a/dbt_jardiner/models/jardiner/intermediate/obt/obt_hourly_incremental.sql +++ b/dbt_jardiner/models/jardiner/intermediate/obt/obt_hourly_incremental.sql @@ -5,6 +5,7 @@ with erp_meter_registries_before_2024 as ( select * from {{ ref('int_erp_meter_registry__hourly') }} where start_hour < '2024-01-01' ), + obt_base as ( select spine.start_hour, @@ -28,30 +29,58 @@ obt_base as ( spine.start_hour between solar_events.sunrise_real and solar_events.sunset_real as is_daylight_real, spine.start_hour between solar_events.sunrise_generous and solar_events.sunset_generous as is_daylight_generous, {#- exported_energy should be in wh we pass it to kwh #} - round(meter_registry.export_energy_wh / 1000, 2) as erp_meter_exported_energy_kwh, - round(meter_registry.import_energy_wh / 1000, 2) as erp_meter_imported_energy_kwh + round( + meter_registry.export_energy_wh / 1000, 2 + ) as erp_meter_exported_energy_kwh, + round( + meter_registry.import_energy_wh / 1000, 2 + ) as erp_meter_imported_energy_kwh from {{ ref('spine_hourly') }} as spine {#- plant_parameters is the single source of truth about which plants we have, unexpected plants won't join#} - left join {{ ref('raw_gestio_actius_plant_parameters') }} as plant_metadata on true - left join {{ ref('int_dset_metrics_wide_hourly') }} as dset using (start_hour, plant_uuid) - left join {{ ref('int_dset_meter__readings_wide_hourly') }} as dset_meter_readings using (start_hour, plant_uuid) - left join {{ ref('int_energy_forecasts__best_from_plantmonitordb') }} as forecast using (start_hour, plant_uuid) - left join {{ ref('int_satellite_readings__hourly') }} as sr using (start_hour, plant_uuid) - left join {{ ref('raw_plantlake_omie_historical_price__with_row_number_per_date') }} as omie using (start_hour) - left join erp_meter_registries_before_2024 as meter_registry using (start_hour, plant_uuid) - left join {{ ref('int_plantmonitordb_solarevent__generous') }} as solar_events - on plant_metadata.plant_uuid = solar_events.plant_uuid and solar_events.day = spine.start_hour::date -), obt_derived as ( + left join + {{ ref('raw_gestio_actius_plant_parameters') }} as plant_metadata + on true + left join + {{ ref('int_dset_metrics_wide_hourly') }} as dset + using (start_hour, plant_uuid) + left join + {{ ref('int_dset_meter__readings_wide_hourly') }} as dset_meter_readings + using (start_hour, plant_uuid) + left join + {{ ref('int_energy_forecasts__best_from_plantmonitordb') }} as forecast + using (start_hour, plant_uuid) + left join + {{ ref('int_satellite_readings__hourly') }} as sr + using (start_hour, plant_uuid) + left join + {{ ref('raw_plantlake_omie_historical_price__with_row_number_per_date') }} as omie + using (start_hour) + left join + erp_meter_registries_before_2024 as meter_registry + using (start_hour, plant_uuid) + left join + {{ ref('int_plantmonitordb_solarevent__generous') }} as solar_events + on + plant_metadata.plant_uuid = solar_events.plant_uuid + and solar_events.day = spine.start_hour::date +), + +obt_derived as ( select *, - coalesce(dset_meter_exported_energy_kwh, erp_meter_exported_energy_kwh) as meter_exported_energy_kwh, - coalesce(dset_meter_imported_energy_kwh, erp_meter_imported_energy_kwh) as meter_imported_energy_kwh, + coalesce( + dset_meter_exported_energy_kwh, erp_meter_exported_energy_kwh + ) as meter_exported_energy_kwh, + coalesce( + dset_meter_imported_energy_kwh, erp_meter_imported_energy_kwh + ) as meter_imported_energy_kwh, {# The /1000 is GSTC[W/m2] #} - (dset_meter_exported_energy_kwh / peak_power_kw) / (nullif(satellite_irradiation_wh_m2, 0.0) / 1000.0) as pr_hourly + (dset_meter_exported_energy_kwh / peak_power_kw) + / (nullif(satellite_irradiation_wh_m2, 0.0) / 1000.0) as pr_hourly from obt_base order by start_hour desc, plant_name asc ) -select - * + +select * from obt_derived diff --git a/dbt_jardiner/models/jardiner/intermediate/simel/raw_airbyte_simel__simel_mhcil_nas.sql b/dbt_jardiner/models/jardiner/intermediate/simel/raw_airbyte_simel__simel_mhcil_nas.sql index 2ac54a88..e98f2a9c 100644 --- a/dbt_jardiner/models/jardiner/intermediate/simel/raw_airbyte_simel__simel_mhcil_nas.sql +++ b/dbt_jardiner/models/jardiner/intermediate/simel/raw_airbyte_simel__simel_mhcil_nas.sql @@ -1,26 +1,26 @@ with - source as ( - select * from {{ source("airbyte_simel", "simel_mhcil_nas") }} +source as ( + select * from {{ source("airbyte_simel", "simel_mhcil_nas") }} ), renamed as ( - select - {{ adapter.quote("cil") }}::text, - {{ adapter.quote("year") }}::integer, - {{ adapter.quote("month") }}::integer, - {{ adapter.quote("day") }}::integer, - {{ adapter.quote("hour") }}::integer, - {{ adapter.quote("is_summer") }}::boolean, - nullif( {{ adapter.quote("energy_kwh") }}, '')::numeric as {{ adapter.quote("energy_kwh") }}, - nullif( {{ adapter.quote("reactive_energy_2_kvarh") }}, '')::numeric as {{ adapter.quote("reactive_energy_2_kvarh") }}, - nullif( {{ adapter.quote("reactive_energy_3_kvarh") }}, '')::numeric as {{ adapter.quote("reactive_energy_3_kvarh") }}, - {{ adapter.quote("measurement_type") }}::text, - {{ adapter.quote("empty_col") }}::text, - {{ adapter.quote("_ab_source_file_last_modified") }}, - {{ adapter.quote("_ab_source_file_url") }}, - {{ adapter.quote("_airbyte_ab_id") }}, - {{ adapter.quote("_airbyte_emitted_at") }}, - {{ adapter.quote("_airbyte_normalized_at") }}, - {{ adapter.quote("_airbyte_simel_mhcil_nas_hashid") }} - from source + select + {{ adapter.quote("cil") }}::text, + {{ adapter.quote("year") }}::integer, + {{ adapter.quote("month") }}::integer, + {{ adapter.quote("day") }}::integer, + {{ adapter.quote("hour") }}::integer, + {{ adapter.quote("is_summer") }}::boolean, + nullif({{ adapter.quote("energy_kwh") }}, '')::numeric as {{ adapter.quote("energy_kwh") }}, + nullif({{ adapter.quote("reactive_energy_2_kvarh") }}, '')::numeric as {{ adapter.quote("reactive_energy_2_kvarh") }}, + nullif({{ adapter.quote("reactive_energy_3_kvarh") }}, '')::numeric as {{ adapter.quote("reactive_energy_3_kvarh") }}, + {{ adapter.quote("measurement_type") }}::text, + {{ adapter.quote("empty_col") }}::text, + {{ adapter.quote("_ab_source_file_last_modified") }}, + {{ adapter.quote("_ab_source_file_url") }}, + {{ adapter.quote("_airbyte_ab_id") }}, + {{ adapter.quote("_airbyte_emitted_at") }}, + {{ adapter.quote("_airbyte_normalized_at") }}, + {{ adapter.quote("_airbyte_simel_mhcil_nas_hashid") }} + from source ) select * from renamed diff --git a/dbt_jardiner/models/jardiner/intermediate/solar_events_ephem/int_plantmonitordb_solarevent__generous.sql b/dbt_jardiner/models/jardiner/intermediate/solar_events_ephem/int_plantmonitordb_solarevent__generous.sql index bfd25351..370d344c 100644 --- a/dbt_jardiner/models/jardiner/intermediate/solar_events_ephem/int_plantmonitordb_solarevent__generous.sql +++ b/dbt_jardiner/models/jardiner/intermediate/solar_events_ephem/int_plantmonitordb_solarevent__generous.sql @@ -11,4 +11,6 @@ select solarevent.solar_hours_real, solarevent.solar_hours_minimum from {{ ref("raw_plantmonitordb_solarevent__generous") }} as solarevent -left join {{ ref("raw_plantmonitor_plants") }} as plantmonitor_plant using(plant_id) + left join + {{ ref("raw_plantmonitor_plants") }} as plantmonitor_plant + on solarevent.plant_id = plantmonitor_plant.plant_id diff --git a/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__denormalized.sql b/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__denormalized.sql index 2247871f..86baf23c 100644 --- a/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__denormalized.sql +++ b/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__denormalized.sql @@ -1,18 +1,20 @@ {{ config(materialized='view') }} select - sg.start_hour, - sg.plant_id, - p.plant_name, - p.plant_uuid, - sg.request_time, - sg.horizontal_irradiation_wh_m2, - sg.tilted_irradiation_wh_m2, - sg.module_temperature_dc, - sg.energy_output_kwh -from {{ ref('raw_solargis_satellite_readings__temp_and_pv_energy') }} sg -left join {{ ref('raw_plantmonitor_plants') }} plantmonitor_plants using(plant_id) -left join {{ref('raw_gestio_actius_plant_parameters')}} p using(plant_uuid) + sg.start_hour, + sg.plant_id, + p.plant_name, + p.plant_uuid, + sg.request_time, + sg.horizontal_irradiation_wh_m2, + sg.tilted_irradiation_wh_m2, + sg.module_temperature_dc, + sg.energy_output_kwh +from {{ ref('raw_solargis_satellite_readings__temp_and_pv_energy') }} as sg + left join {{ ref('raw_plantmonitor_plants') }} using (plant_id) + left join + {{ ref('raw_gestio_actius_plant_parameters') }} as p + using (plant_uuid) -- SolarGis PVOUT (aquí photovoltaic_energy_output_wh) retorna l'energia en kwh però plantmonitor per error ho registra com a wh sense fer cap transformació. -- Entenem que al redash s'està corregint a mà abans de mostrar el valor. diff --git a/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__hourly.sql b/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__hourly.sql index b981a98c..00241ee2 100644 --- a/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__hourly.sql +++ b/dbt_jardiner/models/jardiner/intermediate/solargis/int_satellite_readings__hourly.sql @@ -2,23 +2,23 @@ materialized = 'view' ) }} -WITH satellite AS ( +with satellite as ( - SELECT - *, - ROW_NUMBER() OVER ( - PARTITION BY plant_id, - start_hour - ORDER BY - request_time DESC - ) AS ranking - FROM - {{ ref('int_satellite_readings__denormalized') }} + select + *, + row_number() over ( + partition by + plant_id, + start_hour + order by + request_time desc + ) as ranking + from + {{ ref('int_satellite_readings__denormalized') }} ) -SELECT - * -FROM - satellite -WHERE - ranking = 1 +select * +from + satellite +where + ranking = 1 diff --git a/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview__produccio_cartera.sql b/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview__produccio_cartera.sql index 73bea167..3e2b1627 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview__produccio_cartera.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview__produccio_cartera.sql @@ -1,21 +1,33 @@ {{ config(materialized='view') }} with spine as ( - select generate_series(current_date::timestamptz, current_date + interval '1 day - 1 hour', '1 hour') as start_hour -), today_hourly as ( - select - hora_inici, - tecnologia, - sum(energia_instantania_inversor_kwh) as energia_instantania_inversor_kwh, - sum(energia_exportada_instantania_comptador_kwh) as energia_exportada_instantania_comptador_kwh, - max(preu_omie_eur_mwh) as preu_omie_eur_mwh - from {{ ref("dm_plant_production_hourly") }} - where current_date <= hora_inici and hora_inici < current_date + interval '1 day' - group by hora_inici, tecnologia + select + generate_series( + current_date::timestamptz, + current_date + interval '1 day - 1 hour', + '1 hour' + ) as start_hour +), + +today_hourly as ( + select + hora_inici, + tecnologia, + sum( + energia_instantania_inversor_kwh + ) as energia_instantania_inversor_kwh, + sum( + energia_exportada_instantania_comptador_kwh + ) as energia_exportada_instantania_comptador_kwh, + max(preu_omie_eur_mwh) as preu_omie_eur_mwh + from {{ ref("dm_plant_production_hourly") }} + where + current_date <= hora_inici + and hora_inici < current_date + interval '1 day' + group by hora_inici, tecnologia ) -select - * -from spine -left join today_hourly - on today_hourly.hora_inici = spine.start_hour -order by start_hour desc + +select spine.*, today_hourly.* from spine + left join today_hourly + on spine.start_hour = today_hourly.hora_inici +order by spine.start_hour desc diff --git a/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_instant.sql b/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_instant.sql index 394f5cf1..f53c42a1 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_instant.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_instant.sql @@ -7,15 +7,36 @@ select overview.peak_power_kw as potencia_pic_kw, overview.nominal_power_kw as potencia_nominal_kw, overview.irradiance_w_m2_last_registered_at as irradiancia_ts, - round(overview.irradiance_w_m2::numeric, 2) as irradiancia_w_m2, overview.instant_power_plant_kw_registered_at as potencia_inst_ts, - round(overview.instant_power_plant_kw::numeric, 2) as potencia_inst_kw, - round((overview.instant_power_plant_kw / overview.nominal_power_kw)::numeric, 2) as potencia_inst_vs_nominal, overview.day as dia, + overview.instant_power_source as font_potencia_instantania, + round(overview.irradiance_w_m2::numeric, 2) as irradiancia_w_m2, + round(overview.instant_power_plant_kw::numeric, 2) as potencia_inst_kw, + round( + (overview.instant_power_plant_kw / overview.nominal_power_kw)::numeric, + 2 + ) as potencia_inst_vs_nominal, case when overview.day > now() - interval '2 days' then - round(((overview.meter_exported_energy_kwh - overview.solargis_meter_expected_energy_kwh) / 1000)::numeric, 2) + round( + ( + ( + overview.meter_exported_energy_kwh + - overview.solargis_meter_expected_energy_kwh + ) + / 1000 + )::numeric, + 2 + ) end as energia_perduda_ahir_mwh, - round(((overview.meter_exported_energy_kwh - overview.solargis_meter_expected_energy_kwh) / 1000)::numeric, 2) as energia_perduda_mwh, - overview.instant_power_source as font_potencia_instantania + round( + ( + ( + overview.meter_exported_energy_kwh + - overview.solargis_meter_expected_energy_kwh + ) + / 1000 + )::numeric, + 2 + ) as energia_perduda_mwh from {{ ref("int_plants_overview_instant") }} as overview diff --git a/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_monthly.sql b/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_monthly.sql index c55f9476..8d2fb94d 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_monthly.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_dashboard_overview_monthly.sql @@ -5,11 +5,17 @@ with omie_price_windowed_h_avg as ( select start_hour, price as price_omie_eur_mwh, - avg(price) over (partition by extract(year from start_hour) order by start_hour) as price_eur_mwh_year_average, + avg(price) + over ( + partition by extract(year from start_hour) order by start_hour + ) + as price_eur_mwh_year_average, date_trunc('month', start_hour, 'Europe/Madrid') as month - from {{ ref('raw_plantlake_omie_historical_price__with_row_number_per_date') }} + from + {{ ref('raw_plantlake_omie_historical_price__with_row_number_per_date') }} order by start_hour desc ), + last_avg_price as ( select distinct on (month) month, @@ -19,6 +25,7 @@ last_avg_price as ( from omie_price_windowed_h_avg order by month desc, start_hour desc ), + production_monthly_w_target as ( select production_target.month as mes, @@ -37,13 +44,22 @@ production_monthly_w_target as ( over plant_year_window as cumsum_energia_exportada_comptador_mwh, sum(production_target.energy_production_target_mwh) - over (partition by plant_uuid, extract(year from production_target.month) order by production_target.month) + over ( + partition by + plant_uuid, extract(year from production_target.month) + order by production_target.month + ) as cumsum_energia_objectiu_mwh from {{ ref("int_gda_plants__plants_catalog") }} as plant_catalog left join {{ ref("int_production_target__monthly") }} as production_target using (plant_uuid) left join {{ ref("dm_plant_production_monthly") }} as production_monthly using (plant_uuid, month) left join last_avg_price as omie_price using (month) - window plant_year_window as (partition by plant_uuid, extract(year from production_monthly.month) order by production_monthly.month) + window + plant_year_window as ( + partition by plant_uuid, extract(year from production_monthly.month) + order by production_monthly.month + ) order by production_target.month desc, plant_catalog.plant_name desc ) + select * from production_monthly_w_target diff --git a/dbt_jardiner/models/jardiner/marts/dm_dset__all_signals_received_per_plant.sql b/dbt_jardiner/models/jardiner/marts/dm_dset__all_signals_received_per_plant.sql index a9b70a2d..0d9902c9 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_dset__all_signals_received_per_plant.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_dset__all_signals_received_per_plant.sql @@ -2,16 +2,19 @@ select - nom_planta, - nom_planta_dset, - count(*) filter ( - where uuid_senyal is not null and id_senyal_dset is not null and ultim_ts is not null - ) as senyals_rebudes, - count(*) filter ( - where uuid_senyal is not null and (id_senyal_dset is null or ultim_ts is null) - ) as senyals_no_rebudes, - count(*) filter (where uuid_senyal is not null) as senyals_esperades, - count(*) as total_count -from {{ref('dm_dset_last_reading__from_signal_last_ts')}} + nom_planta, + nom_planta_dset, + count(*) filter ( + where uuid_senyal is not null + and id_senyal_dset is not null + and ultim_ts is not null + ) as senyals_rebudes, + count(*) filter ( + where uuid_senyal is not null + and (id_senyal_dset is null or ultim_ts is null) + ) as senyals_no_rebudes, + count(*) filter (where uuid_senyal is not null) as senyals_esperades, + count(*) as total_count +from {{ ref('dm_dset_last_reading__from_signal_last_ts') }} group by nom_planta, nom_planta_dset order by nom_planta asc, nom_planta_dset asc diff --git a/dbt_jardiner/models/jardiner/marts/dm_dset_energy_inverter__monthly__reset_daily.sql b/dbt_jardiner/models/jardiner/marts/dm_dset_energy_inverter__monthly__reset_daily.sql index d8619452..0f6aec9d 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_dset_energy_inverter__monthly__reset_daily.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_dset_energy_inverter__monthly__reset_daily.sql @@ -1,23 +1,27 @@ {{ config(materialized="view") }} with - inverter_energy_monthly as ( - select - date_trunc('month', day, 'Europe/Madrid') as month_date, - plant_name, - device_name, - coalesce(sum(inverter_total_energy_kwh), sum(inverter_export_energy_kwh))/1000 as inverter_energy_mwh - from {{ ref("int_dset_energy_inverter__daily__reset_daily") }} - group by - date_trunc('month', day, 'Europe/Madrid'), - plant_name, - device_name +inverter_energy_monthly as ( + select + plant_name, + device_name, + date_trunc('month', day, 'Europe/Madrid') as month_date, + coalesce( + sum(inverter_total_energy_kwh), sum(inverter_export_energy_kwh) ) - select - month_date as mes, - plant_name as nom_planta, - device_name as aparell, - inverter_energy_mwh as energia_inversor_mwh - from inverter_energy_monthly - where month_date > '2023-12-01' - order by month_date desc, plant_name, device_name + / 1000 as inverter_energy_mwh + from {{ ref("int_dset_energy_inverter__daily__reset_daily") }} + group by + date_trunc('month', day, 'Europe/Madrid'), + plant_name, + device_name +) + +select + month_date as mes, + plant_name as nom_planta, + device_name as aparell, + inverter_energy_mwh as energia_inversor_mwh +from inverter_energy_monthly +where month_date > '2023-12-01' +order by month_date desc, plant_name asc, device_name asc diff --git a/dbt_jardiner/models/jardiner/marts/dm_plant_production_daily.sql b/dbt_jardiner/models/jardiner/marts/dm_plant_production_daily.sql index 7354f017..969cc862 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_plant_production_daily.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_plant_production_daily.sql @@ -2,38 +2,53 @@ select - date_trunc('day', hora_inici, 'Europe/Madrid')::date as dia, - plant_uuid, - nom_planta, - tecnologia, - potencia_pic_kw, - sum(energia_instantania_inversor_kwh) as energia_instantania_inversor_kwh, - count(energia_instantania_inversor_kwh) as energia_inversor_count, - sum(energia_exportada_instantania_comptador_kwh) as energia_exportada_instantania_comptador_kwh, - count(energia_exportada_instantania_comptador_kwh) as energia_exportada_instantania_comptador_count, - sum(energia_exportada_comptador_kwh) as energia_exportada_comptador_kwh, - sum(energia_importada_comptador_kwh) as energia_importada_comptador_kwh, - min(data_prediccio) as data_prediccio, - sum(energia_predita_meteologica_kwh) as energia_predita_meteologica_kwh, - sum(energia_esperada_solargis_kwh) as energia_esperada_solargis_kwh, - sum(irradiation_wh_m2) as irradiation_wh_m2, - avg(preu_omie_eur_mwh) as preu_omie_eur_mwh, - sum(irradiacio_satellit_wh_m2) as irradiacio_satellit_wh_m2, - avg(temperatura_modul_avg_c) as temperatura_modul_avg_c, - -- fmt: off - sum(energia_exportada_comptador_kwh / potencia_pic_kw) / (nullif(sum(irradiacio_satellit_wh_m2), 0.0) / 1000.0) as pr, - -- fmt: on - sum(hora_disponible) as hora_disponible, - sum(hora_total) as hora_total, - sum(hora_disponible) / nullif(sum(hora_total), 0) as disponibilitat, - sum(energia_desviada_omie_kwh) as energia_desviada_omie_kwh, - sum(abs(energia_desviada_omie_kwh)) as energia_desviada_omie_kwh_absolute, {# should it be abs or we let compensate itself? #} - sum(energia_perduda_kwh) as energia_perduda_kwh, - -- fmt: off - 1 - sum(energia_predita_meteologica_kwh) / nullif(sum(energia_exportada_comptador_kwh), 0) as energia_desviada_percent - -- fmt: on + date_trunc('day', hora_inici, 'Europe/Madrid')::date as dia, + plant_uuid, + nom_planta, + tecnologia, + potencia_pic_kw, + sum(energia_instantania_inversor_kwh) as energia_instantania_inversor_kwh, + count(energia_instantania_inversor_kwh) as energia_inversor_count, + sum( + energia_exportada_instantania_comptador_kwh + ) as energia_exportada_instantania_comptador_kwh, + count( + energia_exportada_instantania_comptador_kwh + ) as energia_exportada_instantania_comptador_count, + sum(energia_exportada_comptador_kwh) as energia_exportada_comptador_kwh, + sum(energia_importada_comptador_kwh) as energia_importada_comptador_kwh, + min(data_prediccio) as data_prediccio, + sum(energia_predita_meteologica_kwh) as energia_predita_meteologica_kwh, + sum(energia_esperada_solargis_kwh) as energia_esperada_solargis_kwh, + sum(irradiation_wh_m2) as irradiation_wh_m2, + avg(preu_omie_eur_mwh) as preu_omie_eur_mwh, + sum(irradiacio_satellit_wh_m2) as irradiacio_satellit_wh_m2, + avg(temperatura_modul_avg_c) as temperatura_modul_avg_c, + -- fmt: off + sum(energia_exportada_comptador_kwh / potencia_pic_kw) + / (nullif(sum(irradiacio_satellit_wh_m2), 0.0) / 1000.0) as pr, + -- fmt: on + sum(hora_disponible) as hora_disponible, + sum(hora_total) as hora_total, + sum(hora_disponible) / nullif(sum(hora_total), 0) as disponibilitat, + sum(energia_desviada_omie_kwh) as energia_desviada_omie_kwh, + sum(abs(energia_desviada_omie_kwh)) as energia_desviada_omie_kwh_absolute, + {# should it be abs or we let compensate itself? #} + sum(energia_perduda_kwh) as energia_perduda_kwh, + -- fmt: off + 1 + - sum(energia_predita_meteologica_kwh) + / nullif( + sum(energia_exportada_comptador_kwh), 0 + ) as energia_desviada_percent +-- fmt: on {# HMCIL #} {# billed_energy as energia_liquidada, sum(billed_energy)/sum(dset_meter_exported_energy_kwh) as energia_liquidada_percent#} from {{ ref("dm_plant_production_hourly") }} -- fmt: on -group by date_trunc('day', hora_inici, 'Europe/Madrid'), plant_uuid, nom_planta, tecnologia, potencia_pic_kw +group by + date_trunc('day', hora_inici, 'Europe/Madrid'), + plant_uuid, + nom_planta, + tecnologia, + potencia_pic_kw diff --git a/dbt_jardiner/models/jardiner/marts/dm_plant_production_monthly.sql b/dbt_jardiner/models/jardiner/marts/dm_plant_production_monthly.sql index 562e879c..2c13d778 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_plant_production_monthly.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_plant_production_monthly.sql @@ -2,33 +2,64 @@ select - date_trunc('month', dia) as "month", plant_uuid, nom_planta as nom_planta, tecnologia as tecnologia, potencia_pic_kw as potencia_pic_kw, - sum(energia_instantania_inversor_kwh) / 1000.0 as energia_instantania_inversor_mwh, - count(energia_instantania_inversor_kwh) as energia_inversor_count, - (sum(energia_exportada_instantania_comptador_kwh) / 1000.0)::numeric as energia_exportada_instantania_comptador_mwh, - count(energia_exportada_instantania_comptador_kwh) as energia_exportada_instantania_comptador_count, - (sum(energia_exportada_comptador_kwh) / 1000.0)::numeric as energia_exportada_comptador_mwh, - (sum(energia_importada_comptador_kwh) / 1000.0)::numeric as energia_importada_comptador_mwh, + ( + sum(energia_exportada_instantania_comptador_kwh) / 1000.0 + )::numeric as energia_exportada_instantania_comptador_mwh, + ( + sum(energia_exportada_comptador_kwh) / 1000.0 + )::numeric as energia_exportada_comptador_mwh, + ( + sum(energia_importada_comptador_kwh) / 1000.0 + )::numeric as energia_importada_comptador_mwh, + ( + sum(energia_predita_meteologica_kwh) / 1000.0 + )::numeric as energia_predita_meteologica_mwh, + ( + sum(energia_esperada_solargis_kwh) / 1000.0 + )::numeric as energia_esperada_solargis_mwh, + ( + sum(energia_exportada_comptador_kwh / potencia_pic_kw) + / (nullif(sum(irradiacio_satellit_wh_m2), 0.0) / 1000.0) + )::numeric as pr, + ( + sum(energia_exportada_comptador_kwh) / potencia_pic_kw + )::numeric as hores_equivalents, --min(data_prediccio) as data_prediccio, - (sum(energia_predita_meteologica_kwh) / 1000.0)::numeric as energia_predita_meteologica_mwh, - (sum(energia_esperada_solargis_kwh) / 1000.0)::numeric as energia_esperada_solargis_mwh, + ( + sum(energia_desviada_omie_kwh) / 1000.0 + )::numeric as energia_desviada_omie_mwh, + ( + sum(abs(energia_desviada_omie_kwh_absolute)) / 1000.0 + )::numeric as energia_desviada_omie_mwh_absolute, + (sum(energia_perduda_kwh) / 1000.0)::numeric as energia_perduda_mwh, + ( + (sum(energia_desviada_omie_kwh_absolute) / 1000.0)::numeric + / nullif((sum(energia_exportada_comptador_kwh) / 1000.0)::numeric, 0) + )::numeric as energia_desviada_percent, + date_trunc('month', dia) as "month", + --avg(temperatura_modul_avg_c) as temperatura_modul_avg_c, + sum(energia_instantania_inversor_kwh) + / 1000.0 as energia_instantania_inversor_mwh, + count(energia_instantania_inversor_kwh) as energia_inversor_count, + count( + energia_exportada_instantania_comptador_kwh + ) as energia_exportada_instantania_comptador_count, avg(preu_omie_eur_mwh) as preu_omie_eur_mwh, sum(irradiation_wh_m2) as irradiation_wh_m2, sum(irradiacio_satellit_wh_m2) as irradiacio_satellit_wh_m2, - --avg(temperatura_modul_avg_c) as temperatura_modul_avg_c, - (sum(energia_exportada_comptador_kwh / potencia_pic_kw) / (nullif(sum(irradiacio_satellit_wh_m2), 0.0) / 1000.0))::numeric as pr, - (sum(energia_exportada_comptador_kwh) / potencia_pic_kw)::numeric as hores_equivalents, sum(hora_disponible) as hora_disponible, + {# should it be abs or we let compensate itself? #} sum(hora_total) as hora_total, - sum(hora_disponible) / nullif(sum(hora_total), 0) as disponibilitat, - (sum(energia_desviada_omie_kwh) / 1000.0)::numeric as energia_desviada_omie_mwh, - (sum(abs(energia_desviada_omie_kwh_absolute)) / 1000.0)::numeric as energia_desviada_omie_mwh_absolute, {# should it be abs or we let compensate itself? #} - (sum(energia_perduda_kwh) / 1000.0)::numeric as energia_perduda_mwh, - ((sum(energia_desviada_omie_kwh_absolute) / 1000.0)::numeric / nullif((sum(energia_exportada_comptador_kwh) / 1000.0)::numeric, 0))::numeric as energia_desviada_percent + sum(hora_disponible) / nullif(sum(hora_total), 0) as disponibilitat from {{ ref("dm_plant_production_daily") }} -group by date_trunc('month', dia), plant_uuid, nom_planta, tecnologia, potencia_pic_kw +group by + date_trunc('month', dia), + plant_uuid, + nom_planta, + tecnologia, + potencia_pic_kw order by month desc diff --git a/dbt_jardiner/models/jardiner/marts/dm_plants.sql b/dbt_jardiner/models/jardiner/marts/dm_plants.sql index dee7895f..f19be307 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_plants.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_plants.sql @@ -1,25 +1,25 @@ {{ config(materialize='view') }} select - plant_uuid, - plant_name as nom_planta, - municipality as municipi, - province as provincia, - latitude as latitud, - longitude as longitud, - peak_power_kw as potencia_pic_kw, - nominal_power_kw as potencia_nominal_kw, - technology as tecnologia, - subtechnology as subtecnologia, - connection_date as connexio, - owner as propietat, - maintenance_company as empresa_manteniment, - manager as responsable_ga, - n_strings_plant as "strings/planta", - n_modules_total as modules_total,--"moduls/string", - n_inverters as inversors,--"strings/inversor", - n_modules_total, - n_inverters, - esquema_unifilar, - layout + plant_uuid, + plant_name as nom_planta, + municipality as municipi, + province as provincia, + latitude as latitud, + longitude as longitud, + peak_power_kw as potencia_pic_kw, + nominal_power_kw as potencia_nominal_kw, + technology as tecnologia, + subtechnology as subtecnologia, + connection_date as connexio, + owner as propietat, + maintenance_company as empresa_manteniment, + manager as responsable_ga, + n_strings_plant as "strings/planta", -- noqa: RF05 + n_modules_total as modules_total,--"moduls/string", + n_inverters as inversors,--"strings/inversor", + n_modules_total, + n_inverters, + esquema_unifilar, + layout from {{ ref('int_gda_plants__plants_catalog') }} diff --git a/dbt_jardiner/models/jardiner/marts/int_dset_energy_inverter__daily__reset_daily.sql b/dbt_jardiner/models/jardiner/marts/int_dset_energy_inverter__daily__reset_daily.sql index 64a54c6a..41391a26 100644 --- a/dbt_jardiner/models/jardiner/marts/int_dset_energy_inverter__daily__reset_daily.sql +++ b/dbt_jardiner/models/jardiner/marts/int_dset_energy_inverter__daily__reset_daily.sql @@ -1,61 +1,65 @@ {{ config(materialized="view") }} with - inverter_energy_daily_metrics as ( - select - date_trunc('day', ts, 'Europe/Madrid') as day, - plant_uuid, - plant_name, - device_uuid, - device_name, - metric_name, - max(inverter_energy_kwh) as max_inverter_energy_kwh, - min(inverter_energy_kwh) as min_inverter_energy_kwh, - max(inverter_energy_kwh) - min(inverter_energy_kwh) as diff_inverter_energy_kwh, - max(inverter_energy_kwh) - min(nullif(inverter_energy_kwh, 0)) as diff_nozero_inverter_energy_kwh - from {{ ref("int_dset_energy_inverter__5m") }} - where - metric_name = 'energia_activa_exportada' - {# filter to discard previous day data which gets reset early morning #} - and extract(hour from ts at time zone 'Europe/Madrid') > 3 - or metric_name = 'energia_activa_exportada_total' - group by - date_trunc('day', ts, 'Europe/Madrid'), - plant_uuid, - plant_name, - device_uuid, - device_name, - metric_name - ), - inverter_energy_daily as ( - select - day, - plant_name, - device_name, - case - when metric_name = 'energia_activa_exportada' - then diff_inverter_energy_kwh - else null - end as inverter_export_energy_kwh, - case - when metric_name = 'energia_activa_exportada_total' - then diff_inverter_energy_kwh - else null - end as inverter_total_energy_kwh - from inverter_energy_daily_metrics - order by day desc, plant_name, device_name - ), - all_inverter_energy_metrics_in_one_tb as ( - select - day, - plant_name, - device_name, - max(inverter_export_energy_kwh) as inverter_export_energy_kwh, - max(inverter_total_energy_kwh) as inverter_total_energy_kwh - from inverter_energy_daily - group by day, plant_name, device_name - ) +inverter_energy_daily_metrics as ( + select + plant_uuid, + plant_name, + device_uuid, + device_name, + metric_name, + date_trunc('day', ts, 'Europe/Madrid') as day, + max(inverter_energy_kwh) as max_inverter_energy_kwh, + min(inverter_energy_kwh) as min_inverter_energy_kwh, + max(inverter_energy_kwh) + - min(inverter_energy_kwh) as diff_inverter_energy_kwh, + max(inverter_energy_kwh) + - min(nullif(inverter_energy_kwh, 0)) as diff_nozero_inverter_energy_kwh + from {{ ref("int_dset_energy_inverter__5m") }} + where + metric_name = 'energia_activa_exportada' + {# filter to discard previous day data which gets reset early morning #} + and extract(hour from ts at time zone 'Europe/Madrid') > 3 + or metric_name = 'energia_activa_exportada_total' + group by + date_trunc('day', ts, 'Europe/Madrid'), + plant_uuid, + plant_name, + device_uuid, + device_name, + metric_name +), + +inverter_energy_daily as ( + select + day, + plant_name, + device_name, + case + when metric_name = 'energia_activa_exportada' + then diff_inverter_energy_kwh + end as inverter_export_energy_kwh, + case + when metric_name = 'energia_activa_exportada_total' + then diff_inverter_energy_kwh + end as inverter_total_energy_kwh + from inverter_energy_daily_metrics + order by day desc, plant_name asc, device_name asc +), + +all_inverter_energy_metrics_in_one_tb as ( + select + day, + plant_name, + device_name, + max(inverter_export_energy_kwh) as inverter_export_energy_kwh, + max(inverter_total_energy_kwh) as inverter_total_energy_kwh + from inverter_energy_daily + group by day, plant_name, device_name +) + select * from all_inverter_energy_metrics_in_one_tb -where day > '2023-12-15' --data aprox en que dset transforma correctament aquestes mètriques -order by day desc, plant_name, device_name +--data aprox en que dset transforma correctament aquestes mètriques +where day > '2023-12-15' +order by day desc, plant_name asc, device_name asc diff --git a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__check_if_we_receive_data_for_signals.sql b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__check_if_we_receive_data_for_signals.sql index ff4f5b58..8e369f28 100644 --- a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__check_if_we_receive_data_for_signals.sql +++ b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__check_if_we_receive_data_for_signals.sql @@ -1,26 +1,31 @@ {{ config(materialized="view") }} with - valors as ( - select true as rebut_from_dset, signal_uuid, ts, queried_at - from {{ ref("int_dset_responses__materialized") }} - where ts = (select max(ts) from {{ ref("int_dset_responses__materialized") }}) - ), +valors as ( + select + true as rebut_from_dset, + signal_uuid, + ts, + queried_at + from {{ ref("int_dset_responses__materialized") }} + where + ts = (select max(ts) from {{ ref("int_dset_responses__materialized") }}) +), - metadata as ( - select - valors.ts, - valors.queried_at, - signals.plant_name, - signals.signal_name, - signals.signal_uuid, - signals.device_name, - signals.device_type, - signals.device_uuid, - coalesce(valors.rebut_from_dset, false) as rebut_from_dset - from {{ ref('raw_gestio_actius__signal_denormalized') }} as signals +metadata as ( + select + valors.ts, + valors.queried_at, + signals.plant_name, + signals.signal_name, + signals.signal_uuid, + signals.device_name, + signals.device_type, + signals.device_uuid, + coalesce(valors.rebut_from_dset, false) as rebut_from_dset + from {{ ref('raw_gestio_actius__signal_denormalized') }} as signals left join valors on signals.signal_uuid = valors.signal_uuid - order by signals.plant_name, signals.signal_name - ) + order by signals.plant_name, signals.signal_name +) select * from metadata diff --git a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__expected_signals.sql b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__expected_signals.sql index 6bba9112..97e8e220 100644 --- a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__expected_signals.sql +++ b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__expected_signals.sql @@ -6,13 +6,21 @@ with esperades as ( from {{ ref("dm_dset_last_reading__from_signal_last_ts") }} where uuid_senyal is not null ) + select nom_planta, count(*) as senyals_esperades, count(*) filter (where ultim_ts is null) as no_rebudes_mai, count(*) filter (where ultim_ts is not null) as rebudes_alguna_vegada, - count(*) filter (where ultim_ts is not null and ultim_ts >= current_date) as rebudes_avui, - count(*) filter (where ultim_ts is not null and ultim_ts < current_date) as no_rebudes_avui, - count(*) filter (where ultim_ts is null or (ultim_ts is not null and ultim_ts < current_date)) as no_rebudes_total + count(*) filter ( + where ultim_ts is not null and ultim_ts >= current_date + ) as rebudes_avui, + count(*) filter ( + where ultim_ts is not null and ultim_ts < current_date + ) as no_rebudes_avui, + count(*) filter ( + where ultim_ts is null + or (ultim_ts is not null and ultim_ts < current_date) + ) as no_rebudes_total from esperades group by nom_planta diff --git a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__instant_power_comparison.sql b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__instant_power_comparison.sql index ecfad299..60147288 100644 --- a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__instant_power_comparison.sql +++ b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__instant_power_comparison.sql @@ -11,6 +11,7 @@ with inverter_power as ( where ts > now() - interval '7 days' and metric_name = 'potencia_activa' group by plant_uuid, plant_name, ts ), + instant_power_comparison as ( select plant_uuid, @@ -18,18 +19,17 @@ instant_power_comparison as ( ts, inverter_power.inverter_power_kw, instant_power.inferred_meter_power_kw, - round(abs(inverter_power.inverter_power_kw - instant_power.inferred_meter_power_kw)::numeric, 2) as delta_power_instant_vs_inferred, inverter_power.num_inverters, instant_power.meter_energy_kwh, instant_power.previous_energy, instant_power.delta_meter_energy_kwh, instant_power.previous_ts, - instant_power.delta_ts + instant_power.delta_ts, + round(abs(inverter_power.inverter_power_kw - instant_power.inferred_meter_power_kw)::numeric, 2) as delta_power_instant_vs_inferred from {{ ref("int_dset__power_from_instant_energy") }} as instant_power - left join - inverter_power - using (plant_uuid, ts) + left join inverter_power using (plant_uuid, ts) ) + select plant_uuid as uuid_planta, plant_name as nom_planta, diff --git a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__unexpected_signals.sql b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__unexpected_signals.sql index d0cc2019..7e8e4b55 100644 --- a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__unexpected_signals.sql +++ b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__unexpected_signals.sql @@ -6,6 +6,9 @@ with inesperades as ( from {{ ref("dm_dset_last_reading__from_signal_last_ts") }} where uuid_senyal is null ) -select nom_planta_dset, count(*) as senyals_inesperades + +select + nom_planta_dset, + count(*) as senyals_inesperades from inesperades group by nom_planta_dset diff --git a/dbt_jardiner/models/jardiner/raw/dset/raw_dset_meter__readings.sql b/dbt_jardiner/models/jardiner/raw/dset/raw_dset_meter__readings.sql index 2ceabc6e..855add4c 100644 --- a/dbt_jardiner/models/jardiner/raw/dset/raw_dset_meter__readings.sql +++ b/dbt_jardiner/models/jardiner/raw/dset/raw_dset_meter__readings.sql @@ -5,7 +5,6 @@ with meter_readings as ( group_name as dset_plant_name, signal_device_external_description as signal_device_type, signal_description, - (ts::timestamp at time zone signal_tz) - signal_frequency::interval as start_ts, ts::timestamp at time zone signal_tz as end_ts, signal_value, group_id as dset_plant_id, @@ -19,11 +18,14 @@ with meter_readings as ( signal_last_ts::timestamp at time zone signal_tz as signal_last_ts, signal_last_value, signal_unit, + signal_external_id as signal_uuid_raw, + signal_device_external_id as signal_device_uuid, + (ts::timestamp at time zone signal_tz) + - signal_frequency::interval as start_ts, case when signal_external_id ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then signal_external_id::uuid -- noqa: LT01 - end as signal_uuid, - signal_external_id as signal_uuid_raw, - signal_device_external_id as signal_device_uuid + end as signal_uuid from {{ source("lake", "dset_meters_readings") }} ) + select * from meter_readings diff --git a/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__api_response.sql b/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__api_response.sql index 3c306890..f4882d27 100644 --- a/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__api_response.sql +++ b/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__api_response.sql @@ -17,21 +17,78 @@ unnested_dset_response as ( select group_name, queried_at, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_id')::text as signal_id, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_tz')::text as signal_tz, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_code')::text as signal_code, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_type')::text as signal_type, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_unit')::text as signal_unit, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_last_ts')::timestamp at time zone (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_tz')::text as signal_last_ts, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_frequency')::text as signal_frequency, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_is_virtual')::boolean as signal_is_virtual, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_last_value')::numeric as signal_last_value, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_description')::text as signal_description, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_device_external_id')::text as signal_device_uuid, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_device_external_description')::text as signal_device_type, - (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_external_id')::text as signal_uuid, - (jsonb_array_elements(jsonb_array_elements((response)::jsonb #> '{signals}') #> '{data}') ->> 'ts')::timestamp at time zone (jsonb_array_elements((response)::jsonb #> '{signals}') ->> 'signal_tz')::text as ts, - (jsonb_array_elements(jsonb_array_elements((response)::jsonb #> '{signals}') #> '{data}') ->> 'value')::numeric as signal_value + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_id' + )::text as signal_id, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_tz' + )::text as signal_tz, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_code' + )::text as signal_code, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_type' + )::text as signal_type, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_unit' + )::text as signal_unit, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_last_ts' + )::timestamp at time zone ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_tz' + )::text as signal_last_ts, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_frequency' + )::text as signal_frequency, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_is_virtual' + )::boolean as signal_is_virtual, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_last_value' + )::numeric as signal_last_value, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_description' + )::text as signal_description, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_device_external_id' + )::text as signal_device_uuid, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_device_external_description' + )::text as signal_device_type, + ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_external_id' + )::text as signal_uuid, + ( + jsonb_array_elements( + jsonb_array_elements((response)::jsonb #> '{signals}') + #> '{data}' + ) + ->> 'ts' + )::timestamp at time zone ( + jsonb_array_elements((response)::jsonb #> '{signals}') + ->> 'signal_tz' + )::text as ts, + ( + jsonb_array_elements( + jsonb_array_elements((response)::jsonb #> '{signals}') + #> '{data}' + ) + ->> 'value' + )::numeric as signal_value from filter_dummy_group ) diff --git a/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__last_signal_reading.sql b/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__last_signal_reading.sql index 83db7f85..cc2e1c5d 100644 --- a/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__last_signal_reading.sql +++ b/dbt_jardiner/models/jardiner/raw/dset/raw_dset_responses__last_signal_reading.sql @@ -9,37 +9,42 @@ last_dset_response as ( unnest_groups as ( select - query_time as queried_at, - (jsonb_array_elements(response) ->> 'signals')::jsonb as signals, - jsonb_array_elements(response) ->> 'group_name' as group_name + query_time as queried_at, + (jsonb_array_elements(response) ->> 'signals')::jsonb as signals, + jsonb_array_elements(response) ->> 'group_name' as group_name from last_dset_response ), filter_dummy_group as ( - select * from unnest_groups where group_name != 'SOMENERGIA' - ), + select * from unnest_groups where group_name != 'SOMENERGIA' +), unnested_dset_response as ( - select - group_name, - queried_at, - jsonb_array_elements(signals) as signals - from filter_dummy_group + select + group_name, + queried_at, + jsonb_array_elements(signals) as signals + from filter_dummy_group ) + select - group_name, - queried_at, - (signals ->> 'signal_id')::integer as signal_id, - (signals ->> 'signal_code')::text as signal_code, - (signals ->> 'signal_type')::text as signal_type, - (signals ->> 'signal_unit')::text as signal_unit, - (signals ->> 'signal_last_ts')::timestamp at time zone (signals ->> 'signal_tz')::text as signal_last_ts, - (signals ->> 'signal_tz')::text as signal_tz, - (signals ->> 'signal_frequency')::text as signal_frequency, - (signals ->> 'signal_is_virtual')::text as signal_is_virtual, - (signals ->> 'signal_last_value')::numeric as signal_last_value, - (signals ->> 'signal_description')::text as signal_description, - (signals ->> 'signal_device_external_id')::text as signal_device_uuid, - (signals ->> 'signal_device_external_description')::text as signal_device_type, - (signals ->> 'signal_external_id')::text as signal_uuid + group_name, + queried_at, + (signals ->> 'signal_id')::integer as signal_id, + (signals ->> 'signal_code')::text as signal_code, + (signals ->> 'signal_type')::text as signal_type, + (signals ->> 'signal_unit')::text as signal_unit, + (signals ->> 'signal_last_ts')::timestamp at time zone ( + signals ->> 'signal_tz' + )::text as signal_last_ts, + (signals ->> 'signal_tz')::text as signal_tz, + (signals ->> 'signal_frequency')::text as signal_frequency, + (signals ->> 'signal_is_virtual')::text as signal_is_virtual, + (signals ->> 'signal_last_value')::numeric as signal_last_value, + (signals ->> 'signal_description')::text as signal_description, + (signals ->> 'signal_device_external_id')::text as signal_device_uuid, + ( + signals ->> 'signal_device_external_description' + )::text as signal_device_type, + (signals ->> 'signal_external_id')::text as signal_uuid from unnested_dset_response diff --git a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql index b9ff5e1f..7782d4a0 100644 --- a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql +++ b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql @@ -10,18 +10,18 @@ fresh as ( base as ( select plant_uuid::uuid, - trim(plant::text) as plant_name, - trim(signal::text) as signal_name, metric::text as metric_name, - trim(device::text) as device_name, device_type::text, device_uuid::uuid, - trim(device_parent::text) as device_parent, device_parent_uuid::uuid, signal_uuid::uuid, - coalesce(is_enabled = 'VERDADERO', false) as is_enabled, inserted_at::timestamptz, - updated_at::timestamptz + updated_at::timestamptz, + trim(plant::text) as plant_name, + trim(signal::text) as signal_name, + trim(device::text) as device_name, + trim(device_parent::text) as device_parent, + coalesce(is_enabled = 'VERDADERO', false) as is_enabled from fresh ) diff --git a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_plant_parameters.sql b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_plant_parameters.sql index e9f5c97e..be224864 100644 --- a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_plant_parameters.sql +++ b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_plant_parameters.sql @@ -5,7 +5,6 @@ current_data as ( select plant_uuid::uuid, plant_id::numeric, - plant_id is not null as has_plantmonitor, planta as plant_name, latitud::numeric as latitude, longitut::numeric as longitude, @@ -27,9 +26,11 @@ current_data as ( data_actualitzacio::date as gestio_actius_updated_at, dbt_updated_at::date as dbt_updated_at, dbt_valid_from::date as dbt_valid_from, - coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to + coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to, + plant_id is not null as has_plantmonitor from {{ ref("snapshot_plant_parameters") }} ) + select * from current_data where dbt_valid_from::date <= current_date and current_date < dbt_valid_to::date diff --git a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_production_target.sql b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_production_target.sql index a1fec8cb..3de87b73 100644 --- a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_production_target.sql +++ b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius_production_target.sql @@ -2,9 +2,6 @@ select planta as plant_name, - case - when plant_uuid ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then plant_uuid::uuid -- noqa: LT01 - end as plant_uuid, enero::numeric as january, febrero::numeric as february, marzo::numeric as march, @@ -20,6 +17,9 @@ select data_actualitzacio::date as gestio_actius_updated_at, dbt_updated_at::date as dbt_updated_at, dbt_valid_from::date as dbt_valid_from, - coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to + coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to, + case + when plant_uuid ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then plant_uuid::uuid -- noqa: LT01 + end as plant_uuid from {{ ref('snapshot_production_target') }} where dbt_valid_to is null diff --git a/dbt_jardiner/models/jardiner/raw/meteologica/raw_energy_forecasts__denormalized_from_plantmonitordb.sql b/dbt_jardiner/models/jardiner/raw/meteologica/raw_energy_forecasts__denormalized_from_plantmonitordb.sql index d349b6da..a3a4de54 100644 --- a/dbt_jardiner/models/jardiner/raw/meteologica/raw_energy_forecasts__denormalized_from_plantmonitordb.sql +++ b/dbt_jardiner/models/jardiner/raw/meteologica/raw_energy_forecasts__denormalized_from_plantmonitordb.sql @@ -2,28 +2,21 @@ materialized = 'view' ) }} -WITH forecast_denormalized AS ( +with forecast_denormalized as ( - SELECT - forecast.time AS "time", - forecastmetadata.plant AS plant_id, - forecastmetadata.forecastdate AS forecastdate, - ROUND( - forecast.percentil50 / 1000.0, - 2 - ) AS energy_kwh - FROM - {{ source( - 'meteologica', - 'forecast' - ) }} - LEFT JOIN {{ source( - 'meteologica', - 'forecastmetadata' - ) }} - ON forecastmetadata.id = forecast.forecastmetadata + select + forecast.time as "time", + forecastmetadata.plant as plant_id, + forecastmetadata.forecastdate as forecastdate, + round( + forecast.percentil50 / 1000.0, + 2 + ) as energy_kwh + from {{ source('meteologica', 'forecast') }} as forecast + left join {{ source('meteologica', 'forecastmetadata') }} as forecastmetadata + on forecastmetadata.id = forecast.forecastmetadata ) -SELECT - * -FROM - forecast_denormalized + +select * +from + forecast_denormalized diff --git a/dbt_jardiner/models/jardiner/raw/omie/raw_plantlake_omie_historical_price__with_row_number_per_date.sql b/dbt_jardiner/models/jardiner/raw/omie/raw_plantlake_omie_historical_price__with_row_number_per_date.sql index 968adebc..e4149cd4 100644 --- a/dbt_jardiner/models/jardiner/raw/omie/raw_plantlake_omie_historical_price__with_row_number_per_date.sql +++ b/dbt_jardiner/models/jardiner/raw/omie/raw_plantlake_omie_historical_price__with_row_number_per_date.sql @@ -1,16 +1,18 @@ {{ config(materialized="view") }} with - recent as ( +recent as ( - select - "date", - create_time, - price, - row_number() over (partition by "date" order by create_time desc) as rc - from {{ source("plantlake", "omie_historical_price_hour") }} - ) + select + "date", + create_time, + price, + row_number() over (partition by "date" order by create_time desc) as rc + from {{ source("plantlake", "omie_historical_price_hour") }} +) -select "date" as start_hour, price +select + "date" as start_hour, + price from recent where rc = 1 diff --git a/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_meterregistry.sql b/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_meterregistry.sql index b1eabd92..f526c870 100644 --- a/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_meterregistry.sql +++ b/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_meterregistry.sql @@ -1,13 +1,13 @@ {{ config(materialized='view') }} -SELECT - mr.time - interval '1 hour' as start_hour, +select mr.export_energy_wh, mr.import_energy_wh, mr.meter as meter_id, meter.name as meter_name, meter.connection_protocol, - meter.plant as plant_id -FROM {{source('plantmonitor_jardiner','meterregistry')}} as mr -left join {{source('plantmonitor_jardiner','meter')}} on meter.id = mr.meter + meter.plant as plant_id, + mr.time - interval '1 hour' as start_hour +from {{ source('plantmonitor_jardiner','meterregistry') }} as mr + left join {{ source('plantmonitor_jardiner','meter') }} as meter on meter.id = mr.meter diff --git a/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_plantmonitor_plants.sql b/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_plantmonitor_plants.sql index d94549cd..a4846572 100644 --- a/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_plantmonitor_plants.sql +++ b/dbt_jardiner/models/jardiner/raw/plantmonitor_erp/raw_plantmonitor_plants.sql @@ -1,10 +1,10 @@ {{ config(materialized='view') }} select - plant.id as plant_id, - plant.name as plant_name, - plant.codename as plant_codename, - plant.description as plant_description, - plant.device_uuid as plant_uuid + plant.id as plant_id, + plant.name as plant_name, + plant.codename as plant_codename, + plant.description as plant_description, + plant.device_uuid as plant_uuid from {{ source('plantmonitor_jardiner','plant') }} as plant where plant.description != 'SomRenovables' diff --git a/dbt_jardiner/models/jardiner/raw/solar_events_ephem/raw_plantmonitordb_solarevent__generous.sql b/dbt_jardiner/models/jardiner/raw/solar_events_ephem/raw_plantmonitordb_solarevent__generous.sql index 01b7bfc2..4d5a3058 100644 --- a/dbt_jardiner/models/jardiner/raw/solar_events_ephem/raw_plantmonitordb_solarevent__generous.sql +++ b/dbt_jardiner/models/jardiner/raw/solar_events_ephem/raw_plantmonitordb_solarevent__generous.sql @@ -3,11 +3,20 @@ select plant as plant_id, - date_trunc('day', sunrise) as day, sunrise as sunrise_real, sunset as sunset_real, + date_trunc('day', sunrise) as day, sunrise + interval '2 hours' as sunrise_generous, sunset - interval '2 hours' as sunset_generous, - round(EXTRACT(EPOCH FROM (sunset - sunrise))::numeric/3600,2) as solar_hours_real, - round(EXTRACT(EPOCH FROM (sunset - interval '2 hours') - (sunrise + interval '2 hours'))::numeric/3600,2) as solar_hours_minimum -from {{source('plantmonitor_jardiner', 'solarevent')}} \ No newline at end of file + round( + extract(epoch from (sunset - sunrise))::numeric / 3600, 2 + ) as solar_hours_real, + round( + extract( + epoch from (sunset - interval '2 hours') + - (sunrise + interval '2 hours') + )::numeric + / 3600, + 2 + ) as solar_hours_minimum +from {{ source('plantmonitor_jardiner', 'solarevent') }} diff --git a/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__solargis.sql b/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__solargis.sql index 833c8ae3..fef4cf1b 100644 --- a/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__solargis.sql +++ b/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__solargis.sql @@ -2,12 +2,11 @@ select - date_trunc('hour', "time") as start_hour, - plant, - global_horizontal_irradiation_wh_m2, - global_tilted_irradiation_wh_m2, - module_temperature_dc*10 as module_temperature_c, - photovoltaic_energy_output_wh, - request_time + plant, + global_horizontal_irradiation_wh_m2, + global_tilted_irradiation_wh_m2, + photovoltaic_energy_output_wh, + request_time, + date_trunc('hour', "time") as start_hour, + module_temperature_dc * 10 as module_temperature_c from {{ source('solargis', 'satellite_readings') }} - diff --git a/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__temp_and_pv_energy.sql b/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__temp_and_pv_energy.sql index bd574125..e8423f7b 100644 --- a/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__temp_and_pv_energy.sql +++ b/dbt_jardiner/models/jardiner/raw/solargis/raw_solargis_satellite_readings__temp_and_pv_energy.sql @@ -1,15 +1,15 @@ {{ config(materialized='view') }} select - date_trunc('hour', "time") as start_hour, - plant as plant_id, - request_time as request_time, - global_horizontal_irradiation_wh_m2 as horizontal_irradiation_wh_m2, - global_tilted_irradiation_wh_m2 as tilted_irradiation_wh_m2, - module_temperature_dc as module_temperature_dc, - photovoltaic_energy_output_wh as energy_output_kwh -from {{ source('solargis','satellite_readings') }} sg + plant as plant_id, + request_time as request_time, + global_horizontal_irradiation_wh_m2 as horizontal_irradiation_wh_m2, + global_tilted_irradiation_wh_m2 as tilted_irradiation_wh_m2, + module_temperature_dc as module_temperature_dc, + photovoltaic_energy_output_wh as energy_output_kwh, + date_trunc('hour', "time") as start_hour +from {{ source('solargis','satellite_readings') }} -- SolarGis PVOUT (aquí photovoltaic_energy_output_wh) retorna l'energia en kwh però plantmonitor per error ho registra com a wh sense fer cap transformació. -- Entenem que al redash s'està corregint a mà abans de mostrar el valor. --- Aquí canviem el nom perquè s'ajusti a la realitat del valor. \ No newline at end of file +-- Aquí canviem el nom perquè s'ajusti a la realitat del valor. diff --git a/dbt_jardiner/models/jardiner/raw/utils/spine_hourly.sql b/dbt_jardiner/models/jardiner/raw/utils/spine_hourly.sql index 3a7c2f87..9ca35105 100644 --- a/dbt_jardiner/models/jardiner/raw/utils/spine_hourly.sql +++ b/dbt_jardiner/models/jardiner/raw/utils/spine_hourly.sql @@ -3,10 +3,15 @@ {%- set start_date = "'2015-01-01 00:00+00'" -%} with spine as ( - select generate_series({{start_date}}::timestamptz, now(), '1 hour') as start_hour + select + generate_series( + {{ start_date }}::timestamptz, now(), '1 hour' + ) as start_hour ) + select * from spine {%- if is_incremental() %} - where coalesce((select max(start_hour) from {{ this }}), {{start_date}}) < start_hour -{%- endif -%} \ No newline at end of file + where + coalesce((select max(start_hour) from {{ this }}), {{ start_date }}) < start_hour +{%- endif %} diff --git a/dbt_jardiner/models/jardiner/raw/utils/spine_monthly__full_year.sql b/dbt_jardiner/models/jardiner/raw/utils/spine_monthly__full_year.sql index b84ebb9c..7677ecfc 100644 --- a/dbt_jardiner/models/jardiner/raw/utils/spine_monthly__full_year.sql +++ b/dbt_jardiner/models/jardiner/raw/utils/spine_monthly__full_year.sql @@ -3,12 +3,20 @@ {%- set start_date = "'2015-01-01'" -%} with spine as ( - select generate_series({{start_date}}::date, date_trunc('year', current_date, 'Europe/Madrid') + interval '1 year -1 day', '1 month') as month + select + generate_series( + {{ start_date }}::date, + date_trunc('year', current_date, 'Europe/Madrid') + + interval '1 year -1 day', + '1 month' + ) as month ) + select * from spine {% if is_incremental() %} - where coalesce((select max(month) from {{ this }}), {{start_date}}) < month + where + coalesce((select max(month) from {{ this }}), {{ start_date }}) < month {% endif %} -order by month desc \ No newline at end of file +order by month desc diff --git a/dbt_jardiner/models/legacy/alarms/alarm_historical.sql b/dbt_jardiner/models/legacy/alarms/alarm_historical.sql index 5c7a25cf..07074cd0 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_historical.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_historical.sql @@ -11,9 +11,10 @@ with alarm_meter_no_readings as ( 'plant' as device_type, plant_name as device_name, 'alarm_meter_no_readings' as alarm_name, - TRUE as is_alarmed + true as is_alarmed from {{ ref('alarm_meter_no_readings') }} ), + alarm_meter_no_energy as ( select day, @@ -22,41 +23,44 @@ alarm_meter_no_energy as ( 'meter' as device_type, plant_name as device_name, 'alarm_meter_no_energy' as alarm_name, - TRUE as is_alarmed + true as is_alarmed from {{ ref('alarm_meter_no_energy') }} ), + alarm_inverter_temperature as ( select distinct - date_trunc('day', time) as day, + date_trunc('day', time)::date as day, plant_id, plant_name, 'inverter' as device_type, inverter_id::text as device_name, 'alarm_inverter_temperature' as alarm_name, - TRUE as is_alarmed + true as is_alarmed from {{ ref('alarm_inverter_temperature') }} ), + alarm_inverter_zero_power_at_daylight_daily as ( - select + select day, plant_id, plant_name, 'inverter' as device_type, inverter_id::text as device_name, 'alarm_inverter_zero_power_at_daylight' as alarm_name, - TRUE as is_alarmed + true as is_alarmed from {{ ref('alarm_inverter_zero_power_at_daylight_daily') }} ), + alarm_everything as ( select * from alarm_meter_no_readings - UNION + union select * from alarm_meter_no_energy - UNION + union select * from alarm_inverter_temperature - UNION + union select * from alarm_inverter_zero_power_at_daylight_daily ) select * from alarm_everything -order by day desc \ No newline at end of file +order by day desc diff --git a/dbt_jardiner/models/legacy/alarms/alarm_inverter_temperature.sql b/dbt_jardiner/models/legacy/alarms/alarm_inverter_temperature.sql index 94af68ab..9682fa57 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_inverter_temperature.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_inverter_temperature.sql @@ -2,16 +2,21 @@ with inverterregistry_sct as ( - SELECT - time, - plant_id, - plant_name, - inverter_id, - inverter_name, - temperature_c as temp, - min(temperature_c) OVER (PARTITION BY inverter_id ORDER By time ROWS BETWEEN 12 PRECEDING AND current row) AS temperature_c_min - FROM - {{ref('inverterregistry_clean')}} + select + time, + plant_id, + plant_name, + inverter_id, + inverter_name, + temperature_c as temp, --noqa: RF04 + min(temperature_c) + over ( + partition by inverter_id + order by time rows between 12 preceding and current row + ) + as temperature_c_min + from + {{ ref('inverterregistry_clean') }} ) diff --git a/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight.sql b/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight.sql index 9d662a3c..ebf6b234 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight.sql @@ -1,34 +1,41 @@ {{ config(materialized='view') }} with sub_ir as ( - SELECT - ir.time, - ir.plant_id, - ir.plant_name, - ir.inverter_id, - ir.inverter_name, - ir.power_kw, - max(ir.power_kw) OVER w AS power_kw_max_last_12_readings, - count(ir.power_kw) OVER w AS power_kw_count_existing_last_12_readings, - ir.time between solar.sunrise_generous and solar.sunset_generous as is_daylight, - solar.sunrise_generous as daylight_start, - solar.sunset_generous as daylight_end - FROM {{ref('inverterregistry_clean')}} as ir - left join {{ref('plantmonitordb_solarevent__generous')}} as solar on ir.time::date = solar.day and ir.plant_id = solar.plant_id - window w as (PARTITION BY ir.inverter_id ORDER By time ROWS BETWEEN 11 PRECEDING AND current row) + select + ir.time, + ir.plant_id, + ir.plant_name, + ir.inverter_id, + ir.inverter_name, + ir.power_kw, + solar.sunrise_generous as daylight_start, + solar.sunset_generous as daylight_end, + max(ir.power_kw) over w as power_kw_max_last_12_readings, + count(ir.power_kw) over w as power_kw_count_existing_last_12_readings, + ir.time between solar.sunrise_generous and solar.sunset_generous as is_daylight + from {{ ref('inverterregistry_clean') }} as ir + left join + {{ ref('plantmonitordb_solarevent__generous') }} as solar + on ir.time::date = solar.day and ir.plant_id = solar.plant_id + window + w as ( + partition by ir.inverter_id + order by time rows between 11 preceding and current row + ) ) select - *, - case - when is_daylight - and power_kw_count_existing_last_12_readings = 12 - and power_kw_max_last_12_readings = 0 - then TRUE - when power_kw_count_existing_last_12_readings < 12 - then NULL - else FALSE - end as alarm_inverter_zero_power_at_daylight + *, + case + when + is_daylight + and power_kw_count_existing_last_12_readings = 12 + and power_kw_max_last_12_readings = 0 + then true + when power_kw_count_existing_last_12_readings < 12 + then null + else false + end as alarm_inverter_zero_power_at_daylight from sub_ir diff --git a/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight_daily.sql b/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight_daily.sql index a536b177..5cac1a88 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight_daily.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_inverter_zero_power_at_daylight_daily.sql @@ -1,19 +1,24 @@ {{ config(materialized='view') }} with alarm_zero_power as ( - SELECT - date_trunc('day', ir.time) as day, - ir.plant_id, - ir.plant_name, - ir.inverter_id, - ir.daylight_start, - ir.daylight_end, - count(*) as alarm_count - FROM {{ref('alarm_inverter_zero_power_at_daylight')}} as ir - group by date_trunc('day', ir.time), plant_id, plant_name, inverter_id, daylight_start, daylight_end + select + ir.plant_id, + ir.plant_name, + ir.inverter_id, + ir.daylight_start, + ir.daylight_end, + date_trunc('day', ir.time) as day, + count(*) as alarm_count + from {{ ref('alarm_inverter_zero_power_at_daylight') }} as ir + group by + date_trunc('day', ir.time), + ir.plant_id, + ir.plant_name, + ir.inverter_id, + ir.daylight_start, + ir.daylight_end ) -select - * +select * from alarm_zero_power order by day desc diff --git a/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy.sql b/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy.sql index 522c56c8..7fbc2c31 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy.sql @@ -1,13 +1,15 @@ {{ config(materialized='view') }} with plant_production_daily as ( - select - * - from {{ ref('plant_production_daily') }} as ppd - left join {{ ref('alarm_meter_no_energy_plant_thresholds') }} as amnept USING (plant_name) + select prod.*, alarm.num_hours_threshold, alarm.threshold_type + from {{ ref('plant_production_daily') }} as prod + left join + {{ ref('alarm_meter_no_energy_plant_thresholds') }} as alarm + using (plant_name) ) + select -*, -2 as alarm_priority + *, + 2 as alarm_priority from plant_production_daily -where unexpected_hours_without_energy >= num_hours_threshold \ No newline at end of file +where unexpected_hours_without_energy >= num_hours_threshold diff --git a/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy_plant_thresholds.sql b/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy_plant_thresholds.sql index 1f1c865e..c6f9fb05 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy_plant_thresholds.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_meter_no_energy_plant_thresholds.sql @@ -1,15 +1,14 @@ {{ config(materialized='view') }} with threshold_values as ( - SELECT * from ( - VALUES - ('Manlleu_Piscina', 4, 'hours'), - ('Manlleu_Pavello', 4, 'hours'), - ('Torrefarrera', 4, 'hours'), - ('Lleida', 4, 'hours'), - ('Picanya', 1, 'single_hour') - ) - AS t (plant_name, num_hours_threshold, threshold_type) + select thv.* from ( + values + ('Manlleu_Piscina', 4, 'hours'), + ('Manlleu_Pavello', 4, 'hours'), + ('Torrefarrera', 4, 'hours'), + ('Lleida', 4, 'hours'), + ('Picanya', 1, 'single_hour') + ) as thv (plant_name, num_hours_threshold, threshold_type) ) -select * from threshold_values \ No newline at end of file +select * from threshold_values diff --git a/dbt_jardiner/models/legacy/alarms/alarm_meter_no_readings.sql b/dbt_jardiner/models/legacy/alarms/alarm_meter_no_readings.sql index 07dc28dc..26e648c1 100644 --- a/dbt_jardiner/models/legacy/alarms/alarm_meter_no_readings.sql +++ b/dbt_jardiner/models/legacy/alarms/alarm_meter_no_readings.sql @@ -3,15 +3,18 @@ {# TODO we have to limit per connection_day per plant to avoid having NULL readings before plant existed#} select - day, - somplants.plant_id, - somplants.plant_name, - meter_name, - current_meter_connection_protocol, - COALESCE(meter_registry_hours_with_readings,0) as meter_registry_hours_with_readings, - 1 as alarm_priority -from {{ref('plant_production_daily') }} as plant_production_daily -inner join {{ ref('som_plants') }} as somplants using(plant_id) -where day < CURRENT_DATE - and COALESCE(meter_registry_hours_with_readings,0) < 12 -order by day desc \ No newline at end of file + prod.day, + somplants.plant_id, + somplants.plant_name, + prod.meter_name, + prod.current_meter_connection_protocol, + 1 as alarm_priority, + coalesce( + prod.meter_registry_hours_with_readings, 0 + ) as meter_registry_hours_with_readings +from {{ ref('plant_production_daily') }} as prod + inner join {{ ref('som_plants') }} as somplants using (plant_id) +where + prod.day < current_date + and coalesce(prod.meter_registry_hours_with_readings, 0) < 12 +order by prod.day desc diff --git a/dbt_jardiner/models/legacy/alarms/stage/alarm_current_alarmed.sql b/dbt_jardiner/models/legacy/alarms/stage/alarm_current_alarmed.sql index 9eb0d3c1..f2fd668f 100644 --- a/dbt_jardiner/models/legacy/alarms/stage/alarm_current_alarmed.sql +++ b/dbt_jardiner/models/legacy/alarms/stage/alarm_current_alarmed.sql @@ -7,7 +7,7 @@ select device_name, alarm_name, is_alarmed -from {{ref('alarm_historical_one_day_ago')}} +from {{ ref('alarm_historical_one_day_ago') }} except select plant_id, @@ -16,4 +16,4 @@ select device_name, alarm_name, is_alarmed -from {{ref('alarm_historical_two_days_ago')}} +from {{ ref('alarm_historical_two_days_ago') }} diff --git a/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_one_day_ago.sql b/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_one_day_ago.sql index ee427925..8aafab06 100644 --- a/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_one_day_ago.sql +++ b/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_one_day_ago.sql @@ -1,12 +1,12 @@ {{ config(materialized='view') }} select - day, - plant_id, - plant_name, - device_type, - device_name, - alarm_name, - is_alarmed -from {{ref('alarm_historical')}} -where (CURRENT_DATE - 1) = day \ No newline at end of file + day, + plant_id, + plant_name, + device_type, + device_name, + alarm_name, + is_alarmed +from {{ ref('alarm_historical') }} +where (current_date - 1) = day diff --git a/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_two_days_ago.sql b/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_two_days_ago.sql index 80d04460..10ed1c36 100644 --- a/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_two_days_ago.sql +++ b/dbt_jardiner/models/legacy/alarms/stage/alarm_historical_two_days_ago.sql @@ -8,5 +8,5 @@ select device_name, alarm_name, is_alarmed -from {{ref('alarm_historical')}} -where (CURRENT_DATE - 2) = day +from {{ ref('alarm_historical') }} +where (current_date - 2) = day diff --git a/dbt_jardiner/models/legacy/alerts/alert_inverter_interinverter_relative_temperature.sql b/dbt_jardiner/models/legacy/alerts/alert_inverter_interinverter_relative_temperature.sql index 539537ce..f6605cbd 100644 --- a/dbt_jardiner/models/legacy/alerts/alert_inverter_interinverter_relative_temperature.sql +++ b/dbt_jardiner/models/legacy/alerts/alert_inverter_interinverter_relative_temperature.sql @@ -1,22 +1,22 @@ {{ config(materialized='view') }} with inverterregistry_sct as ( - select - time, - plant_id, - plant_name, - 10 < ( max(temperature_c) - min(temperature_c) ) as is_alarmed - from {{ref('alert_inverterregistry_clean_last_hour')}} as inverterregistry_sct - group by time, plant_id, plant_name + select + time, + plant_id, + plant_name, + 10 < (max(temperature_c) - min(temperature_c)) as is_alarmed + from {{ ref('alert_inverterregistry_clean_last_hour') }} + group by time, plant_id, plant_name ) select - max(time) as time, - plant_id, - plant_name, - 'plant' as device_type, - plant_name as device_name, - 'alert_inverter_interinverter_relative_temperature' as alarm_name, - every(is_alarmed) as is_alarmed + plant_id, + plant_name, + 'plant' as device_type, + plant_name as device_name, + 'alert_inverter_interinverter_relative_temperature' as alarm_name, + max(time) as time, --noqa: RF04 + every(is_alarmed) as is_alarmed from inverterregistry_sct group by plant_id, plant_name diff --git a/dbt_jardiner/models/legacy/alerts/alert_inverter_temperature.sql b/dbt_jardiner/models/legacy/alerts/alert_inverter_temperature.sql index 4b1e447f..877d8d84 100644 --- a/dbt_jardiner/models/legacy/alerts/alert_inverter_temperature.sql +++ b/dbt_jardiner/models/legacy/alerts/alert_inverter_temperature.sql @@ -1,29 +1,29 @@ {{ config(materialized='view') }} with inverterregistry_sct as ( - SELECT - time, - plant_id, - plant_name, - inverter_id, - inverter_name, - temperature_c as temperature_c, - 50 < temperature_c as is_alarmed - FROM - {{ref('alert_inverterregistry_clean_last_hour')}} + select + alert.time, + alert.plant_id, + alert.plant_name, + alert.inverter_id, + alert.inverter_name, + alert.temperature_c as temperature_c, + 50 < alert.temperature_c as is_alarmed + from + {{ ref('alert_inverterregistry_clean_last_hour') }} as alert ) select - max(time) as time, - plant_id, - plant_name, - inverter_id, - inverter_name, - min(temperature_c) as min_temperature_c, - max(temperature_c) as max_temperature_c, - 'inverter' as device_type, - inverter_name as device_name, - 'alert_inverter_temperature' as alarm_name, - every(is_alarmed) as is_alarmed + plant_id, + plant_name, + inverter_id, + inverter_name, + 'inverter' as device_type, + inverter_name as device_name, + 'alert_inverter_temperature' as alarm_name, + max(time) as time, --noqa: RF04 + min(temperature_c) as min_temperature_c, + max(temperature_c) as max_temperature_c, + every(is_alarmed) as is_alarmed from inverterregistry_sct group by plant_id, plant_name, inverter_id, inverter_name diff --git a/dbt_jardiner/models/legacy/alerts/alert_inverter_zero_power_at_daylight.sql b/dbt_jardiner/models/legacy/alerts/alert_inverter_zero_power_at_daylight.sql index 6ecb598a..a16bfb92 100644 --- a/dbt_jardiner/models/legacy/alerts/alert_inverter_zero_power_at_daylight.sql +++ b/dbt_jardiner/models/legacy/alerts/alert_inverter_zero_power_at_daylight.sql @@ -1,45 +1,59 @@ {{ config(materialized='view') }} with sub_ir as ( - SELECT - ir.time, - ir.plant_id, - ir.plant_name, - ir.inverter_id, - ir.inverter_name, - ir.power_kw, - max(ir.power_kw) OVER w AS power_kw_max_last_12_readings, - count(ir.power_kw) OVER w AS power_kw_count_existing_last_12_readings, - ir.time between solar.sunrise_generous and solar.sunset_generous as is_daylight, - solar.sunrise_generous as daylight_start, - solar.sunset_generous as daylight_end - FROM {{ref('alert_inverterregistry_clean_last_hour')}} as ir - left join {{ref('plantmonitordb_solarevent__generous')}} as solar on ir.time::date = solar.day and ir.plant_id = solar.plant_id - window w as (PARTITION BY ir.inverter_id ORDER By time ROWS BETWEEN 11 PRECEDING AND current row) -), sub_alarm as ( - select - *, + select + ir.time, + ir.plant_id, + ir.plant_name, + ir.inverter_id, + ir.inverter_name, + ir.power_kw, + solar.sunrise_generous as daylight_start, + solar.sunset_generous as daylight_end, + max(ir.power_kw) over w as power_kw_max_last_12_readings, + count(ir.power_kw) over w as power_kw_count_existing_last_12_readings, + ir.time between solar.sunrise_generous and solar.sunset_generous as is_daylight + from {{ ref('alert_inverterregistry_clean_last_hour') }} as ir + left join + {{ ref('plantmonitordb_solarevent__generous') }} as solar + on ir.time::date = solar.day and ir.plant_id = solar.plant_id + window + w as ( + partition by ir.inverter_id + order by ir.time rows between 11 preceding and current row + ) +), + +sub_alarm as ( + select + *, case - when not is_daylight - then NULL - when power_kw_count_existing_last_12_readings < 12 - then NULL - when power_kw_max_last_12_readings = 0 - then TRUE - else FALSE + when not is_daylight + then null + when power_kw_count_existing_last_12_readings < 12 + then null + when power_kw_max_last_12_readings = 0 + then true + else false end as is_alarmed - from sub_ir + from sub_ir ) -SELECT *, - 'inverter' as device_type, - inverter_id as device_id, - inverter_name as device_name, - 'alert_inverter_zero_power_at_daylight' as alarm_name -FROM sub_alarm -where (time,plant_id, inverter_name) in - ( select max(time) as time, plant_id, inverter_name - from sub_alarm - group by plant_id, inverter_name - ) -- get last row for each plant and deivce -order by plant_id, inverter_name desc +select + *, + 'inverter' as device_type, + inverter_id as device_id, + inverter_name as device_name, + 'alert_inverter_zero_power_at_daylight' as alarm_name +from sub_alarm +where + (time, plant_id, inverter_name) in + ( + select + max(time) as time, --noqa: RF04 + plant_id, + inverter_name + from sub_alarm + group by plant_id, inverter_name + ) -- get last row for each plant and deivce +order by plant_id asc, inverter_name desc diff --git a/dbt_jardiner/models/legacy/alerts/alert_meter_no_readings.sql b/dbt_jardiner/models/legacy/alerts/alert_meter_no_readings.sql index 537aaf07..b51e8570 100644 --- a/dbt_jardiner/models/legacy/alerts/alert_meter_no_readings.sql +++ b/dbt_jardiner/models/legacy/alerts/alert_meter_no_readings.sql @@ -1,11 +1,11 @@ {{ config(materialized='view') }} select - *, - newest_reading_time as time, + mrg.*, + mrg.newest_reading_time as time, --noqa: RF04 'meter' as device_type, mrg.meter_name as device_name, 'alert_meter_no_readings' as alarm_name, - alarm_no_reading as is_alarmed -FROM {{ ref('alert_meter_newest_reading') }} as mrg -order by newest_reading_time desc \ No newline at end of file + mrg.alarm_no_reading as is_alarmed +from {{ ref('alert_meter_newest_reading') }} as mrg +order by mrg.newest_reading_time desc diff --git a/dbt_jardiner/models/legacy/alerts/alert_meter_zero_energy.sql b/dbt_jardiner/models/legacy/alerts/alert_meter_zero_energy.sql index ab9b4189..db6e1f47 100644 --- a/dbt_jardiner/models/legacy/alerts/alert_meter_zero_energy.sql +++ b/dbt_jardiner/models/legacy/alerts/alert_meter_zero_energy.sql @@ -24,19 +24,23 @@ sets to false alarms with NULL with meterregistry_last_readings as ( - select * from {{ ref('meter_registry_raw') }} - where current_date - interval '30 days' < time + select * from {{ ref('meter_registry_raw') }} + where current_date - interval '30 days' < time ), meterregistry_daylight as ( - select - mr.*, - mr.time between solar.sunrise_generous and solar.sunset_generous as is_daylight, - solar.sunrise_generous as daylight_start, - solar.sunset_generous as daylight_end - from meterregistry_last_readings as mr - left join {{ ref('meters_with_thresholds') }} as m on m.meter_id = mr.meter_id - left join {{ ref('plantmonitordb_solarevent__generous') }} as solar on mr.time::date = solar.day and m.plant_id = solar.plant_id + select + mr.*, + solar.sunrise_generous as daylight_start, + solar.sunset_generous as daylight_end, + mr.time between solar.sunrise_generous and solar.sunset_generous as is_daylight + from meterregistry_last_readings as mr + left join + {{ ref('meters_with_thresholds') }} as m + on mr.meter_id = m.meter_id + left join + {{ ref('plantmonitordb_solarevent__generous') }} as solar + on mr.time::date = solar.day and m.plant_id = solar.plant_id ), meter_registry_hourly_raw as ( @@ -53,40 +57,46 @@ meter_registry_hourly_raw as ( mr.daylight_start, mr.daylight_end from {{ ref('meters_with_thresholds') }} as meters - left join lateral ( -- use left if you want meters without readings, inner otherwise - select - mr.time, - mr.meter_id, - mr.export_energy_wh, - mr.is_daylight, - mr.daylight_start, - mr.daylight_end - from meterregistry_daylight as mr - where + -- use left if you want meters without readings, inner otherwise + left join lateral ( + select + mr.time, + mr.meter_id, + mr.export_energy_wh, + mr.is_daylight, + mr.daylight_start, + mr.daylight_end + from meterregistry_daylight as mr + where mr.meter_id = meters.meter_id and mr.time between mr.daylight_start and mr.daylight_end - order by time desc - limit meters.num_hours_threshold - ) as mr on true + order by mr.time desc + limit meters.num_hours_threshold + ) as mr on true ), meter_registry_group as ( select - max(time) as time, - min(time) as from_time, num_hours_threshold, - every(is_daylight) as is_daylight, - min(daylight_start) as daylight_start, - min(daylight_end) as daylight_end, plant_id, meter_id, meter_name, meter_connection_protocol, + max(time) as time, --noqa: RF04 + min(time) as from_time, + every(is_daylight) as is_daylight, + min(daylight_start) as daylight_start, + min(daylight_end) as daylight_end, count(export_energy_wh) as reading_count, max(export_energy_wh) as export_energy_wh, max(export_energy_wh) = 0 as alarm_zero_energy from meter_registry_hourly_raw - group by plant_id, meter_id, meter_name, meter_connection_protocol, num_hours_threshold + group by + plant_id, + meter_id, + meter_name, + meter_connection_protocol, + num_hours_threshold ) select @@ -102,11 +112,6 @@ select mrg.daylight_start, mrg.daylight_end, mrg.alarm_zero_energy, - CASE - WHEN reading_count > 0 and export_energy_wh > 0 THEN 1 - WHEN reading_count > 0 and export_energy_wh = 0 THEN 0 - ELSE NULL - END as has_energy, mrg.from_time, mrg.time as to_time, nr.newest_reading_time, @@ -114,7 +119,13 @@ select 'meter' as device_type, mrg.meter_name as device_name, 'alert_meter_zero_energy' as alarm_name, - alarm_zero_energy as is_alarmed -FROM meter_registry_group as mrg -inner join {{ ref('som_plants') }} as plants using(plant_id) -left join {{ ref('alert_meter_newest_reading') }} as nr on nr.meter_id = mrg.meter_id + mrg.alarm_zero_energy as is_alarmed, + case + when mrg.reading_count > 0 and mrg.export_energy_wh > 0 then 1 + when mrg.reading_count > 0 and mrg.export_energy_wh = 0 then 0 + end as has_energy +from meter_registry_group as mrg + inner join {{ ref('som_plants') }} as plants on mrg.plant_id = plants.plant_id + left join + {{ ref('alert_meter_newest_reading') }} as nr + on mrg.meter_id = nr.meter_id diff --git a/dbt_jardiner/models/legacy/alerts/alert_string_low_intensity_at_daylight.sql b/dbt_jardiner/models/legacy/alerts/alert_string_low_intensity_at_daylight.sql index 32982113..97b95e03 100644 --- a/dbt_jardiner/models/legacy/alerts/alert_string_low_intensity_at_daylight.sql +++ b/dbt_jardiner/models/legacy/alerts/alert_string_low_intensity_at_daylight.sql @@ -1,45 +1,52 @@ {{ config(materialized='view') }} with inverterregistry_last_readings as ( - select * from {{ ref('inverterregistry_clean') }} - where now() - interval '1 hour' < time + select ir.* from {{ ref('inverterregistry_clean') }} as ir + where now() - interval '1 hour' < ir.time ), inverter_join as ( - select - sub_sr.time as time, - plant.plant_id, - plant.plant_name as plant_name, - sub_sr.inverter_id, - sub_sr.inverter_name, - sub_sr.string_name, - sub_sr.stringdevice_name, - sub_sr.intensity_ma, - ir.power_kw, - sub_sr.intensity_ma < 500 and ir.power_kw > 10 as is_low_intensity - from {{ ref('stringregistry_latest_hour') }} as sub_sr - left join inverterregistry_last_readings as ir on sub_sr.time = ir.time and sub_sr.inverter_id = ir.inverter_id - left join {{ ref('som_plants_raw') }} as plant on plant.plant_id = sub_sr.plant_id + select + sub_sr.time as time, --noqa: RF04 + plant.plant_id, + plant.plant_name as plant_name, + sub_sr.inverter_id, + sub_sr.inverter_name, + sub_sr.string_name, + sub_sr.stringdevice_name, + sub_sr.intensity_ma, + ir.power_kw, + sub_sr.intensity_ma < 500 and ir.power_kw > 10 as is_low_intensity + from {{ ref('stringregistry_latest_hour') }} as sub_sr + left join + inverterregistry_last_readings as ir + on sub_sr.time = ir.time and sub_sr.inverter_id = ir.inverter_id + left join + {{ ref('som_plants_raw') }} as plant + on sub_sr.plant_id = plant.plant_id ), sr_grouped as ( - select - max(time) as time, - plant_id, - plant_name, - inverter_id, - inverter_name, - string_name, - stringdevice_name, - sum(intensity_ma) as intensity_ma, - sum(power_kw) as power_kw, - every(is_low_intensity) as low_intensity_alarm - from inverter_join - GROUP BY plant_id, plant_name, inverter_id, inverter_name, string_name, stringdevice_name + select + plant_id, + plant_name, + inverter_id, + inverter_name, + string_name, + stringdevice_name, + max(time) as time, --noqa: RF04 + sum(intensity_ma) as intensity_ma, + sum(power_kw) as power_kw, + every(is_low_intensity) as low_intensity_alarm + from inverter_join + group by + plant_id, + plant_name, + inverter_id, + inverter_name, + string_name, + stringdevice_name ) -select -* +select * from sr_grouped - - diff --git a/dbt_jardiner/models/legacy/alerts/alerts_historic.sql b/dbt_jardiner/models/legacy/alerts/alerts_historic.sql index fc1dd4f1..e9db9322 100644 --- a/dbt_jardiner/models/legacy/alerts/alerts_historic.sql +++ b/dbt_jardiner/models/legacy/alerts/alerts_historic.sql @@ -1,17 +1,25 @@ {{ config(materialized='view') }} with alarm_everything as ( - select * from {{ source('plantmonitor_legacy', 'alert_inverter_zero_power_at_daylight_historic') }} - UNION - select * from {{ source('plantmonitor_legacy', 'alert_inverter_interinverter_relative_temperature_historic') }} - UNION - select * from {{ source('plantmonitor_legacy', 'alert_inverter_temperature_historic') }} - UNION - select * from {{ source('plantmonitor_legacy', 'alert_meter_no_readings_historic') }} - UNION - select * from {{ source('plantmonitor_legacy', 'alert_meter_zero_energy_historic') }} + select * + from + {{ source('plantmonitor_legacy', 'alert_inverter_zero_power_at_daylight_historic') }} + union + select * + from + {{ source('plantmonitor_legacy', 'alert_inverter_interinverter_relative_temperature_historic') }} + union + select * + from + {{ source('plantmonitor_legacy', 'alert_inverter_temperature_historic') }} + union + select * + from {{ source('plantmonitor_legacy', 'alert_meter_no_readings_historic') }} + union + select * + from {{ source('plantmonitor_legacy', 'alert_meter_zero_energy_historic') }} ) select * from alarm_everything -order by notification_time desc \ No newline at end of file +order by notification_time desc diff --git a/dbt_jardiner/models/legacy/alerts/stage/alert_inverterregistry_clean_last_hour.sql b/dbt_jardiner/models/legacy/alerts/stage/alert_inverterregistry_clean_last_hour.sql index 7d742f3a..47c2e44e 100644 --- a/dbt_jardiner/models/legacy/alerts/stage/alert_inverterregistry_clean_last_hour.sql +++ b/dbt_jardiner/models/legacy/alerts/stage/alert_inverterregistry_clean_last_hour.sql @@ -26,6 +26,8 @@ {# example vars{“inverter_test_sample”:“inverter_test_name”} #} {% else %} - select * from {{ ref("inverterregistry_clean") }} as ir where time between (now() - interval '2 hour') and now() + select * + from {{ ref("inverterregistry_clean") }} + where time between (now() - interval '2 hour') and now() {% endif %} diff --git a/dbt_jardiner/models/legacy/alerts/stage/alert_meter_newest_reading.sql b/dbt_jardiner/models/legacy/alerts/stage/alert_meter_newest_reading.sql index d5c2d114..a9cac536 100644 --- a/dbt_jardiner/models/legacy/alerts/stage/alert_meter_newest_reading.sql +++ b/dbt_jardiner/models/legacy/alerts/stage/alert_meter_newest_reading.sql @@ -1,8 +1,8 @@ {{ config(materialized='view') }} with meterregistry_newest_readings as ( - select * from {{ ref('meter_registry_raw') }} - where current_date - interval '90 day' < time + select * from {{ ref('meter_registry_raw') }} + where current_date - interval '90 day' < time ), meter_registry_newest_reading as ( @@ -10,30 +10,41 @@ meter_registry_newest_reading as ( meters.*, mr.time as newest_reading_time from {{ ref('meters_with_thresholds') }} as meters - inner join lateral ( - select - mr.meter_id, - time - from meterregistry_newest_readings as mr - where + inner join lateral ( + select + mr.meter_id, + mr.time + from meterregistry_newest_readings as mr + where mr.meter_id = meters.meter_id - order by time desc - limit 1 - ) as mr on mr.meter_id = meters.meter_id + order by mr.time desc + limit 1 + ) as mr on meters.meter_id = mr.meter_id ) select - *, - case - when mrg.meter_connection_protocol ilike 'moxa' then now() - newest_reading_time - when mrg.meter_connection_protocol ilike 'ip' then now() - newest_reading_time - else NULL + plant.plant_name, + plant.plant_codename, + plant.peak_power_w, + mrg.*, + case + when + mrg.meter_connection_protocol ilike 'moxa' + then now() - mrg.newest_reading_time + when + mrg.meter_connection_protocol ilike 'ip' + then now() - mrg.newest_reading_time end as time_from_last_reading, case - when mrg.meter_connection_protocol ilike 'moxa' then now() - newest_reading_time > interval '30 hours' -- 24 hours + 6h margin - when mrg.meter_connection_protocol ilike 'ip' then now() - newest_reading_time > interval '12 hours' + -- 24 hours + 6h margin + when + mrg.meter_connection_protocol ilike 'moxa' + then now() - mrg.newest_reading_time > interval '30 hours' + when + mrg.meter_connection_protocol ilike 'ip' + then now() - mrg.newest_reading_time > interval '12 hours' else false end as alarm_no_reading -FROM meter_registry_newest_reading as mrg -inner join {{ ref('som_plants') }} as plants using(plant_id) -order by newest_reading_time desc \ No newline at end of file +from meter_registry_newest_reading as mrg + inner join {{ ref('som_plants') }} as plant using (plant_id) +order by mrg.newest_reading_time desc diff --git a/dbt_jardiner/models/legacy/alerts/stage/alert_meter_zero_energy_by_now.sql b/dbt_jardiner/models/legacy/alerts/stage/alert_meter_zero_energy_by_now.sql index 8d5dc3d8..e59993b0 100644 --- a/dbt_jardiner/models/legacy/alerts/stage/alert_meter_zero_energy_by_now.sql +++ b/dbt_jardiner/models/legacy/alerts/stage/alert_meter_zero_energy_by_now.sql @@ -1,35 +1,38 @@ {{ config(materialized='view') }} - -- see below - -- use left join lateral if you want meters without readings within threshold range - -- (moxa usually has 12h delay which means we never have readings within range) - -- use **inner join lateral** if you only want meters with readings +-- see below +-- use left join lateral if you want meters without readings within threshold range +-- (moxa usually has 12h delay which means we never have readings within range) +-- use **inner join lateral** if you only want meters with readings - -- tcp usually has a 2h delay. The ERP polls every 2h or so, and we sync every 20 minutes, - -- so we'll likely get at least 1h20 delay. +-- tcp usually has a 2h delay. The ERP polls every 2h or so, and we sync every 20 minutes, +-- so we'll likely get at least 1h20 delay. - -- to mitigate this we add a baseline +2h to the thresholds +-- to mitigate this we add a baseline +2h to the thresholds with meterregistry_last_readings as ( - select * from {{ ref('meter_registry_raw') }} - where current_date < time + select * from {{ ref('meter_registry_raw') }} + where current_date < time ), meterregistry_daylight as ( - select - mr.*, - mr.time between solar.sunrise_generous and solar.sunset_generous as is_daylight, - solar.sunrise_generous as daylight_start, - solar.sunset_generous as daylight_end - from meterregistry_last_readings as mr - left join {{ ref('meters_with_thresholds') }} as m on m.meter_id = mr.meter_id - left join {{ ref('plantmonitordb_solarevent__generous') }} as solar on mr.time::date = solar.day and m.plant_id = solar.plant_id + select + mr.*, + solar.sunrise_generous as daylight_start, + solar.sunset_generous as daylight_end, + mr.time between solar.sunrise_generous and solar.sunset_generous as is_daylight + from meterregistry_last_readings as mr + left join + {{ ref('meters_with_thresholds') }} as m + on mr.meter_id = m.meter_id + left join + {{ ref('plantmonitordb_solarevent__generous') }} as solar + on mr.time::date = solar.day and m.plant_id = solar.plant_id ), meter_registry_hourly_raw as ( select mr.time, - now() - make_interval(hours=>meters.num_hours_threshold + 2) as from_time, meters.num_hours_threshold, meters.plant_id, meters.meter_id, @@ -37,35 +40,45 @@ meter_registry_hourly_raw as ( mr.meter_id as mr_meter_id, meters.meter_connection_protocol, mr.export_energy_wh, - mr.is_daylight + mr.is_daylight, + now() + - make_interval(hours => meters.num_hours_threshold + 2) as from_time from {{ ref('meters_with_thresholds') }} as meters - left join lateral ( -- use left if you want meters without readings, inner otherwise - select - mr.time, - mr.meter_id, - mr.export_energy_wh, - mr.is_daylight - from meterregistry_daylight as mr - where + -- use left if you want meters without readings, inner otherwise + left join lateral ( + select + mr.time, + mr.meter_id, + mr.export_energy_wh, + mr.is_daylight + from meterregistry_daylight as mr + where mr.meter_id = meters.meter_id - and now() - make_interval(hours=>(meters.num_hours_threshold + 2)) < time - ) as mr on true + and now() - make_interval(hours => (meters.num_hours_threshold + 2)) + < mr.time + ) as mr on true ), meter_registry_group as ( select - max(time) as time, from_time, num_hours_threshold, plant_id, meter_id, meter_name, meter_connection_protocol, + max(time) as time, --noqa: RF04 count(export_energy_wh) as reading_count, max(export_energy_wh) as export_energy_wh, max(export_energy_wh) = 0 as alarm_zero_energy from meter_registry_hourly_raw - group by plant_id, meter_id, meter_name, meter_connection_protocol, from_time, num_hours_threshold + group by + plant_id, + meter_id, + meter_name, + meter_connection_protocol, + from_time, + num_hours_threshold ) select @@ -78,22 +91,22 @@ select mrg.reading_count, mrg.export_energy_wh, mrg.alarm_zero_energy, - CASE - WHEN reading_count > 0 and export_energy_wh > 0 THEN 1 - WHEN reading_count > 0 and export_energy_wh = 0 THEN 0 - ELSE NULL - END as has_energy, nr.newest_reading_time, mrg.from_time, mrg.num_hours_threshold, 'meter' as device_type, mrg.meter_name as device_name, 'alert_meter_zero_energy' as alarm_name, - CASE - WHEN reading_count > 0 and export_energy_wh > 0 THEN FALSE - WHEN reading_count > 0 and export_energy_wh = 0 THEN TRUE - ELSE NULL - END as is_alarmed -FROM meter_registry_group as mrg -inner join {{ ref('som_plants') }} as plants using(plant_id) -left join {{ ref('alert_meter_newest_reading') }} as nr on nr.meter_id = mrg.meter_id + case + when mrg.reading_count > 0 and mrg.export_energy_wh > 0 then 1 + when mrg.reading_count > 0 and mrg.export_energy_wh = 0 then 0 + end as has_energy, + case + when mrg.reading_count > 0 and mrg.export_energy_wh > 0 then false + when mrg.reading_count > 0 and mrg.export_energy_wh = 0 then true + end as is_alarmed +from meter_registry_group as mrg + inner join {{ ref('som_plants') }} as plants on mrg.plant_id = plants.plant_id + left join + {{ ref('alert_meter_newest_reading') }} as nr + on mrg.meter_id = nr.meter_id diff --git a/dbt_jardiner/models/legacy/datasets/gestio_dactius/average_yearly_production_plant.sql b/dbt_jardiner/models/legacy/datasets/gestio_dactius/average_yearly_production_plant.sql index e1312c01..2d9befc2 100644 --- a/dbt_jardiner/models/legacy/datasets/gestio_dactius/average_yearly_production_plant.sql +++ b/dbt_jardiner/models/legacy/datasets/gestio_dactius/average_yearly_production_plant.sql @@ -1,15 +1,29 @@ -SELECT yearly.plant_id, -yearly.plant, -avg(yearly.export_energy_mwh) AS historic_avg, -yearly.time_year AS "time" -FROM (SELECT plant.id as plant_id, - plant.name AS plant, - avg(meterregistry.export_energy_wh) / 1000000.0 * 24::numeric * 365::numeric AS export_energy_mwh, - date_trunc('year'::text, meterregistry."time") AS time_year, - date_part('year'::text, meterregistry."time") AS year - FROM meterregistry - LEFT JOIN meter ON meterregistry.meter = meter.id - LEFT JOIN plant ON meter.plant = plant.id - WHERE meterregistry."time" IS NOT NULL AND date_part('year'::text, meterregistry."time") < date_part('year'::text, now()) - GROUP BY plant.id, plant.name, (date_trunc('year'::text, meterregistry."time")), (date_part('year'::text, meterregistry."time"))) yearly -GROUP BY yearly.plant_id, yearly.plant, yearly.time_year \ No newline at end of file +select + yearly.plant_id, + yearly.plant, + yearly.time_year as "time", + avg(yearly.export_energy_mwh) as historic_avg +from ( + select + plant.id as plant_id, + plant.name as plant, + avg(meterregistry.export_energy_wh) + / 1000000.0 + * 24::numeric + * 365::numeric as export_energy_mwh, + date_trunc('year'::text, meterregistry."time") as time_year, + date_part('year'::text, meterregistry."time") as year + from meterregistry + left join meter on meterregistry.meter = meter.id + left join plant on meter.plant = plant.id + where + meterregistry."time" is not null + and date_part('year'::text, meterregistry."time") + < date_part('year'::text, now()) + group by + plant.id, + plant.name, + (date_trunc('year'::text, meterregistry."time")), + (date_part('year'::text, meterregistry."time")) +) as yearly +group by yearly.plant_id, yearly.plant, yearly.time_year diff --git a/dbt_jardiner/models/legacy/datasets/plant_production_daily.sql b/dbt_jardiner/models/legacy/datasets/plant_production_daily.sql index 7593f9ed..afb15c11 100644 --- a/dbt_jardiner/models/legacy/datasets/plant_production_daily.sql +++ b/dbt_jardiner/models/legacy/datasets/plant_production_daily.sql @@ -10,19 +10,48 @@ with ppd as ( meter_name as meter_name, current_meter_connection_protocol as current_meter_connection_protocol, plant_peak_power_kw as plant_peak_power_kw, - sum(coalesce(meter_registry_export_energy_kwh,0)) as meter_registry_export_energy_kwh, - SUM(CASE WHEN meter_registry_export_energy_kwh is not null THEN 1 else 0 END) as meter_registry_hours_with_readings, - SUM(CASE WHEN meter_registry_export_energy_kwh > 0 THEN 1 else 0 END) as meter_registry_hours_with_energy, - SUM(satellite_readings_horizontal_irradiation_kwh_m2) as satellite_readings_horizontal_irradiation_kwh_m2, - SUM(satellite_readings_tilted_irradiation_kwh_m2) as satellite_readings_tilted_irradiation_kwh_m2, - round(avg(satellite_readings_module_temperature_dc),2) as satellite_readings_module_temperature_dc_mean, - SUM(satellite_readings_energy_output_kwh) as satellite_readings_energy_output_kwh, - SUM(CASE WHEN satellite_readings_energy_output_kwh is not null THEN 1 else 0 END) as satellite_hours_with_readings, - SUM(CASE WHEN daylight_real is TRUE THEN 1 else 0 END) as solar_hours_real, - SUM(CASE WHEN daylight_generous is TRUE THEN 1 else 0 END) as solar_hours_minimum, - SUM(forecast_energy_kwh) as forecast_energy_kwh, - sum(abs(deviation_exported_vs_forecast_expected_kwh)) as deviation_exported_vs_forecast_expected_kwh_cumsum_abs - from {{ref('plant_production_hourly')}} + sum( + coalesce(meter_registry_export_energy_kwh, 0) + ) as meter_registry_export_energy_kwh, + sum( + case + when meter_registry_export_energy_kwh is not null then 1 else 0 + end + ) as meter_registry_hours_with_readings, + sum( + case when meter_registry_export_energy_kwh > 0 then 1 else 0 end + ) as meter_registry_hours_with_energy, + sum( + satellite_readings_horizontal_irradiation_kwh_m2 + ) as satellite_readings_horizontal_irradiation_kwh_m2, + sum( + satellite_readings_tilted_irradiation_kwh_m2 + ) as satellite_readings_tilted_irradiation_kwh_m2, + round( + avg(satellite_readings_module_temperature_dc), 2 + ) as satellite_readings_module_temperature_dc_mean, + sum( + satellite_readings_energy_output_kwh + ) as satellite_readings_energy_output_kwh, + sum( + case + when + satellite_readings_energy_output_kwh is not null + then 1 + else 0 + end + ) as satellite_hours_with_readings, + sum( + case when daylight_real is true then 1 else 0 end + ) as solar_hours_real, + sum( + case when daylight_generous is true then 1 else 0 end + ) as solar_hours_minimum, + sum(forecast_energy_kwh) as forecast_energy_kwh, + sum( + abs(deviation_exported_vs_forecast_expected_kwh) + ) as deviation_exported_vs_forecast_expected_kwh_cumsum_abs + from {{ ref('plant_production_hourly') }} group by day, plant_id, @@ -36,19 +65,55 @@ with ppd as ( select *, - meter_registry_export_energy_kwh - satellite_readings_energy_output_kwh as deviation_exported_vs_satellite_expected_kwh, - solar_hours_minimum - meter_registry_hours_with_energy as unexpected_hours_without_energy, - round((deviation_exported_vs_forecast_expected_kwh_cumsum_abs / NULLIF(meter_registry_export_energy_kwh,0)) * 100,2) as deviation_from_forecast_cumsum_ratio, - CASE - WHEN meter_registry_export_energy_kwh is not null and meter_registry_export_energy_kwh > 0 - AND satellite_readings_energy_output_kwh is not null and satellite_readings_energy_output_kwh > 0 - THEN round(((meter_registry_export_energy_kwh / satellite_readings_energy_output_kwh)-1)::numeric*100,2) - END as p_deviation_exported_vs_expected_satellite, - CASE - WHEN meter_registry_export_energy_kwh is not null and meter_registry_export_energy_kwh > 0 - AND plant_peak_power_kw is not null and plant_peak_power_kw > 0 - AND satellite_readings_tilted_irradiation_kwh_m2 is not null and satellite_readings_tilted_irradiation_kwh_m2 > 0 - THEN round(((meter_registry_export_energy_kwh / plant_peak_power_kw)::float / satellite_readings_tilted_irradiation_kwh_m2)::numeric,4) - END as performance_ratio - from ppd -order by day desc \ No newline at end of file + meter_registry_export_energy_kwh + - satellite_readings_energy_output_kwh as deviation_exported_vs_satellite_expected_kwh, + solar_hours_minimum + - meter_registry_hours_with_energy as unexpected_hours_without_energy, + round( + ( + deviation_exported_vs_forecast_expected_kwh_cumsum_abs + / nullif(meter_registry_export_energy_kwh, 0) + ) + * 100, + 2 + ) as deviation_from_forecast_cumsum_ratio, + case + when + meter_registry_export_energy_kwh is not null + and meter_registry_export_energy_kwh > 0 + and satellite_readings_energy_output_kwh is not null + and satellite_readings_energy_output_kwh > 0 + then + round( + ( + ( + meter_registry_export_energy_kwh + / satellite_readings_energy_output_kwh + ) + - 1 + )::numeric + * 100, + 2 + ) + end as p_deviation_exported_vs_expected_satellite, + case + when + meter_registry_export_energy_kwh is not null + and meter_registry_export_energy_kwh > 0 + and plant_peak_power_kw is not null and plant_peak_power_kw > 0 + and satellite_readings_tilted_irradiation_kwh_m2 is not null + and satellite_readings_tilted_irradiation_kwh_m2 > 0 + then + round( + ( + ( + meter_registry_export_energy_kwh + / plant_peak_power_kw + )::float + / satellite_readings_tilted_irradiation_kwh_m2 + )::numeric, + 4 + ) + end as performance_ratio +from ppd +order by day desc diff --git a/dbt_jardiner/models/legacy/datasets/plant_production_hourly.sql b/dbt_jardiner/models/legacy/datasets/plant_production_hourly.sql index 9c32e56d..406f9c7d 100644 --- a/dbt_jardiner/models/legacy/datasets/plant_production_hourly.sql +++ b/dbt_jardiner/models/legacy/datasets/plant_production_hourly.sql @@ -1,75 +1,104 @@ {{ config(materialized="table") }} with - combined_meter_satellite as ( - select - spine.start_hour as start_hour, - spine.start_hour::date as "day", - spine.plant_id as plant_id, - spine.plant_name as plant_name, - spine.plant_codename as plant_codename, - spine.meter_id as meter_id, - spine.meter_name as meter_name, - spine.meter_connection_protocol as current_meter_connection_protocol, - --fmt:off - round(spine.peak_power_w::numeric / 1000, 2) as plant_peak_power_kw, - round(meter_registry.export_energy_wh::numeric / 1000, 2) as meter_registry_export_energy_kwh, - round(satellite_readings.horizontal_irradiation_wh_m2::numeric / 1000, 2) as satellite_readings_horizontal_irradiation_kwh_m2, - round(satellite_readings.tilted_irradiation_wh_m2::numeric / 1000, 2) as satellite_readings_tilted_irradiation_kwh_m2, - round(satellite_readings.module_temperature_dc::numeric / 100, 2) as satellite_readings_module_temperature_dc, - round(satellite_readings.energy_output_kwh::numeric, 2) as satellite_readings_energy_output_kwh, - spine.start_hour between solar_events.sunrise_real and solar_events.sunset_real as daylight_real, - spine.start_hour between solar_events.sunrise_generous and solar_events.sunset_generous as daylight_generous, - --fmt:on - round(forecast.energy_kwh, 2) as forecast_energy_kwh - from {{ ref("spine_plant_meter_until_last_hour") }} as spine - left join - {{ ref("meter_registry_hourly") }} as meter_registry - on meter_registry.plant_id = spine.plant_id - and meter_registry.time_start_hour = spine.start_hour - left join - {{ ref("satellite_readings__hourly_legacy") }} as satellite_readings - on satellite_readings.plant_id = spine.plant_id - and satellite_readings.start_hour = spine.start_hour - left join - {{ ref("plantmonitordb_solarevent__generous") }} as solar_events - on solar_events.plant_id = spine.plant_id - and solar_events.day = spine.start_hour::date - left join - {{ ref("forecasts_hourly") }} as forecast - on forecast.plant_id = spine.plant_id - and forecast.time_start_hour = spine.start_hour - ) +combined_meter_satellite as ( + select + spine.start_hour as start_hour, + spine.start_hour::date as "day", + spine.plant_id as plant_id, + spine.plant_name as plant_name, + spine.plant_codename as plant_codename, + spine.meter_id as meter_id, + spine.meter_name as meter_name, + spine.meter_connection_protocol as current_meter_connection_protocol, + --fmt:off + round(spine.peak_power_w::numeric / 1000, 2) as plant_peak_power_kw, + round( + meter_registry.export_energy_wh::numeric / 1000, 2 + ) as meter_registry_export_energy_kwh, + round( + satellite_readings.horizontal_irradiation_wh_m2::numeric / 1000, 2 + ) as satellite_readings_horizontal_irradiation_kwh_m2, + round( + satellite_readings.tilted_irradiation_wh_m2::numeric / 1000, 2 + ) as satellite_readings_tilted_irradiation_kwh_m2, + round( + satellite_readings.module_temperature_dc::numeric / 100, 2 + ) as satellite_readings_module_temperature_dc, + round( + satellite_readings.energy_output_kwh::numeric, 2 + ) as satellite_readings_energy_output_kwh, + spine.start_hour between solar_events.sunrise_real and solar_events.sunset_real as daylight_real, + spine.start_hour between solar_events.sunrise_generous and solar_events.sunset_generous as daylight_generous, + --fmt:on + round(forecast.energy_kwh, 2) as forecast_energy_kwh + from {{ ref("spine_plant_meter_until_last_hour") }} as spine + left join + {{ ref("meter_registry_hourly") }} as meter_registry + on + spine.plant_id = meter_registry.plant_id + and spine.start_hour = meter_registry.time_start_hour + left join + {{ ref("satellite_readings__hourly_legacy") }} as satellite_readings + on + spine.plant_id = satellite_readings.plant_id + and spine.start_hour = satellite_readings.start_hour + left join + {{ ref("plantmonitordb_solarevent__generous") }} as solar_events + on + spine.plant_id = solar_events.plant_id + and solar_events.day = spine.start_hour::date + left join + {{ ref("forecasts_hourly") }} as forecast + on + spine.plant_id = forecast.plant_id + and spine.start_hour = forecast.time_start_hour +) select - *, - meter_registry_export_energy_kwh - - satellite_readings_energy_output_kwh as deviation_exported_vs_satellite_expected_kwh, - meter_registry_export_energy_kwh - forecast_energy_kwh as deviation_exported_vs_forecast_expected_kwh, - case - when - meter_registry_export_energy_kwh is not null - and meter_registry_export_energy_kwh > 0 - and satellite_readings_energy_output_kwh is not null - and satellite_readings_energy_output_kwh > 0 - then round(((meter_registry_export_energy_kwh / satellite_readings_energy_output_kwh) - 1)::numeric * 100, 2) - end as p_deviation_exported_vs_expected_satellite, - case - when - meter_registry_export_energy_kwh is not null - and meter_registry_export_energy_kwh > 0 - and plant_peak_power_kw is not null - and plant_peak_power_kw > 0 - and satellite_readings_tilted_irradiation_kwh_m2 is not null - and satellite_readings_tilted_irradiation_kwh_m2 > 0 - then - round( - ( - (meter_registry_export_energy_kwh / plant_peak_power_kw)::float - / satellite_readings_tilted_irradiation_kwh_m2 - )::numeric, - 4 + *, + meter_registry_export_energy_kwh + - satellite_readings_energy_output_kwh as deviation_exported_vs_satellite_expected_kwh, + meter_registry_export_energy_kwh + - forecast_energy_kwh as deviation_exported_vs_forecast_expected_kwh, + case + when + meter_registry_export_energy_kwh is not null + and meter_registry_export_energy_kwh > 0 + and satellite_readings_energy_output_kwh is not null + and satellite_readings_energy_output_kwh > 0 + then + round( + ( + ( + meter_registry_export_energy_kwh + / satellite_readings_energy_output_kwh ) - end as performance_ratio + - 1 + )::numeric + * 100, + 2 + ) + end as p_deviation_exported_vs_expected_satellite, + case + when + meter_registry_export_energy_kwh is not null + and meter_registry_export_energy_kwh > 0 + and plant_peak_power_kw is not null + and plant_peak_power_kw > 0 + and satellite_readings_tilted_irradiation_kwh_m2 is not null + and satellite_readings_tilted_irradiation_kwh_m2 > 0 + then + round( + ( + ( + meter_registry_export_energy_kwh + / plant_peak_power_kw + )::float + / satellite_readings_tilted_irradiation_kwh_m2 + )::numeric, + 4 + ) + end as performance_ratio from combined_meter_satellite order by start_hour desc diff --git a/dbt_jardiner/models/legacy/datasets/plant_production_monthly.sql b/dbt_jardiner/models/legacy/datasets/plant_production_monthly.sql index bd11bc4b..00dc8cab 100644 --- a/dbt_jardiner/models/legacy/datasets/plant_production_monthly.sql +++ b/dbt_jardiner/models/legacy/datasets/plant_production_monthly.sql @@ -6,29 +6,51 @@ with plant_production_daily as ( select - date_trunc('month', day) as month, - plant_id, - plant_name, - plant_codename, - meter_id, - meter_name, - current_meter_connection_protocol, - plant_peak_power_kw, - round(sum(meter_registry_hours_with_readings)/(count(*)*24)*100,2) as p_meter_hours_with_readings, --ens mengem el cavni d'hora - round(sum(satellite_hours_with_readings)/(count(*)*24)*100,2) as p_satellite_hours_with_readings,--ens mengem el cavni d'hora - sum(meter_registry_export_energy_kwh) as meter_registry_export_energy_kwh, - sum(meter_registry_hours_with_energy) as meter_registry_hours_with_energy, - sum(satellite_readings_horizontal_irradiation_kwh_m2) as satellite_readings_horizontal_irradiation_kwh_m2, - sum(satellite_readings_tilted_irradiation_kwh_m2) as satellite_readings_tilted_irradiation_kwh_m2, - round(avg(satellite_readings_module_temperature_dc_mean),2) as satellite_readings_module_temperature_dc_mean, - sum(satellite_readings_energy_output_kwh) as satellite_readings_energy_output_kwh, - sum(solar_hours_real) as solar_hours_real, - sum(solar_hours_minimum) as solar_hours_minimum, - sum(forecast_energy_kwh) as forecast_energy_kwh, - sum(deviation_exported_vs_satellite_expected_kwh) as deviation_exported_vs_satellite_expected_kwh, - sum(deviation_exported_vs_forecast_expected_kwh_cumsum_abs) as deviation_exported_vs_forecast_expected_kwh_cumsum_abs, - sum(unexpected_hours_without_energy) as unexpected_hours_without_energy - from {{ref('plant_production_daily')}} + plant_id, + plant_name, + plant_codename, + meter_id, + meter_name, + current_meter_connection_protocol, + plant_peak_power_kw, + date_trunc('month', day) as month, + --ens mengem el cavni d'hora + round( + sum(meter_registry_hours_with_readings) / (count(*) * 24) * 100, 2 + ) as p_meter_hours_with_readings, + --ens mengem el cavni d'hora + round( + sum(satellite_hours_with_readings) / (count(*) * 24) * 100, 2 + ) as p_satellite_hours_with_readings, + sum( + meter_registry_export_energy_kwh + ) as meter_registry_export_energy_kwh, + sum( + meter_registry_hours_with_energy + ) as meter_registry_hours_with_energy, + sum( + satellite_readings_horizontal_irradiation_kwh_m2 + ) as satellite_readings_horizontal_irradiation_kwh_m2, + sum( + satellite_readings_tilted_irradiation_kwh_m2 + ) as satellite_readings_tilted_irradiation_kwh_m2, + round( + avg(satellite_readings_module_temperature_dc_mean), 2 + ) as satellite_readings_module_temperature_dc_mean, + sum( + satellite_readings_energy_output_kwh + ) as satellite_readings_energy_output_kwh, + sum(solar_hours_real) as solar_hours_real, + sum(solar_hours_minimum) as solar_hours_minimum, + sum(forecast_energy_kwh) as forecast_energy_kwh, + sum( + deviation_exported_vs_satellite_expected_kwh + ) as deviation_exported_vs_satellite_expected_kwh, + sum( + deviation_exported_vs_forecast_expected_kwh_cumsum_abs + ) as deviation_exported_vs_forecast_expected_kwh_cumsum_abs, + sum(unexpected_hours_without_energy) as unexpected_hours_without_energy + from {{ ref('plant_production_daily') }} group by month, plant_id, @@ -39,19 +61,54 @@ with plant_production_daily as ( current_meter_connection_protocol, plant_peak_power_kw ) + select *, - round((deviation_exported_vs_forecast_expected_kwh_cumsum_abs / NULLIF(meter_registry_export_energy_kwh,0)) * 100,2) as deviation_from_forecast_cumsum_ratio, - CASE - WHEN meter_registry_export_energy_kwh is not null and meter_registry_export_energy_kwh > 0 - AND satellite_readings_energy_output_kwh is not null and satellite_readings_energy_output_kwh > 0 - THEN round(((meter_registry_export_energy_kwh / satellite_readings_energy_output_kwh)-1)::numeric*100,2) - END as p_deviation_exported_vs_expected_satellite, - CASE - WHEN meter_registry_export_energy_kwh is not null and meter_registry_export_energy_kwh > 0 - AND plant_peak_power_kw is not null and plant_peak_power_kw > 0 - AND satellite_readings_tilted_irradiation_kwh_m2 is not null and satellite_readings_tilted_irradiation_kwh_m2 > 0 - THEN round(((meter_registry_export_energy_kwh / plant_peak_power_kw)::float / satellite_readings_tilted_irradiation_kwh_m2)::numeric,4) - END as performance_ratio + round( + ( + deviation_exported_vs_forecast_expected_kwh_cumsum_abs + / nullif(meter_registry_export_energy_kwh, 0) + ) + * 100, + 2 + ) as deviation_from_forecast_cumsum_ratio, + case + when + meter_registry_export_energy_kwh is not null + and meter_registry_export_energy_kwh > 0 + and satellite_readings_energy_output_kwh is not null + and satellite_readings_energy_output_kwh > 0 + then + round( + ( + ( + meter_registry_export_energy_kwh + / satellite_readings_energy_output_kwh + ) + - 1 + )::numeric + * 100, + 2 + ) + end as p_deviation_exported_vs_expected_satellite, + case + when + meter_registry_export_energy_kwh is not null + and meter_registry_export_energy_kwh > 0 + and plant_peak_power_kw is not null and plant_peak_power_kw > 0 + and satellite_readings_tilted_irradiation_kwh_m2 is not null + and satellite_readings_tilted_irradiation_kwh_m2 > 0 + then + round( + ( + ( + meter_registry_export_energy_kwh + / plant_peak_power_kw + )::float + / satellite_readings_tilted_irradiation_kwh_m2 + )::numeric, + 4 + ) + end as performance_ratio from plant_production_daily -order by month desc \ No newline at end of file +order by month desc diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_energes.sql b/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_energes.sql index c8579e5c..d50c7887 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_energes.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_energes.sql @@ -1,14 +1,21 @@ {{ config(post_hook="grant select on {{ this }} to group energes") }} - SELECT reg."time", - plant.id AS plant_id, - plant.name AS plant_name, - inverter.name AS inverter_name, - string.name AS string_name, - string.stringbox_name, - reg.intensity_ma - FROM {{ source('plantmonitor_legacy', 'stringregistry') }} as reg - LEFT JOIN {{ source('plantmonitor_legacy', 'string') }} as string ON string.id = reg.string - LEFT JOIN {{ source('plantmonitor_legacy', 'inverter') }} as inverter ON inverter.id = string.inverter - LEFT JOIN {{ source('plantmonitor_legacy', 'plant') }} as plant ON plant.id = inverter.plant - WHERE plant.id in (select id from {{ ref('plants_energes') }}) \ No newline at end of file +select + reg."time", + plant.id as plant_id, + plant.name as plant_name, + inverter.name as inverter_name, + string.name as string_name, + string.stringbox_name, + reg.intensity_ma +from {{ source('plantmonitor_legacy', 'stringregistry') }} as reg + left join + {{ source('plantmonitor_legacy', 'string') }} as string + on reg.string = string.id + left join + {{ source('plantmonitor_legacy', 'inverter') }} as inverter + on string.inverter = inverter.id + left join + {{ source('plantmonitor_legacy', 'plant') }} as plant + on inverter.plant = plant.id +where plant.id in (select id from {{ ref('plants_energes') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_ercam.sql b/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_ercam.sql index ca237f94..051187b2 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_ercam.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_ercam.sql @@ -1,14 +1,21 @@ {{ config(post_hook="grant select on {{ this }} to group ercam") }} - SELECT reg."time", - plant.id AS plant_id, - plant.name AS plant_name, - inverter.name AS inverter_name, - string.name AS string_name, - string.stringbox_name, - reg.intensity_ma - FROM {{ source('plantmonitor_legacy', 'stringregistry') }} as reg - LEFT JOIN {{ source('plantmonitor_legacy', 'string') }} as string ON string.id = reg.string - LEFT JOIN {{ source('plantmonitor_legacy', 'inverter') }} as inverter ON inverter.id = string.inverter - LEFT JOIN {{ source('plantmonitor_legacy', 'plant') }} as plant ON plant.id = inverter.plant - WHERE plant.id in (select id from {{ ref('plants_ercam') }}) \ No newline at end of file +select + reg."time", + plant.id as plant_id, + plant.name as plant_name, + inverter.name as inverter_name, + string.name as string_name, + string.stringbox_name, + reg.intensity_ma +from {{ source('plantmonitor_legacy', 'stringregistry') }} as reg + left join + {{ source('plantmonitor_legacy', 'string') }} as string + on reg.string = string.id + left join + {{ source('plantmonitor_legacy', 'inverter') }} as inverter + on string.inverter = inverter.id + left join + {{ source('plantmonitor_legacy', 'plant') }} as plant + on inverter.plant = plant.id +where plant.id in (select id from {{ ref('plants_ercam') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_exiom.sql b/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_exiom.sql index 736e059c..f87d076b 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_exiom.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/inverter_intensity_exiom.sql @@ -1,14 +1,21 @@ {{ config(post_hook="grant select on {{ this }} to group exiom") }} - SELECT reg."time", - plant.id AS plant_id, - plant.name AS plant_name, - inverter.name AS inverter_name, - string.name AS string_name, - string.stringbox_name, - reg.intensity_ma - FROM {{ ref('stringregistry_multisource') }} as reg - LEFT JOIN {{ source('plantmonitor_legacy', 'string') }} as string ON string.id = reg.string_id - LEFT JOIN {{ source('plantmonitor_legacy', 'inverter') }} as inverter ON inverter.id = string.inverter - LEFT JOIN {{ source('plantmonitor_legacy', 'plant') }} as plant ON plant.id = inverter.plant - WHERE plant.id in (select id from {{ ref('plants_exiom') }}) \ No newline at end of file +select + reg."time", + plant.id as plant_id, + plant.name as plant_name, + inverter.name as inverter_name, + string.name as string_name, + string.stringbox_name, + reg.intensity_ma +from {{ ref('stringregistry_multisource') }} as reg + left join + {{ source('plantmonitor_legacy', 'string') }} as string + on reg.string_id = string.id + left join + {{ source('plantmonitor_legacy', 'inverter') }} as inverter + on string.inverter = inverter.id + left join + {{ source('plantmonitor_legacy', 'plant') }} as plant + on inverter.plant = plant.id +where plant.id in (select id from {{ ref('plants_exiom') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_energes.sql b/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_energes.sql index 7aaec8b0..237375c2 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_energes.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_energes.sql @@ -1,4 +1,4 @@ {{ config(post_hook="grant select on {{ this }} to group energes") }} select * from {{ ref('inverterregistry_clean') }} -where plant_id in (select id from {{ ref('plants_energes') }}) \ No newline at end of file +where plant_id in (select id from {{ ref('plants_energes') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_ercam.sql b/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_ercam.sql index 36803cac..782061f6 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_ercam.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_ercam.sql @@ -1,4 +1,4 @@ {{ config(post_hook="grant select on {{ this }} to group ercam") }} select * from {{ ref('inverterregistry_clean') }} -where plant_id in (select id from {{ ref('plants_ercam') }}) \ No newline at end of file +where plant_id in (select id from {{ ref('plants_ercam') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_exiom.sql b/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_exiom.sql index 8180d94f..33db01d5 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_exiom.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/inverterregistry_exiom.sql @@ -1,4 +1,4 @@ {{ config(post_hook="grant select on {{ this }} to group exiom") }} select * from {{ ref('inverterregistry_clean') }} -where plant_id in (select id from {{ ref('plants_exiom') }}) \ No newline at end of file +where plant_id in (select id from {{ ref('plants_exiom') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/meterregistry_energetica.sql b/dbt_jardiner/models/legacy/datasets/tecnics/meterregistry_energetica.sql index cd100907..07543278 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/meterregistry_energetica.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/meterregistry_energetica.sql @@ -1,7 +1,7 @@ {{ config(post_hook="grant select on {{ this }} to group energetica") }} select - time_start_hour as time, + time_start_hour as time, --noqa: RF04 export_energy_wh, import_energy_wh, meter_id, @@ -11,4 +11,4 @@ select plant_name, plant_code from {{ ref('meter_registry') }} -where plant_id in (select id from {{ ref('plants_energetica') }}) \ No newline at end of file +where plant_id in (select id from {{ ref('plants_energetica') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_energes.sql b/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_energes.sql index 526dea2f..ce59e8d4 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_energes.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_energes.sql @@ -1,13 +1,18 @@ {{ config(post_hook="grant select on {{ this }} to group energes") }} -SELECT reg."time", - plant.id AS plant_id, - plant.name AS plant_name, - reg.sensor AS sensor_id, - sensor.name AS sensor_name, - reg.irradiation_w_m2, - reg.temperature_dc -FROM {{source('plantmonitor_legacy','sensorirradiationregistry')}} reg - LEFT JOIN {{source('plantmonitor_legacy','sensor')}} as sensor ON (sensor.id = reg.sensor) - LEFT JOIN {{source('plantmonitor_legacy','plant')}} as plant ON (plant.id = sensor.plant) -where plant.id in (select id from {{ ref('plants_energes') }}) \ No newline at end of file +select + reg."time", + plant.id as plant_id, + plant.name as plant_name, + reg.sensor as sensor_id, + sensor.name as sensor_name, + reg.irradiation_w_m2, + reg.temperature_dc +from {{ source('plantmonitor_legacy','sensorirradiationregistry') }} as reg + left join + {{ source('plantmonitor_legacy','sensor') }} as sensor + on (reg.sensor = sensor.id) + left join + {{ source('plantmonitor_legacy','plant') }} as plant + on (sensor.plant = plant.id) +where plant.id in (select id from {{ ref('plants_energes') }}) diff --git a/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_ercam.sql b/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_ercam.sql index 31c50587..7e383000 100644 --- a/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_ercam.sql +++ b/dbt_jardiner/models/legacy/datasets/tecnics/sensorirradiationregistry_ercam.sql @@ -1,13 +1,18 @@ {{ config(post_hook="grant select on {{ this }} to group ercam") }} -SELECT reg."time", - plant.id AS plant_id, - plant.name AS plant_name, - reg.sensor AS sensor_id, - sensor.name AS sensor_name, - reg.irradiation_w_m2, - reg.temperature_dc -FROM {{source('plantmonitor_legacy','sensorirradiationregistry')}} reg - LEFT JOIN {{source('plantmonitor_legacy','sensor')}} as sensor ON (sensor.id = reg.sensor) - LEFT JOIN {{source('plantmonitor_legacy','plant')}} as plant ON (plant.id = sensor.plant) -where plant.id in (select id from {{ ref('plants_ercam') }}) \ No newline at end of file +select + reg."time", + plant.id as plant_id, + plant.name as plant_name, + reg.sensor as sensor_id, + sensor.name as sensor_name, + reg.irradiation_w_m2, + reg.temperature_dc +from {{ source('plantmonitor_legacy','sensorirradiationregistry') }} as reg + left join + {{ source('plantmonitor_legacy','sensor') }} as sensor + on (reg.sensor = sensor.id) + left join + {{ source('plantmonitor_legacy','plant') }} as plant + on (sensor.plant = plant.id) +where plant.id in (select id from {{ ref('plants_ercam') }}) diff --git a/dbt_jardiner/models/legacy/operational/derived/view_availability.sql b/dbt_jardiner/models/legacy/operational/derived/view_availability.sql index 6f063e10..caae691b 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_availability.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_availability.sql @@ -1,13 +1,14 @@ -{{config(materialized='view')}} +{{ config(materialized='view') }} -SELECT - date_trunc('month', view_ht_daily."time") AS "time", - view_ht_daily.plant, - sum(hd) as hd, - sum(ht) as ht, - sum(hd)::float/sum(ht) AS "availability" -FROM {{ref('view_ht_daily')}} as view_ht_daily -JOIN {{ref('view_hd_daily')}} as view_hd_daily - ON view_hd_daily.time = view_ht_daily.time - and view_hd_daily.plant = view_ht_daily.plant -group by date_trunc('month', view_ht_daily."time"), view_ht_daily.plant \ No newline at end of file +select + view_ht_daily.plant, + date_trunc('month', view_ht_daily."time") as "time", + sum(view_hd_daily.hd) as hd, + sum(view_ht_daily.ht) as ht, + sum(view_hd_daily.hd)::float / sum(view_ht_daily.ht) as "availability" +from {{ ref('view_ht_daily') }} as view_ht_daily + inner join {{ ref('view_hd_daily') }} as view_hd_daily + on + view_ht_daily.time = view_hd_daily.time + and view_ht_daily.plant = view_hd_daily.plant +group by date_trunc('month', view_ht_daily."time"), view_ht_daily.plant diff --git a/dbt_jardiner/models/legacy/operational/derived/view_clean_irradiation.sql b/dbt_jardiner/models/legacy/operational/derived/view_clean_irradiation.sql index 8a0e1380..c364103b 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_clean_irradiation.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_clean_irradiation.sql @@ -6,20 +6,32 @@ Time it to decide #} {# this view is a legacy view which #} - SELECT sir.sensor, - sir."time", - COALESCE(sir.irradiation_wh_m2, sat.global_tilted_irradiation_wh_m2) AS irradiation_wh_m2_coalesce, - sir.irradiation_wh_m2 AS sonda_irradiation_wh_m2, - sat.global_tilted_irradiation_wh_m2 AS satellite_irradiation_wh_m2, - CASE - WHEN sir.quality < 1 THEN sat.global_tilted_irradiation_wh_m2 - ELSE sir.irradiation_wh_m2 - END AS irradiation_wh_m2, - CASE - WHEN sir.quality < 1 THEN 'satellite' - ELSE 'sonda' - END AS reading_source - FROM {{ source('plantmonitor_legacy', 'irradiationregistry') }} sir - LEFT JOIN {{ source('plantmonitor_legacy', 'sensor') }} sensor ON sensor.id = sir.sensor - LEFT JOIN {{ source('plantmonitor_legacy', 'plant') }} plant ON plant.id = sensor.plant - LEFT JOIN {{ source('plantmonitor_legacy', 'satellite_readings') }} sat ON sat.plant = sensor.plant AND time_bucket('1 hour', sat."time") = time_bucket('1 hour', sir."time") \ No newline at end of file +select + sir.sensor, + sir."time", + sir.irradiation_wh_m2 as sonda_irradiation_wh_m2, + sat.global_tilted_irradiation_wh_m2 as satellite_irradiation_wh_m2, + coalesce( + sir.irradiation_wh_m2, sat.global_tilted_irradiation_wh_m2 + ) as irradiation_wh_m2_coalesce, + case + when sir.quality < 1 then sat.global_tilted_irradiation_wh_m2 + else sir.irradiation_wh_m2 + end as irradiation_wh_m2, + case + when sir.quality < 1 then 'satellite' + else 'sonda' + end as reading_source +from {{ source('plantmonitor_legacy', 'irradiationregistry') }} as sir + left join + {{ source('plantmonitor_legacy', 'sensor') }} as sensor + on sir.sensor = sensor.id + left join + {{ source('plantmonitor_legacy', 'plant') }} as plant + on sensor.plant = plant.id + left join + {{ source('plantmonitor_legacy', 'satellite_readings') }} as sat + on + sensor.plant = sat.plant + and time_bucket('1 hour', sat."time") + = time_bucket('1 hour', sir."time") diff --git a/dbt_jardiner/models/legacy/operational/derived/view_cos_phi_monthly.sql b/dbt_jardiner/models/legacy/operational/derived/view_cos_phi_monthly.sql index d74a6f93..3522d781 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_cos_phi_monthly.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_cos_phi_monthly.sql @@ -1,15 +1,26 @@ {{ config(materialized='view') }} select - "time", - plant_id, - meter_id, - import_energy_wh, - export_energy_wh, - qimportada, - qexportada, - sqrt(qimportada*qimportada + export_energy_wh*export_energy_wh) as simportada, --pendent de validar amb el grafana, mensual = suma(horaria), segur? - sqrt(qexportada*qexportada + import_energy_wh*import_energy_wh) as sexportada, - qimportada / NULLIF(sqrt(qimportada*qimportada + export_energy_wh*export_energy_wh),0) as cos_phi_importacio, - qexportada / NULLIF(sqrt(qexportada*qexportada + import_energy_wh*import_energy_wh),0) as cos_phi_exportacio -from {{ref('view_q_monthly')}} \ No newline at end of file + "time", + plant_id, + meter_id, + import_energy_wh, + export_energy_wh, + qimportada, + qexportada, + --pendent de validar amb el grafana, mensual = suma(horaria), segur? + sqrt( + qimportada * qimportada + export_energy_wh * export_energy_wh + ) as simportada, + sqrt( + qexportada * qexportada + import_energy_wh * import_energy_wh + ) as sexportada, + qimportada + / nullif( + sqrt(qimportada * qimportada + export_energy_wh * export_energy_wh), 0 + ) as cos_phi_importacio, + qexportada + / nullif( + sqrt(qexportada * qexportada + import_energy_wh * import_energy_wh), 0 + ) as cos_phi_exportacio +from {{ ref('view_q_monthly') }} diff --git a/dbt_jardiner/models/legacy/operational/derived/view_expected_power.sql b/dbt_jardiner/models/legacy/operational/derived/view_expected_power.sql index 2640b391..826e5293 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_expected_power.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_expected_power.sql @@ -1,34 +1,43 @@ {{ config(materialized='view') }} -SELECT - time, - sensor.plant AS plant, - sensor.id AS sensor, - CASE WHEN reg.irradiation_w_m2 <=0 THEN 0 ELSE ( - par.n_modules - * par.max_power_current_ma/1000.0 - * par.max_power_voltage_mv/1000.0 - * ( - reg.irradiation_w_m2 / par.standard_conditions_irradiation_w_m2::float + - (reg.temperature_dc - par.standard_conditions_temperature_dc)/10.0 * par.current_temperature_coefficient_mpercent_c/100000.0 - ) - * ( - 1 + - (reg.temperature_dc - par.standard_conditions_temperature_dc)/10.0 * par.voltage_temperature_coefficient_mpercent_c/100000.0 - ) - * par.degradation_cpercent/10000.0 -- c% -> factor - * par.expected_power_correction_factor_cpercent/10000.0 -- c% -> factor - / 1000.0 -- W -> kW +select + reg.time, + sensor.plant as plant, + sensor.id as sensor, + reg.temperature_dc, + reg.irradiation_w_m2, + case + when reg.irradiation_w_m2 <= 0 then 0 else ( + par.n_modules + * par.max_power_current_ma / 1000.0 + * par.max_power_voltage_mv / 1000.0 + * ( + reg.irradiation_w_m2 + / par.standard_conditions_irradiation_w_m2::float + + (reg.temperature_dc - par.standard_conditions_temperature_dc) + / 10.0 + * par.current_temperature_coefficient_mpercent_c + / 100000.0 + ) + * ( + 1 + + (reg.temperature_dc - par.standard_conditions_temperature_dc) + / 10.0 + * par.voltage_temperature_coefficient_mpercent_c + / 100000.0 + ) + * par.degradation_cpercent / 10000.0 -- c% -> factor + -- c% -> factor + * par.expected_power_correction_factor_cpercent / 10000.0 + / 1000.0 -- W -> kW ) - END AS expectedpower, - reg.temperature_dc, - reg.irradiation_w_m2 - FROM {{source('plantmonitor_legacy','sensorirradiationregistry')}} AS reg - LEFT JOIN {{source('plantmonitor_legacy','sensor')}} AS sensor - ON sensor.id = reg.sensor - LEFT JOIN {{source('plantmonitor_legacy','plantmoduleparameters')}} AS par - ON par.plant = sensor.plant - ORDER BY time, plant, sensor + end as expectedpower +from {{ source('plantmonitor_legacy','sensorirradiationregistry') }} as reg + left join {{ source('plantmonitor_legacy','sensor') }} as sensor + on reg.sensor = sensor.id + left join {{ source('plantmonitor_legacy','plantmoduleparameters') }} as par + on sensor.plant = par.plant +order by reg.time, sensor.plant, sensor.id {# Expected Power @@ -61,5 +70,3 @@ factorV = 1 + (T(t)-T_{sc}) * C_{coefTempVoc} factorI = G(t)/G_{SCT} + (T(t)-T_{sc}) * C_{coefTempIsc} nModules I_{mp} V_ {mp} * factorV * factorI * degradacion/100.0 / 1000.0 #} - - diff --git a/dbt_jardiner/models/legacy/operational/derived/view_hd_daily.sql b/dbt_jardiner/models/legacy/operational/derived/view_hd_daily.sql index c378dd69..6206a905 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_hd_daily.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_hd_daily.sql @@ -1,8 +1,11 @@ -{{config(materialized='view')}} +{{ config(materialized='view') }} -SELECT date_trunc('day', "time" at time zone 'Europe/Madrid') at time zone 'Europe/Madrid' AS "time", - plant, - count(*) AS hd -FROM {{ref('view_pr_hourly')}} -WHERE pr_hourly > 0.70 -GROUP BY date_trunc('day', "time" at time zone 'Europe/Madrid'), plant \ No newline at end of file +select + plant, + date_trunc( + 'day', "time" at time zone 'Europe/Madrid' + ) at time zone 'Europe/Madrid' as "time", + count(*) as hd +from {{ ref('view_pr_hourly') }} +where pr_hourly > 0.70 +group by date_trunc('day', "time" at time zone 'Europe/Madrid'), plant diff --git a/dbt_jardiner/models/legacy/operational/derived/view_ht_daily.sql b/dbt_jardiner/models/legacy/operational/derived/view_ht_daily.sql index b684c24d..2aed7c27 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_ht_daily.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_ht_daily.sql @@ -1,11 +1,15 @@ -{{config(materialized='view')}} +{{ config(materialized='view') }} -SELECT reg.plant, - date_trunc('day', reg.time AT TIME ZONE 'Europe/Madrid') AS "time", - count(*) AS ht -FROM {{ref('view_satellite_irradiation')}} as reg -INNER JOIN {{source('plantmonitor_legacy', 'plant')}} as plant ON reg.plant = plant.id -WHERE irradiation_wh_m2 > 5 -GROUP BY date_trunc('day', reg.time AT TIME ZONE 'Europe/Madrid'), - reg.plant -ORDER BY date_trunc('day', reg.time AT TIME ZONE 'Europe/Madrid') +select + reg.plant, + date_trunc('day', reg.time at time zone 'Europe/Madrid') as "time", + count(*) as ht +from {{ ref('view_satellite_irradiation') }} as reg + inner join + {{ source('plantmonitor_legacy', 'plant') }} as plant + on reg.plant = plant.id +where reg.irradiation_wh_m2 > 5 +group by + date_trunc('day', reg.time at time zone 'Europe/Madrid'), + reg.plant +order by date_trunc('day', reg.time at time zone 'Europe/Madrid') diff --git a/dbt_jardiner/models/legacy/operational/derived/view_ht_monthly.sql b/dbt_jardiner/models/legacy/operational/derived/view_ht_monthly.sql index e109dc11..0525736e 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_ht_monthly.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_ht_monthly.sql @@ -1,11 +1,15 @@ -{{config(materialized='view')}} +{{ config(materialized='view') }} -SELECT reg.plant, - date_trunc('month', reg.time AT TIME ZONE 'Europe/Madrid') AS "time", - count(*) AS ht -FROM {{ref('view_satellite_irradiation')}} as reg -INNER JOIN {{source('plantmonitor_legacy', 'plant')}} as plant ON reg.plant = plant.id -WHERE irradiation_wh_m2 > 5 -GROUP BY date_trunc('month', reg.time AT TIME ZONE 'Europe/Madrid'), - reg.plant -ORDER BY date_trunc('month', reg.time AT TIME ZONE 'Europe/Madrid') \ No newline at end of file +select + reg.plant, + date_trunc('month', reg.time at time zone 'Europe/Madrid') as "time", + count(*) as ht +from {{ ref('view_satellite_irradiation') }} as reg + inner join + {{ source('plantmonitor_legacy', 'plant') }} as plant + on reg.plant = plant.id +where reg.irradiation_wh_m2 > 5 +group by + date_trunc('month', reg.time at time zone 'Europe/Madrid'), + reg.plant +order by date_trunc('month', reg.time at time zone 'Europe/Madrid') diff --git a/dbt_jardiner/models/legacy/operational/derived/view_inverter_metrics_daily.sql b/dbt_jardiner/models/legacy/operational/derived/view_inverter_metrics_daily.sql index 8f20e624..cf1ba410 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_inverter_metrics_daily.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_inverter_metrics_daily.sql @@ -1,17 +1,28 @@ - {# TODO: change inverterregistry for inverterregistry_clean which includes asomada#} +{# TODO: change inverterregistry for inverterregistry_clean which includes asomada#} - SELECT date_trunc('day'::text, inverterregistry."time") AS "time", - plant.id AS plant, - plant.name AS plant_name, - inverter.id AS inverter, - inverter.name AS inverter_name, - max(inverterregistry.energy_wh) AS energy_wh, - max(inverterregistry.temperature_dc) AS max_temperature_dc, - min(inverterregistry.temperature_dc) AS min_temperature_dc, - avg(inverterregistry.temperature_dc) AS avg_temperature_dc, - max(inverterregistry.power_w) AS power_w - FROM {{ source('plantmonitor_legacy', 'inverterregistry') }} - LEFT JOIN inverter ON inverter.id = inverterregistry.inverter - LEFT JOIN plant ON plant.id = inverter.plant - GROUP BY (date_trunc('day'::text, inverterregistry."time")), plant.id, plant.name, inverter.id, inverter.name - ORDER BY (date_trunc('day'::text, inverterregistry."time")), plant.id, plant.name, inverter.id, inverter.name \ No newline at end of file +select + plant.id as plant, + plant.name as plant_name, + inverter.id as inverter, + inverter.name as inverter_name, + date_trunc('day'::text, inverterregistry."time") as "time", + max(inverterregistry.energy_wh) as energy_wh, + max(inverterregistry.temperature_dc) as max_temperature_dc, + min(inverterregistry.temperature_dc) as min_temperature_dc, + avg(inverterregistry.temperature_dc) as avg_temperature_dc, + max(inverterregistry.power_w) as power_w +from {{ source('plantmonitor_legacy', 'inverterregistry') }} as inverterregistry + left join inverter on inverter.id = inverterregistry.inverter + left join plant on inverter.plant = plant.id +group by + (date_trunc('day'::text, inverterregistry."time")), + plant.id, + plant.name, + inverter.id, + inverter.name +order by + (date_trunc('day'::text, inverterregistry."time")), + plant.id, + plant.name, + inverter.id, + inverter.name diff --git a/dbt_jardiner/models/legacy/operational/derived/view_plant_export_energy_monthly.sql b/dbt_jardiner/models/legacy/operational/derived/view_plant_export_energy_monthly.sql index 3bf5c1ac..b4782954 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_plant_export_energy_monthly.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_plant_export_energy_monthly.sql @@ -1,23 +1,37 @@ {{ config(materialized='view') }} -SELECT - date_trunc('month', (meterregistry."time" - interval '1 hour'), 'Europe/Madrid') as "time", - plant.id AS plant_id, - plant.name AS plant_name, - meter.id AS meter_id, - meter.name AS meter_name, - sum(meterregistry.export_energy_wh) AS export_energy_wh -FROM - {{ source('plantmonitor_legacy','meter') }} as meter -LEFT JOIN - {{ source('plantmonitor_legacy','plant') }} as plant -ON - plant.id = meter.plant -LEFT JOIN - {{ source('plantmonitor_legacy','meterregistry') }} as meterregistry -ON - meter.id = meterregistry.meter -GROUP BY - date_trunc('month', (meterregistry."time" - interval '1 hour'), 'Europe/Madrid'), plant_id, plant_name, meter_id, meter_name -ORDER BY - date_trunc('month', (meterregistry."time" - interval '1 hour'), 'Europe/Madrid'), plant_id, plant_name, meter_id, meter_name +select + plant.id as plant_id, + plant.name as plant_name, + meter.id as meter_id, + meter.name as meter_name, + date_trunc( + 'month', (meterregistry."time" - interval '1 hour'), 'Europe/Madrid' + ) as "time", + sum(meterregistry.export_energy_wh) as export_energy_wh +from + {{ source('plantmonitor_legacy','meter') }} as meter + left join + {{ source('plantmonitor_legacy','plant') }} as plant + on + meter.plant = plant.id + left join + {{ source('plantmonitor_legacy','meterregistry') }} as meterregistry + on + meter.id = meterregistry.meter +group by + date_trunc( + 'month', (meterregistry."time" - interval '1 hour'), 'Europe/Madrid' + ), + plant_id, + plant_name, + meter_id, + meter_name +order by + date_trunc( + 'month', (meterregistry."time" - interval '1 hour'), 'Europe/Madrid' + ), + plant_id, + plant_name, + meter_id, + meter_name diff --git a/dbt_jardiner/models/legacy/operational/derived/view_pr_hourly.sql b/dbt_jardiner/models/legacy/operational/derived/view_pr_hourly.sql index 42a4a0ab..2dc548ad 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_pr_hourly.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_pr_hourly.sql @@ -1,18 +1,23 @@ {{ config(materialized='view') }} - SELECT - meterregistry."time" AS "time", - plant.id as plant, - meter.id as meter, - export_energy_wh, - irr.irradiation_wh_m2, - (export_energy_wh / plantparameters.peak_power_w::float) / (NULLIF(irr.irradiation_wh_m2, 0.0) / 1000.0) AS pr_hourly - FROM {{source ('plantmonitor_legacy', 'meterregistry')}} as meterregistry - JOIN {{source ('plantmonitor_legacy', 'meter')}} as meter - ON meterregistry.meter = meter.id - JOIN {{source ('plantmonitor_legacy', 'view_satellite_irradiation')}} as irr - ON meter.plant = irr.plant and irr."time" + interval '1 hour' = meterregistry."time" - JOIN {{source ('plantmonitor_legacy', 'plant')}} as plant - ON meter.plant = plant.id - LEFT JOIN {{source ('plantmonitor_legacy', 'plantparameters')}} as plantparameters - on plant.id = plantparameters.plant \ No newline at end of file +select + meterregistry."time" as "time", + plant.id as plant, + meter.id as meter, + meterregistry.export_energy_wh, + irr.irradiation_wh_m2, + (meterregistry.export_energy_wh / plantparameters.peak_power_w::float) + / (nullif(irr.irradiation_wh_m2, 0.0) / 1000.0) as pr_hourly +from {{ source ('plantmonitor_legacy', 'meterregistry') }} as meterregistry + inner join {{ source ('plantmonitor_legacy', 'meter') }} as meter + on meterregistry.meter = meter.id + inner join + {{ source ('plantmonitor_legacy', 'view_satellite_irradiation') }} as irr + on + meter.plant = irr.plant + and irr."time" + interval '1 hour' = meterregistry."time" + inner join {{ source ('plantmonitor_legacy', 'plant') }} as plant + on meter.plant = plant.id + left join + {{ source ('plantmonitor_legacy', 'plantparameters') }} as plantparameters + on plant.id = plantparameters.plant diff --git a/dbt_jardiner/models/legacy/operational/derived/view_q_monthly.sql b/dbt_jardiner/models/legacy/operational/derived/view_q_monthly.sql index a78ec69b..3b68d8fa 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_q_monthly.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_q_monthly.sql @@ -1,17 +1,20 @@ {{ config(materialized='view') }} select - date_trunc('month', reg.time at time zone 'Europe/Madrid') at time zone 'Europe/Madrid' AS time, - plant.id as plant_id, - meter.id as meter_id, - sum(reg.import_energy_wh) AS import_energy_wh, - sum(reg.export_energy_wh) AS export_energy_wh, - sum(reg.r1_varh-reg.r4_varh) AS qimportada, - sum(reg.r2_varh-reg.r3_varh) AS qexportada -from {{source('plantmonitor_legacy','meterregistry')}} as reg -left join {{source('plantmonitor_legacy','meter')}} on meter.id = reg.meter -left join {{source('plantmonitor_legacy','plant')}} on plant.id = meter.plant -GROUP BY date_trunc('month', reg.time at time zone 'Europe/Madrid'), - plant.id, - reg.meter, - meter.id \ No newline at end of file + plant.id as plant_id, + meter.id as meter_id, + date_trunc( + 'month', reg.time at time zone 'Europe/Madrid' + ) at time zone 'Europe/Madrid' as time, --noqa: RF04 + sum(reg.import_energy_wh) as import_energy_wh, + sum(reg.export_energy_wh) as export_energy_wh, + sum(reg.r1_varh - reg.r4_varh) as qimportada, + sum(reg.r2_varh - reg.r3_varh) as qexportada +from {{ source('plantmonitor_legacy','meterregistry') }} as reg + left join {{ source('plantmonitor_legacy','meter') }} as meter on meter.id = reg.meter + left join {{ source('plantmonitor_legacy','plant') }} as plant on plant.id = meter.plant +group by + date_trunc('month', reg.time at time zone 'Europe/Madrid'), + plant.id, + reg.meter, + meter.id diff --git a/dbt_jardiner/models/legacy/operational/derived/view_satellite_irradiation.sql b/dbt_jardiner/models/legacy/operational/derived/view_satellite_irradiation.sql index ca1d8e19..9e6a8c3f 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_satellite_irradiation.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_satellite_irradiation.sql @@ -1,10 +1,11 @@ -{{config(materialized='view')}} +{{ config(materialized='view') }} -SELECT sir.plant, - time_bucket('1 hour', sir."time") as time, - sir.global_tilted_irradiation_wh_m2 AS irradiation_wh_m2 -FROM {{source('plantmonitor_legacy', 'satellite_readings')}} as sir +select + sir.plant, + sir.global_tilted_irradiation_wh_m2 as irradiation_wh_m2, + time_bucket('1 hour', sir."time") as time --noqa: RF04 +from {{ source('plantmonitor_legacy', 'satellite_readings') }} as sir -- https://solargis.atlassian.net/wiki/spaces/public/pages/7602367/Solargis+API+User+Guide#SolargisAPIUserGuide-XMLresponse --- #GTI - Global tilted irradiance [W/m2] (fixed inclination: 25 deg. azimuth: 180 deg.), no data value -9 \ No newline at end of file +-- #GTI - Global tilted irradiance [W/m2] (fixed inclination: 25 deg. azimuth: 180 deg.), no data value -9 diff --git a/dbt_jardiner/models/legacy/operational/derived/view_target_energy.sql b/dbt_jardiner/models/legacy/operational/derived/view_target_energy.sql index 71c875ee..83a8f382 100644 --- a/dbt_jardiner/models/legacy/operational/derived/view_target_energy.sql +++ b/dbt_jardiner/models/legacy/operational/derived/view_target_energy.sql @@ -1,18 +1,18 @@ -{{config(materialized='view', enabled=false)}} +{{ config(materialized='view', enabled=false) }} {# needs generalizing (it's alcolea) #} -SELECT * FROM (values ('2020-01-01'::TIMESTAMP WITH TIME ZONE,221730), -('2020-02-01',228929), -('2020-03-01',320482), -('2020-04-01',300308), -('2020-05-01',342791), -('2020-06-01',333266), -('2020-07-01',361993), -('2020-08-01',344381), -('2020-09-01',318789), -('2020-10-01',279877), -('2020-11-01',200209), -('2020-12-01',199275) +SELECT * FROM ( + VALUES ('2020-01-01'::TIMESTAMP WITH TIME ZONE, 221730), + ('2020-02-01', 228929), + ('2020-03-01', 320482), + ('2020-04-01', 300308), + ('2020-05-01', 342791), + ('2020-06-01', 333266), + ('2020-07-01', 361993), + ('2020-08-01', 344381), + ('2020-09-01', 318789), + ('2020-10-01', 279877), + ('2020-11-01', 200209), + ('2020-12-01', 199275) ) -AS t(time,target_energy_kwh) \ No newline at end of file diff --git a/dbt_jardiner/models/legacy/operational/external_data/energy_forecasts__denormalized_from_plantmonitordb.sql b/dbt_jardiner/models/legacy/operational/external_data/energy_forecasts__denormalized_from_plantmonitordb.sql index 0368087c..661b4bbd 100644 --- a/dbt_jardiner/models/legacy/operational/external_data/energy_forecasts__denormalized_from_plantmonitordb.sql +++ b/dbt_jardiner/models/legacy/operational/external_data/energy_forecasts__denormalized_from_plantmonitordb.sql @@ -1,29 +1,17 @@ -{{ config( - materialized = 'view' -) }} +{{ config(materialized = 'view') }} -WITH forecast_denormalized AS ( +with forecast_denormalized as ( - SELECT - forecast.time AS "time", - forecastmetadata.plant AS plant_id, - forecastmetadata.forecastdate AS forecastdate, - ROUND( - forecast.percentil50 / 1000.0, - 2 - ) AS energy_kwh - FROM - {{ source( - 'meteologica_legacy', - 'forecast' - ) }} - LEFT JOIN {{ source( - 'meteologica_legacy', - 'forecastmetadata' - ) }} - ON forecastmetadata.id = forecast.forecastmetadata + select + forecast.time as "time", + forecastmetadata.plant as plant_id, + forecastmetadata.forecastdate as forecastdate, + round(forecast.percentil50 / 1000.0, 2) as energy_kwh + from {{ source('meteologica_legacy','forecast') }} as forecast + left join {{ source('meteologica_legacy', 'forecastmetadata') }} as forecastmetadata + on forecastmetadata.id = forecast.forecastmetadata ) -SELECT - * -FROM - forecast_denormalized + +select * +from + forecast_denormalized diff --git a/dbt_jardiner/models/legacy/operational/external_data/forecasts_daily.sql b/dbt_jardiner/models/legacy/operational/external_data/forecasts_daily.sql index bb99a6f8..335cae4c 100644 --- a/dbt_jardiner/models/legacy/operational/external_data/forecasts_daily.sql +++ b/dbt_jardiner/models/legacy/operational/external_data/forecasts_daily.sql @@ -1,8 +1,8 @@ {{ config(materialized='view') }} select - date_trunc('day', time_start_hour) as day, - plant_id, - sum(energy_kwh) as energy_kwh -FROM {{ref('forecasts_hourly')}} as srh + plant_id, + date_trunc('day', time_start_hour) as day, + sum(energy_kwh) as energy_kwh +from {{ ref('forecasts_hourly') }} group by date_trunc('day', time_start_hour), plant_id diff --git a/dbt_jardiner/models/legacy/operational/external_data/forecasts_hourly.sql b/dbt_jardiner/models/legacy/operational/external_data/forecasts_hourly.sql index aa95b583..291f1d53 100644 --- a/dbt_jardiner/models/legacy/operational/external_data/forecasts_hourly.sql +++ b/dbt_jardiner/models/legacy/operational/external_data/forecasts_hourly.sql @@ -2,21 +2,23 @@ materialized = 'view' ) }} -SELECT - *, - DATE_TRUNC( - 'hour', - "time" - ) AS time_start_hour -FROM - ( - SELECT - *, - ROW_NUMBER() OVER (PARTITION BY plant_id, DATE_TRUNC('hour', "time") - ORDER BY - forecastdate DESC) AS ranking - FROM - {{ ref('energy_forecasts__denormalized_from_plantmonitordb') }} - ) forecast -WHERE - ranking = 1 +select + *, + date_trunc( + 'hour', + "time" + ) as time_start_hour +from + ( + select + *, + row_number() over ( + partition by plant_id, date_trunc('hour', "time") + order by + forecastdate desc + ) as ranking + from + {{ ref('energy_forecasts__denormalized_from_plantmonitordb') }} + ) as forecast +where + ranking = 1 diff --git a/dbt_jardiner/models/legacy/operational/external_data/plantmonitordb_solarevent__generous.sql b/dbt_jardiner/models/legacy/operational/external_data/plantmonitordb_solarevent__generous.sql index bd975f47..47f30fd1 100644 --- a/dbt_jardiner/models/legacy/operational/external_data/plantmonitordb_solarevent__generous.sql +++ b/dbt_jardiner/models/legacy/operational/external_data/plantmonitordb_solarevent__generous.sql @@ -3,11 +3,20 @@ select plant as plant_id, - date_trunc('day', sunrise) as day, sunrise as sunrise_real, sunset as sunset_real, + date_trunc('day', sunrise) as day, sunrise + interval '2 hours' as sunrise_generous, sunset - interval '2 hours' as sunset_generous, - round(EXTRACT(EPOCH FROM (sunset - sunrise))::numeric/3600,2) as solar_hours_real, - round(EXTRACT(EPOCH FROM (sunset - interval '2 hours') - (sunrise + interval '2 hours'))::numeric/3600,2) as solar_hours_minimum -from {{source('plantmonitor_legacy', 'solarevent')}} + round( + extract(epoch from (sunset - sunrise))::numeric / 3600, 2 + ) as solar_hours_real, + round( + extract( + epoch from (sunset - interval '2 hours') + - (sunrise + interval '2 hours') + )::numeric + / 3600, + 2 + ) as solar_hours_minimum +from {{ source('plantmonitor_legacy', 'solarevent') }} diff --git a/dbt_jardiner/models/legacy/operational/external_data/satellite_readings_daily.sql b/dbt_jardiner/models/legacy/operational/external_data/satellite_readings_daily.sql index 57e71839..f6bd8202 100644 --- a/dbt_jardiner/models/legacy/operational/external_data/satellite_readings_daily.sql +++ b/dbt_jardiner/models/legacy/operational/external_data/satellite_readings_daily.sql @@ -1,13 +1,12 @@ {{ config(materialized='view') }} select - date_trunc('day', start_hour) as day, - plant_id, - sum(horizontal_irradiation_wh_m2) as horizontal_irradiation_wh_m2, - sum(tilted_irradiation_wh_m2) as tilted_irradiation_wh_m2, - round(avg(module_temperature_dc),2) as module_temperature_dc_mean, - sum(energy_output_kwh) as energy_output_kwh, - count(*) as hours_with_reading -from {{ref('satellite_readings__hourly_legacy')}} as srh + plant_id, + date_trunc('day', start_hour) as day, + sum(horizontal_irradiation_wh_m2) as horizontal_irradiation_wh_m2, + sum(tilted_irradiation_wh_m2) as tilted_irradiation_wh_m2, + round(avg(module_temperature_dc), 2) as module_temperature_dc_mean, + sum(energy_output_kwh) as energy_output_kwh, + count(*) as hours_with_reading +from {{ ref('satellite_readings__hourly_legacy') }} group by date_trunc('day', start_hour), plant_id - diff --git a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean.sql b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean.sql index 84a3b852..6aa1868a 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean.sql @@ -1,24 +1,28 @@ {{ config(materialized='view') }} -SELECT - ir.time, - plant.id as plant_id, - plant.name as plant_name, - plant.codename as plant_code, - ir.inverter_id, - inverter.name as inverter_name, - inverter.brand as inverter_brand, - inverter.model as inverter_model, - round(inverter.nominal_power_w/1000.0,2) as inverter_nominal_power_kw, - ir.power_kw, - ir.energy_kwh, - ir.intensity_cc_a, - ir.intensity_ca_a, - ir.voltage_cc_v, - ir.voltage_ca_v, - ir.uptime_h, - ir.temperature_c, - ir.readings -FROM {{ref('inverterregistry_gapfilled')}} as ir -left join {{source('plantmonitor_legacy','inverter')}} as inverter on inverter.id = ir.inverter_id -left join {{source('plantmonitor_legacy','plant')}} as plant on plant.id = inverter.plant \ No newline at end of file +select + ir.time, + plant.id as plant_id, + plant.name as plant_name, + plant.codename as plant_code, + ir.inverter_id, + inverter.name as inverter_name, + inverter.brand as inverter_brand, + inverter.model as inverter_model, + ir.power_kw, + ir.energy_kwh, + ir.intensity_cc_a, + ir.intensity_ca_a, + ir.voltage_cc_v, + ir.voltage_ca_v, + ir.uptime_h, + ir.temperature_c, + ir.readings, + round(inverter.nominal_power_w / 1000.0, 2) as inverter_nominal_power_kw +from {{ ref('inverterregistry_gapfilled') }} as ir + left join + {{ source('plantmonitor_legacy','inverter') }} as inverter + on ir.inverter_id = inverter.id + left join + {{ source('plantmonitor_legacy','plant') }} as plant + on inverter.plant = plant.id diff --git a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean_daily.sql b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean_daily.sql index 62bf55ea..4ec785ac 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean_daily.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_clean_daily.sql @@ -2,43 +2,43 @@ with inverterregistry_clean_nrows as ( -select + select *, 1 as nrows -from {{ref('inverterregistry_clean')}} as irc + from {{ ref('inverterregistry_clean') }} ) -SELECT - date_trunc('day',irc.time) as time, - irc.plant_id, - irc.plant_name, - irc.plant_code, - irc.inverter_id, - irc.inverter_name, - irc.inverter_brand, - irc.inverter_model, - irc.inverter_nominal_power_kw, - round(sum(irc.power_kw),2) as power_kw, - round(sum(irc.energy_kwh),2) as energy_kwh, - round(avg(irc.intensity_cc_a),2) as intensity_cc_a_day_avg, - round(avg(irc.intensity_ca_a),2) as intensity_ca_a_day_avg, - round(avg(irc.voltage_cc_v),2) as voltage_cc_v_day_avg, - round(avg(irc.voltage_ca_v),2) as voltage_ca_v_day_avg, - round(max(irc.uptime_h),2) as uptime_h, - round(avg(irc.temperature_c),2) as temperature_cd_day_avg, - round(min(irc.temperature_c),2) as temperature_c_day_min, - round(max(irc.temperature_c),2) as temperature_c_day_max, - round(sum(irc.readings)/sum(irc.nrows),2)as confidence_index -FROM inverterregistry_clean_nrows as irc +select + irc.plant_id, + irc.plant_name, + irc.plant_code, + irc.inverter_id, + irc.inverter_name, + irc.inverter_brand, + irc.inverter_model, + irc.inverter_nominal_power_kw, + date_trunc('day', irc.time) as time, --noqa: RF04 + round(sum(irc.power_kw), 2) as power_kw, + round(sum(irc.energy_kwh), 2) as energy_kwh, + round(avg(irc.intensity_cc_a), 2) as intensity_cc_a_day_avg, + round(avg(irc.intensity_ca_a), 2) as intensity_ca_a_day_avg, + round(avg(irc.voltage_cc_v), 2) as voltage_cc_v_day_avg, + round(avg(irc.voltage_ca_v), 2) as voltage_ca_v_day_avg, + round(max(irc.uptime_h), 2) as uptime_h, + round(avg(irc.temperature_c), 2) as temperature_cd_day_avg, + round(min(irc.temperature_c), 2) as temperature_c_day_min, + round(max(irc.temperature_c), 2) as temperature_c_day_max, + round(sum(irc.readings) / sum(irc.nrows), 2) as confidence_index +from inverterregistry_clean_nrows as irc group by - date_trunc('day',irc.time), - irc.plant_id, - irc.plant_name, - irc.plant_code, - irc.inverter_id, - irc.inverter_name, - irc.inverter_brand, - irc.inverter_model, - irc.inverter_nominal_power_kw + date_trunc('day', irc.time), + irc.plant_id, + irc.plant_name, + irc.plant_code, + irc.inverter_id, + irc.inverter_name, + irc.inverter_brand, + irc.inverter_model, + irc.inverter_nominal_power_kw diff --git a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_multisource.sql b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_multisource.sql index 9e4a1ab0..6b5b3597 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_multisource.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_multisource.sql @@ -2,35 +2,35 @@ with inverterregistry as ( - SELECT - time, - inverter_id, - power_kw, - energy_kwh, - intensity_cc_a, - intensity_ca_a, - voltage_cc_v, - voltage_ca_v, - uptime_h, - temperature_c, - readings - FROM {{ ref('inverterregistry_raw') }} + select + time, + inverter_id, + power_kw, + energy_kwh, + intensity_cc_a, + intensity_ca_a, + voltage_cc_v, + voltage_ca_v, + uptime_h, + temperature_c, + readings + from {{ ref('inverterregistry_raw') }} -UNION ALL + union all - SELECT - time, - inverter_id, - power_kw, - energy_kwh, - intensity_cc_a, - intensity_ca_a, - voltage_cc_v, - voltage_ca_v, - uptime_h, - temperature_c, - readings - FROM {{ ref('lake_modbusreadings_inverter_standard') }} + select + time, + inverter_id, + power_kw, + energy_kwh, + intensity_cc_a, + intensity_ca_a, + voltage_cc_v, + voltage_ca_v, + uptime_h, + temperature_c, + readings + from {{ ref('lake_modbusreadings_inverter_standard') }} ) diff --git a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_raw.sql b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_raw.sql index bec452e1..450d6c8a 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_raw.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/inverterregistry_raw.sql @@ -2,20 +2,20 @@ {# wrong unit temperature_dc is temperature_mc #} -SELECT - ir.time, - ir.inverter as inverter_id, - round(ir.power_w/1000.0,2) as power_kw, - round(ir.energy_wh/1000.0,2) as energy_kwh, - round(ir.intensity_cc_ma/1000.0,2) as intensity_cc_a, - round(ir.intensity_ca_ma/1000.0,2) as intensity_ca_a, - round(ir.voltage_cc_mv/1000.0,2) as voltage_cc_v, - round(ir.voltage_ca_mv/1000.0,2) as voltage_ca_v, - ir.uptime_h, - round(ir.temperature_dc/10.0,2) as temperature_c, - 1 as readings -FROM {{source('plantmonitor_legacy','inverterregistry')}} as ir +select + ir.time, + ir.inverter as inverter_id, + ir.uptime_h, + 1 as readings, + round(ir.power_w / 1000.0, 2) as power_kw, + round(ir.energy_wh / 1000.0, 2) as energy_kwh, + round(ir.intensity_cc_ma / 1000.0, 2) as intensity_cc_a, + round(ir.intensity_ca_ma / 1000.0, 2) as intensity_ca_a, + round(ir.voltage_cc_mv / 1000.0, 2) as voltage_cc_v, + round(ir.voltage_ca_mv / 1000.0, 2) as voltage_ca_v, + round(ir.temperature_dc / 10.0, 2) as temperature_c +from {{ source('plantmonitor_legacy','inverterregistry') }} as ir -{% if target.name == 'pre' %} +{% if target.name == 'pre' -%} where ir.time >= current_date - interval '3 days' -{% endif %} +{%- endif %} diff --git a/dbt_jardiner/models/legacy/operational/inverter/inverters_raw.sql b/dbt_jardiner/models/legacy/operational/inverter/inverters_raw.sql index a0b6b992..8d1a56fb 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/inverters_raw.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/inverters_raw.sql @@ -1,9 +1,9 @@ {{ config(materialized='view') }} -SELECT - id as inverter_id, - name as inverter_name, - plant as plant_id, - model as inverter_model, - round(nominal_power_w/1000.0, 2) as nominal_power_kw -FROM {{source('plantmonitor_legacy','inverter')}} +select + id as inverter_id, + name as inverter_name, + plant as plant_id, + model as inverter_model, + round(nominal_power_w / 1000.0, 2) as nominal_power_kw +from {{ source('plantmonitor_legacy','inverter') }} diff --git a/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter.sql b/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter.sql index 1efbee06..f072f1c3 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter.sql @@ -2,5 +2,5 @@ {# modbus_unit = 3 assumes asomada at the moment, which is the unit of the inverter #} -select * from {{ ref("lake_modbusreadings_selected_newest")}} +select * from {{ ref("lake_modbusreadings_selected_newest") }} where modbus_unit = 3 diff --git a/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_concat.sql b/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_concat.sql index ebcc4701..c090de4c 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_concat.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_concat.sql @@ -8,17 +8,17 @@ Assumes signed registries. We've seen that only uptime is unsigned, meaning that #} with inverter_modbus as ( - select - five_minute, - inverter_id, - {{concat_bytes('power_kw')}}, - {{concat_bytes('temperature_dc')}}, - {{concat_bytes('uptime_s')}}, - {{concat_bytes('energy_total_10kwh')}}, - {{concat_bytes('energy_10kwh_daily')}} - from {{ ref("lake_modbusreadings_inverter_pivoted")}} + select + five_minute, + inverter_id, + {{ concat_bytes('power_kw') }}, + {{ concat_bytes('temperature_dc') }}, + {{ concat_bytes('uptime_s') }}, + {{ concat_bytes('energy_total_10kwh') }}, + {{ concat_bytes('energy_10kwh_daily') }} + from {{ ref("lake_modbusreadings_inverter_pivoted") }} ) select - * + * from inverter_modbus diff --git a/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_standard.sql b/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_standard.sql index f8da806f..f63dc70a 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_standard.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/lake_modbusreadings_inverter_standard.sql @@ -8,21 +8,20 @@ Assumes signed registries. We've seen that only uptime is unsigned, meaning that #} with inverter_modbus as ( - select - five_minute as time, - inverter_id, - power_kw::numeric as power_kw, - (energy_10kwh_daily*10)::numeric as energy_kwh, - NULL::numeric as intensity_cc_a, - NULL::numeric as intensity_ca_a, - NULL::numeric as voltage_cc_v, - NULL::numeric as voltage_ca_v, - (uptime_s/3600.0)::numeric as uptime_h, - (0.1*temperature_dc)::numeric as temperature_c, - 1 as readings - from {{ ref("lake_modbusreadings_inverter_concat")}} + select + five_minute as time, --noqa: RF04 + inverter_id, + power_kw::numeric as power_kw, + (energy_10kwh_daily * 10)::numeric as energy_kwh, + null::numeric as intensity_cc_a, + null::numeric as intensity_ca_a, + null::numeric as voltage_cc_v, + null::numeric as voltage_ca_v, + (uptime_s / 3600.0)::numeric as uptime_h, + (0.1 * temperature_dc)::numeric as temperature_c, + 1 as readings + from {{ ref("lake_modbusreadings_inverter_concat") }} ) -select - * +select * from inverter_modbus diff --git a/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_gapfilled.sql b/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_gapfilled.sql index 78546c16..102899ae 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_gapfilled.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_gapfilled.sql @@ -1,18 +1,18 @@ {%- set start_date = '2020-01-01' -%} -SELECT - time_bucket_gapfill('5 minutes', ir.time) as time, - ir.inverter_id as inverter_id, - round(avg(ir.power_kw),2) as power_kw, - round(avg(ir.energy_kwh),2) as energy_kwh, - round(avg(ir.intensity_cc_a),2) as intensity_cc_a, - round(avg(ir.intensity_ca_a),2) as intensity_ca_a, - round(avg(ir.voltage_cc_v),2) as voltage_cc_v, - round(avg(ir.voltage_ca_v),2) as voltage_ca_v, - round(max(ir.uptime_h),2) as uptime_h, - round(min(ir.temperature_c),2) as temperature_c, - round(sum(readings),2) as readings -FROM {{ref('inverterregistry_multisource')}} as ir -WHERE time >= '{{ start_date }}' and time < NOW() - interval '5 minutes' -GROUP by ir.inverter_id, time_bucket_gapfill('5 minutes', ir.time) +select + ir.inverter_id as inverter_id, + time_bucket_gapfill('5 minutes', ir.time) as time, --noqa: RF04 + round(avg(ir.power_kw), 2) as power_kw, + round(avg(ir.energy_kwh), 2) as energy_kwh, + round(avg(ir.intensity_cc_a), 2) as intensity_cc_a, + round(avg(ir.intensity_ca_a), 2) as intensity_ca_a, + round(avg(ir.voltage_cc_v), 2) as voltage_cc_v, + round(avg(ir.voltage_ca_v), 2) as voltage_ca_v, + round(max(ir.uptime_h), 2) as uptime_h, + round(min(ir.temperature_c), 2) as temperature_c, + round(sum(readings), 2) as readings +from {{ ref('inverterregistry_multisource') }} as ir +where time >= '{{ start_date }}' and time < now() - interval '5 minutes' +group by ir.inverter_id, time_bucket_gapfill('5 minutes', ir.time) diff --git a/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_sample.sql b/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_sample.sql index 69aa85fa..5bb99d08 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_sample.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/stg/inverterregistry_sample.sql @@ -1,8 +1,8 @@ {{ config(materialized='view') }} -SELECT - * -FROM {{ref('inverterregistry_clean')}} as ir -where time between '2022-10-01 00:00' and '2022-10-02 00:00' -and inverter_id = 16 +select * +from {{ ref('inverterregistry_clean') }} +where + time between '2022-10-01 00:00' and '2022-10-02 00:00' + and inverter_id = 16 diff --git a/dbt_jardiner/models/legacy/operational/inverter/stg/lake_modbusreadings_inverter_pivoted.sql b/dbt_jardiner/models/legacy/operational/inverter/stg/lake_modbusreadings_inverter_pivoted.sql index b0f9cfe0..010f48ca 100644 --- a/dbt_jardiner/models/legacy/operational/inverter/stg/lake_modbusreadings_inverter_pivoted.sql +++ b/dbt_jardiner/models/legacy/operational/inverter/stg/lake_modbusreadings_inverter_pivoted.sql @@ -1,15 +1,15 @@ {{ config(materialized='view') }} with inverter_modbus_pivoted as ( - select - five_minute, - device_id as inverter_id, - {{ pivot(column='register_name', names=dbt_utils.get_column_values(table=ref('lake_modbusreadings_inverter'), column='register_name'), value_column='value', agg='max') }} - from {{ ref('lake_modbusreadings_inverter') }} - where five_minute is not NULL - group by five_minute, device_type, device_id + select + five_minute, + device_id as inverter_id, + {{ pivot(column='register_name', names=dbt_utils.get_column_values(table=ref('lake_modbusreadings_inverter'), column='register_name'), value_column='value', agg='max') }} + from {{ ref('lake_modbusreadings_inverter') }} + where five_minute is not null + group by five_minute, device_type, device_id ) select - * + * from inverter_modbus_pivoted diff --git a/dbt_jardiner/models/legacy/operational/meter/meter_noenergy_alarm.sql b/dbt_jardiner/models/legacy/operational/meter/meter_noenergy_alarm.sql index 78b9bf7c..2302ce61 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meter_noenergy_alarm.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meter_noenergy_alarm.sql @@ -1,3 +1,3 @@ {{ config(materialized='view') }} -select * from {{source('plantmonitor_legacy', 'alarm_status') }} +select * from {{ source('plantmonitor_legacy', 'alarm_status') }} diff --git a/dbt_jardiner/models/legacy/operational/meter/meter_registry.sql b/dbt_jardiner/models/legacy/operational/meter/meter_registry.sql index ae1ea696..91703247 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meter_registry.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meter_registry.sql @@ -1,17 +1,18 @@ {{ config(materialized='view') }} -SELECT - time - interval '1 hour' as time_start_hour, - export_energy_wh, - import_energy_wh, - meter as meter_id, +select + mr.export_energy_wh, + mr.import_energy_wh, + meter.id as meter_id, meter.name as meter_name, meter.connection_protocol, plant.id as plant_id, plant.name as plant_name, plant.codename as plant_code, - TRUE -FROM {{source('plantmonitor_legacy','meterregistry')}} as mr -left join {{source('plantmonitor_legacy','meter')}} on meter.id = mr.meter -left join {{source('plantmonitor_legacy','plant')}} as plant on plant.id = meter.plant + mr.time - interval '1 hour' as time_start_hour +from {{ source('plantmonitor_legacy','meterregistry') }} as mr + left join {{ source('plantmonitor_legacy','meter') }} as meter on meter.id = mr.meter + left join + {{ source('plantmonitor_legacy','plant') }} as plant + on plant.id = meter.plant diff --git a/dbt_jardiner/models/legacy/operational/meter/meter_registry_daily.sql b/dbt_jardiner/models/legacy/operational/meter/meter_registry_daily.sql index f3344cb7..7317ff7f 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meter_registry_daily.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meter_registry_daily.sql @@ -1,14 +1,20 @@ {{ config(materialized='view') }} select - date_trunc('day', time_start_hour) AS day, meter_id, meter_name, plant_id, plant_name, plant_code, + date_trunc('day', time_start_hour) as day, count(*) as hours_with_reading, sum(export_energy_wh) as export_energy_wh, sum(has_energy) as hours_with_energy -FROM {{ref('meter_registry_hourly')}} -group by date_trunc('day', time_start_hour), plant_id, plant_name, plant_code, meter_id, meter_name \ No newline at end of file +from {{ ref('meter_registry_hourly') }} +group by + date_trunc('day', time_start_hour), + plant_id, + plant_name, + plant_code, + meter_id, + meter_name diff --git a/dbt_jardiner/models/legacy/operational/meter/meter_registry_hourly.sql b/dbt_jardiner/models/legacy/operational/meter/meter_registry_hourly.sql index bc1692f2..73ca3ead 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meter_registry_hourly.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meter_registry_hourly.sql @@ -3,24 +3,29 @@ with meter_registry_hourly_raw as ( select - date_trunc('hour', time_start_hour) AS time_start_hour, meter_id, meter_name, plant_id, plant_name, plant_code, - round(avg(export_energy_wh),2) as export_energy_wh, - round(avg(import_energy_wh),2) as import_energy_wh - from {{ ref('meter_registry') }} as meter_registry + date_trunc('hour', time_start_hour) as time_start_hour, + round(avg(export_energy_wh), 2) as export_energy_wh, + round(avg(import_energy_wh), 2) as import_energy_wh + from {{ ref('meter_registry') }} --WHERE time >= '' and time < max(time) - group by date_trunc('hour', time_start_hour), plant_id, plant_name, plant_code, meter_id, meter_name + group by + date_trunc('hour', time_start_hour), + plant_id, + plant_name, + plant_code, + meter_id, + meter_name ) select *, - CASE - WHEN export_energy_wh > 0 THEN 1 - WHEN export_energy_wh = 0 THEN 0 - ELSE NULL - END as has_energy -FROM meter_registry_hourly_raw \ No newline at end of file + case + when export_energy_wh > 0 then 1 + when export_energy_wh = 0 then 0 + end as has_energy +from meter_registry_hourly_raw diff --git a/dbt_jardiner/models/legacy/operational/meter/meter_registry_raw.sql b/dbt_jardiner/models/legacy/operational/meter/meter_registry_raw.sql index a64057ee..5693024a 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meter_registry_raw.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meter_registry_raw.sql @@ -15,15 +15,17 @@ {% else %} - SELECT - time - interval '1 hour' as time, - export_energy_wh, - import_energy_wh, - meter as meter_id, + select + mr.export_energy_wh, + mr.import_energy_wh, + meter.id as meter_id, meter.name as meter_name, - meter.connection_protocol - FROM {{source('plantmonitor_legacy','meterregistry')}} as mr - left join {{source('plantmonitor_legacy','meter')}} on meter.id = mr.meter - left join {{source('plantmonitor_legacy','plant')}} as plant on plant.id = meter.plant + meter.connection_protocol, + mr.time - interval '1 hour' as time --noqa: RF04 + from {{ source('plantmonitor_legacy','meterregistry') }} as mr + left join {{ source('plantmonitor_legacy','meter') }} as meter on meter.id = mr.meter + left join + {{ source('plantmonitor_legacy','plant') }} as plant + on plant.id = meter.plant -{% endif %} \ No newline at end of file +{% endif %} diff --git a/dbt_jardiner/models/legacy/operational/meter/meters_raw.sql b/dbt_jardiner/models/legacy/operational/meter/meters_raw.sql index dfa9bcd5..450d1218 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meters_raw.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meters_raw.sql @@ -1,8 +1,8 @@ {{ config(materialized='view') }} -SELECT - id as meter_id, - plant as plant_id, - name as meter_name, - connection_protocol as meter_connection_protocol -FROM {{source('plantmonitor_legacy','meter')}} +select + id as meter_id, + plant as plant_id, + name as meter_name, + connection_protocol as meter_connection_protocol +from {{ source('plantmonitor_legacy','meter') }} diff --git a/dbt_jardiner/models/legacy/operational/meter/meters_with_thresholds.sql b/dbt_jardiner/models/legacy/operational/meter/meters_with_thresholds.sql index 9d9e003f..008f74a5 100644 --- a/dbt_jardiner/models/legacy/operational/meter/meters_with_thresholds.sql +++ b/dbt_jardiner/models/legacy/operational/meter/meters_with_thresholds.sql @@ -1,8 +1,12 @@ {{ config(materialized='view') }} select - meter.*, - coalesce(mth.num_hours_threshold, 1) as num_hours_threshold + meter.*, + coalesce(mth.num_hours_threshold, 1) as num_hours_threshold from {{ ref('meters_raw') }} as meter -left join {{ ref('som_plants_raw') }} as plant using(plant_id) -left join {{ ref('alarm_meter_no_energy_plant_thresholds')}} as mth on mth.plant_name = plant.plant_name + left join + {{ ref('som_plants_raw') }} as plant + on meter.plant_id = plant.plant_id + left join + {{ ref('alarm_meter_no_energy_plant_thresholds') }} as mth + on plant.plant_name = mth.plant_name diff --git a/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_raw.sql b/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_raw.sql index 13409f79..c836da5c 100644 --- a/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_raw.sql +++ b/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_raw.sql @@ -1,13 +1,12 @@ {{ config(materialized='view') }} -SELECT - ip as plant_ip, - port::integer as modbus_port, - unit::integer as modbus_unit, - register_address::integer as modbus_register_address, - value::integer as value, - is_valid, - query_time, - create_date -FROM {{ source('plantlake', 'modbus_readings') }} - +select + ip as plant_ip, + port::integer as modbus_port, + unit::integer as modbus_unit, + register_address::integer as modbus_register_address, + value::integer as value, --noqa: RF04 + is_valid, + query_time, + create_date +from {{ source('plantlake', 'modbus_readings') }} diff --git a/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_selected_newest.sql b/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_selected_newest.sql index 1aedb1e5..9bc34942 100644 --- a/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_selected_newest.sql +++ b/dbt_jardiner/models/legacy/operational/modbus/lake_modbusreadings_selected_newest.sql @@ -1,21 +1,31 @@ {{ config(materialized='view') }} select - plant_ip, - modbus_port, - modbus_unit, - modbus_register_address, - value, - query_time, - create_date, - device_type, - device_id, - register_name, - time_bucket('5 minutes', query_time) as five_minute + plant_ip, + modbus_port, + modbus_unit, + modbus_register_address, + value, + query_time, + create_date, + device_type, + device_id, + register_name, + time_bucket('5 minutes', query_time) as five_minute from ( - select + select *, - row_number() over (partition by plant_ip, modbus_port, modbus_unit, modbus_register_address, time_bucket('5 minutes', query_time) order by create_date desc) as row_number - from {{ ref('lake_modbusreadings_selected_raw') }} -) lake_modbusreadings_newest -where row_number=1 \ No newline at end of file + row_number() + over ( + partition by + plant_ip, + modbus_port, + modbus_unit, + modbus_register_address, + time_bucket('5 minutes', query_time) + order by create_date desc + ) + as row_number + from {{ ref('lake_modbusreadings_selected_raw') }} +) as lake_modbusreadings_newest +where row_number = 1 diff --git a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_invalid.sql b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_invalid.sql index d7ded2ec..d1e1fdc0 100644 --- a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_invalid.sql +++ b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_invalid.sql @@ -1,6 +1,5 @@ {{ config(materialized='view') }} -SELECT - * -FROM {{ ref('lake_modbusreadings_raw')}} +select * +from {{ ref('lake_modbusreadings_raw') }} where is_valid is not true diff --git a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_selected_raw.sql b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_selected_raw.sql index 4e4f5ea5..ce36156c 100644 --- a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_selected_raw.sql +++ b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_selected_raw.sql @@ -1,26 +1,26 @@ {{ config(materialized='view') }} with inverter_modbus_readings as ( - SELECT - * - FROM {{ ref('lake_modbusreadings_valid') }} + select * + from {{ ref('lake_modbusreadings_valid') }} ) select - plant_ip, - modbus_port, - modbus_unit, - modbus_register_address, - value, - query_time, - create_date, - device_type, - device_id, - register_name + meta.ip as plant_ip, + meta.port as modbus_port, + meta.unit as modbus_unit, + meta.register_address as modbus_register_address, + inverter_modbus_readings.value, + inverter_modbus_readings.query_time, + inverter_modbus_readings.create_date, + meta.device_type, + meta.device_id, + meta.register_name from inverter_modbus_readings -inner join {{ ref('modbus_registries_selected_meta')}} as meta -on - meta.ip = inverter_modbus_readings.plant_ip -and meta.port = inverter_modbus_readings.modbus_port -and meta.unit = inverter_modbus_readings.modbus_unit -and meta.register_address = inverter_modbus_readings.modbus_register_address + inner join {{ ref('modbus_registries_selected_meta') }} as meta + on + inverter_modbus_readings.plant_ip = meta.ip + and inverter_modbus_readings.modbus_port = meta.port + and inverter_modbus_readings.modbus_unit = meta.unit + and inverter_modbus_readings.modbus_register_address + = meta.register_address diff --git a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_stringbox_raw.sql b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_stringbox_raw.sql index d5ced6b9..35bb108d 100644 --- a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_stringbox_raw.sql +++ b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_stringbox_raw.sql @@ -1,8 +1,7 @@ {{ config(materialized='view') }} -SELECT - * -FROM {{ ref('lake_modbusreadings_raw') }} -where plant_ip ilike 'planta-asomada.somenergia.coop' -and (modbus_register_address = 33 or modbus_register_address = 32) - +select * +from {{ ref('lake_modbusreadings_raw') }} +where + plant_ip ilike 'planta-asomada.somenergia.coop' + and (modbus_register_address = 33 or modbus_register_address = 32) diff --git a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_valid.sql b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_valid.sql index ffab43da..4465b3fb 100644 --- a/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_valid.sql +++ b/dbt_jardiner/models/legacy/operational/modbus/stg/lake_modbusreadings_valid.sql @@ -1,7 +1,6 @@ {{ config(materialized='view') }} -SELECT - * -FROM {{ ref('lake_modbusreadings_raw')}} -where is_valid is true \ No newline at end of file +select * +from {{ ref('lake_modbusreadings_raw') }} +where is_valid is true diff --git a/dbt_jardiner/models/legacy/operational/plant/gestio_actius_plant_parameters__raw_legacy.sql b/dbt_jardiner/models/legacy/operational/plant/gestio_actius_plant_parameters__raw_legacy.sql index 48b69c72..554c3c4f 100644 --- a/dbt_jardiner/models/legacy/operational/plant/gestio_actius_plant_parameters__raw_legacy.sql +++ b/dbt_jardiner/models/legacy/operational/plant/gestio_actius_plant_parameters__raw_legacy.sql @@ -1,38 +1,39 @@ {{ config(materialized="view") }} with - current_data as ( - select - plant_id::numeric, - plant_uuid::uuid, - case - when planta = 'Vallehermoso' then 'Alcolea' - when planta = 'Tahal' then 'Terborg' - else planta - end as plant_name, - latitud::numeric as latitude, - longitut::numeric as longitude, - municipi as municipality, - provincia as province, - tecnologia as technology, - subtecnologia as subtechnology, - data_connexio::date as connection_date, - potencia_nominal_kw::numeric as nominal_power_kw, - potencia_pic_kw::numeric as peak_power_kw, - spv as "owner", - empresa_manteniment as maintenance_company, - responsable_ga as manager, - n_strings_plant::numeric, - n_modules_total::numeric, - n_inverters::numeric, - esquema_unifilar, - layout, - data_actualitzacio::date as gestio_actius_updated_at, - dbt_updated_at::date as dbt_updated_at, - dbt_valid_from::date as dbt_valid_from, - coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to - from {{ ref("snapshot_plant_parameters") }} as pl - ) +current_data as ( + select + plant_id::numeric, + plant_uuid::uuid, + latitud::numeric as latitude, + longitut::numeric as longitude, + municipi as municipality, + provincia as province, + tecnologia as technology, + subtecnologia as subtechnology, + data_connexio::date as connection_date, + potencia_nominal_kw::numeric as nominal_power_kw, + potencia_pic_kw::numeric as peak_power_kw, + spv as "owner", + empresa_manteniment as maintenance_company, + responsable_ga as manager, + n_strings_plant::numeric, + n_modules_total::numeric, + n_inverters::numeric, + esquema_unifilar, + layout, + data_actualitzacio::date as gestio_actius_updated_at, + dbt_updated_at::date as dbt_updated_at, + dbt_valid_from::date as dbt_valid_from, + coalesce(dbt_valid_to::date, '2050-01-01'::date)::date as dbt_valid_to, + case + when planta = 'Vallehermoso' then 'Alcolea' + when planta = 'Tahal' then 'Terborg' + else planta + end as plant_name + from {{ ref("snapshot_plant_parameters") }} +) + select * from current_data where dbt_valid_from::date <= current_date and current_date < dbt_valid_to::date diff --git a/dbt_jardiner/models/legacy/operational/plant/plantparameters_raw.sql b/dbt_jardiner/models/legacy/operational/plant/plantparameters_raw.sql index 94ef0c6a..31affbbb 100644 --- a/dbt_jardiner/models/legacy/operational/plant/plantparameters_raw.sql +++ b/dbt_jardiner/models/legacy/operational/plant/plantparameters_raw.sql @@ -1,16 +1,16 @@ select - id as plantparameters_id, - plant as plant_id, - peak_power_w/1000 as peak_power_kw, - nominal_power_w/1000 as nominal_power_kw, - connection_date, - n_strings_plant, - n_strings_inverter, - n_modules_string, - inverter_loss_mpercent::float/1000000.0 as inverter_loss_ratio, - meter_loss_mpercent::float/1000000.0 as meter_loss_ratio, - target_monthly_energy_wh/1000000 as target_monthly_energy_mwh, - historic_monthly_energy_wh/1000000 as historic_monthly_energy_mwh, - month_theoric_pr_cpercent, - year_theoric_pr_cpercent -from {{ source('plantmonitor_legacy', 'plantparameters')}} pp \ No newline at end of file + id as plantparameters_id, + plant as plant_id, + connection_date, + n_strings_plant, + n_strings_inverter, + n_modules_string, + month_theoric_pr_cpercent, + year_theoric_pr_cpercent, + peak_power_w / 1000 as peak_power_kw, + nominal_power_w / 1000 as nominal_power_kw, + inverter_loss_mpercent::float / 1000000.0 as inverter_loss_ratio, + meter_loss_mpercent::float / 1000000.0 as meter_loss_ratio, + target_monthly_energy_wh / 1000000 as target_monthly_energy_mwh, + historic_monthly_energy_wh / 1000000 as historic_monthly_energy_mwh +from {{ source('plantmonitor_legacy', 'plantparameters') }} diff --git a/dbt_jardiner/models/legacy/operational/plant/som_plants_extended.sql b/dbt_jardiner/models/legacy/operational/plant/som_plants_extended.sql index 1e76e720..1318d2e6 100644 --- a/dbt_jardiner/models/legacy/operational/plant/som_plants_extended.sql +++ b/dbt_jardiner/models/legacy/operational/plant/som_plants_extended.sql @@ -1,11 +1,10 @@ {{ config(materialized='view') }} -SELECT - plant.*, - pp.peak_power_kw, - pp.nominal_power_kw, - pp.connection_date, - pp.target_monthly_energy_mwh -FROM {{ ref('som_plants_raw') }} as plant -left join {{ ref('plantparameters_raw')}} pp using(plant_id) - +select + plant.*, + pp.peak_power_kw, + pp.nominal_power_kw, + pp.connection_date, + pp.target_monthly_energy_mwh +from {{ ref('som_plants_raw') }} as plant + left join {{ ref('plantparameters_raw') }} as pp on plant.plant_id = pp.plant_id diff --git a/dbt_jardiner/models/legacy/operational/plant/som_plants_raw.sql b/dbt_jardiner/models/legacy/operational/plant/som_plants_raw.sql index e178769e..a8c435af 100644 --- a/dbt_jardiner/models/legacy/operational/plant/som_plants_raw.sql +++ b/dbt_jardiner/models/legacy/operational/plant/som_plants_raw.sql @@ -1,8 +1,8 @@ {{ config(materialized='view') }} -SELECT - plant.id as plant_id, - plant.name as plant_name, - plant.codename as plant_codename -FROM {{source('plantmonitor_legacy','plant')}} -where description != 'SomRenovables' +select + plant.id as plant_id, + plant.name as plant_name, + plant.codename as plant_codename +from {{ source('plantmonitor_legacy','plant') }} as plant +where plant.description != 'SomRenovables' diff --git a/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__denormalized_legacy.sql b/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__denormalized_legacy.sql index 9fb4f2f0..2e6bf246 100644 --- a/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__denormalized_legacy.sql +++ b/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__denormalized_legacy.sql @@ -1,16 +1,17 @@ {{ config(materialized='view') }} select - sg.start_hour, - sg.plant_id, - plant_name, - request_time, - horizontal_irradiation_wh_m2, - tilted_irradiation_wh_m2, - module_temperature_dc, - energy_output_kwh -from {{ ref("solargis_satellite_readings__temp_and_pv_energy_legacy") }} sg -left join {{ref('gestio_actius_plant_parameters__raw_legacy')}} p on p.plant_id = sg.plant_id + sg.start_hour, + plant_id, + p.plant_name, + sg.request_time, + sg.horizontal_irradiation_wh_m2, + sg.tilted_irradiation_wh_m2, + sg.module_temperature_dc, + sg.energy_output_kwh +from {{ ref("solargis_satellite_readings__temp_and_pv_energy_legacy") }} as sg + left join + {{ ref('gestio_actius_plant_parameters__raw_legacy') }} as p using (plant_id) -- SolarGis PVOUT (aquí photovoltaic_energy_output_wh) retorna l'energia en kwh però plantmonitor per error ho registra com a wh sense fer cap transformació. -- Entenem que al redash s'està corregint a mà abans de mostrar el valor. diff --git a/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__hourly_legacy.sql b/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__hourly_legacy.sql index a8be57bd..d6d04b22 100644 --- a/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__hourly_legacy.sql +++ b/dbt_jardiner/models/legacy/operational/solargis/satellite_readings__hourly_legacy.sql @@ -1,24 +1,15 @@ -{{ config( - materialized = 'view' -) }} +{{ config(materialized = 'view') }} -WITH satellite AS ( - - SELECT - *, - ROW_NUMBER() OVER ( - PARTITION BY plant_id, - start_hour - ORDER BY - request_time DESC - ) AS ranking - FROM - {{ ref('satellite_readings__denormalized_legacy') }} +with satellite as ( + select + *, + row_number() over ( + partition by plant_id, start_hour + order by request_time desc + ) as ranking + from + {{ ref('satellite_readings__denormalized_legacy') }} ) -SELECT - * -FROM - satellite -WHERE - ranking = 1 +select * from satellite +where ranking = 1 diff --git a/dbt_jardiner/models/legacy/operational/solargis/solargis_satellite_readings__temp_and_pv_energy_legacy.sql b/dbt_jardiner/models/legacy/operational/solargis/solargis_satellite_readings__temp_and_pv_energy_legacy.sql index bd574125..e8423f7b 100644 --- a/dbt_jardiner/models/legacy/operational/solargis/solargis_satellite_readings__temp_and_pv_energy_legacy.sql +++ b/dbt_jardiner/models/legacy/operational/solargis/solargis_satellite_readings__temp_and_pv_energy_legacy.sql @@ -1,15 +1,15 @@ {{ config(materialized='view') }} select - date_trunc('hour', "time") as start_hour, - plant as plant_id, - request_time as request_time, - global_horizontal_irradiation_wh_m2 as horizontal_irradiation_wh_m2, - global_tilted_irradiation_wh_m2 as tilted_irradiation_wh_m2, - module_temperature_dc as module_temperature_dc, - photovoltaic_energy_output_wh as energy_output_kwh -from {{ source('solargis','satellite_readings') }} sg + plant as plant_id, + request_time as request_time, + global_horizontal_irradiation_wh_m2 as horizontal_irradiation_wh_m2, + global_tilted_irradiation_wh_m2 as tilted_irradiation_wh_m2, + module_temperature_dc as module_temperature_dc, + photovoltaic_energy_output_wh as energy_output_kwh, + date_trunc('hour', "time") as start_hour +from {{ source('solargis','satellite_readings') }} -- SolarGis PVOUT (aquí photovoltaic_energy_output_wh) retorna l'energia en kwh però plantmonitor per error ho registra com a wh sense fer cap transformació. -- Entenem que al redash s'està corregint a mà abans de mostrar el valor. --- Aquí canviem el nom perquè s'ajusti a la realitat del valor. \ No newline at end of file +-- Aquí canviem el nom perquè s'ajusti a la realitat del valor. diff --git a/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes.sql b/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes.sql index fccae0b3..573ca9cb 100644 --- a/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes.sql +++ b/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes.sql @@ -5,32 +5,36 @@ {# check that signed 16-bit is equivalent to this in complement two (and wether or not it's what it's using) #} with stringboxes as ( - select - query_time, - device_id as string_id, - register_name as string_name, - 100*(mr.value-2^16*(mr.value > 2^15)::int) as intensity_ma - from {{ ref("lake_modbusreadings_selected_newest")}} as mr - where modbus_unit = 32 or modbus_unit = 33 + select + mr.query_time, + mr.device_id as string_id, + mr.register_name as string_name, + 100 * (mr.value - 2 ^ 16 * (mr.value > 2 ^ 15)::int) as intensity_ma + from {{ ref("lake_modbusreadings_selected_newest") }} as mr + where mr.modbus_unit = 32 or mr.modbus_unit = 33 ), + stringboxes_rounded as ( - select - time_bucket('5 minutes', query_time) as five_minute, - string_id, - string_name, - intensity_ma - from stringboxes + select + string_id, + string_name, + intensity_ma, + time_bucket('5 minutes', query_time) as five_minute + from stringboxes ), + stringboxes_joined as ( - select - str_round.five_minute, - str_round.string_id as string_id, - str.string_name, - str.string_id as transactional_string_id, - str_round.intensity_ma - from stringboxes_rounded as str_round - left join {{ ref("strings_raw") }} as str on str_round.string_id = str.string_id + select + str_round.five_minute, + str_round.string_id as string_id, + str.string_name, + str.string_id as transactional_string_id, + str_round.intensity_ma + from stringboxes_rounded as str_round + left join + {{ ref("strings_raw") }} as str + on str_round.string_id = str.string_id ) -select * from stringboxes_joined \ No newline at end of file +select * from stringboxes_joined diff --git a/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_invalid.sql b/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_invalid.sql index c705dcf5..da92d53b 100644 --- a/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_invalid.sql +++ b/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_invalid.sql @@ -3,8 +3,9 @@ {# modbus_unit = 32 and 33 is specific of asomada. This is a model for adomada standarization #} with stringboxes as ( - select transactional_string_id from {{ ref('lake_modbusreadings_stringboxes')}} + select transactional_string_id + from {{ ref('lake_modbusreadings_stringboxes') }} ) select * from stringboxes -where transactional_string_id is NULL \ No newline at end of file +where transactional_string_id is null diff --git a/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_standard.sql b/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_standard.sql index 536e4061..55d76e3a 100644 --- a/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_standard.sql +++ b/dbt_jardiner/models/legacy/operational/string/stg/lake_modbusreadings_stringboxes_standard.sql @@ -1,7 +1,11 @@ {{ config(materialized='view') }} with stringboxes as ( - select five_minute as time, string_id, intensity_ma from {{ ref('lake_modbusreadings_stringboxes')}} + select + five_minute as time, --noqa: RF04 + string_id, + intensity_ma + from {{ ref('lake_modbusreadings_stringboxes') }} ) -select * from stringboxes \ No newline at end of file +select * from stringboxes diff --git a/dbt_jardiner/models/legacy/operational/string/stringregistry_denormalized.sql b/dbt_jardiner/models/legacy/operational/string/stringregistry_denormalized.sql index ef5d66b1..41a78bfd 100644 --- a/dbt_jardiner/models/legacy/operational/string/stringregistry_denormalized.sql +++ b/dbt_jardiner/models/legacy/operational/string/stringregistry_denormalized.sql @@ -3,14 +3,16 @@ -- TODO warning this is latest readings denormalized. select - sr.time as time, - inverter.plant_id, - inverter.inverter_id, - inverter.inverter_name, - sr.string_id, - string.string_name, - coalesce(string.stringbox_name, string.string_name) as stringdevice_name, - sr.intensity_ma -FROM {{ ref('stringregistry_multisource') }} AS sr -LEFT JOIN {{ ref('strings_raw') }} as string on sr.string_id = string.string_id -LEFT JOIN {{ ref('inverters_raw') }} as inverter on inverter.inverter_id = string.inverter_id + sr.time as time, -- noqa: RF04 + inverter.plant_id, + inverter.inverter_id, + inverter.inverter_name, + sr.string_id, + string.string_name, + sr.intensity_ma, + coalesce(string.stringbox_name, string.string_name) as stringdevice_name +from {{ ref('stringregistry_multisource') }} as sr + left join {{ ref('strings_raw') }} as string on sr.string_id = string.string_id + left join + {{ ref('inverters_raw') }} as inverter + on string.inverter_id = inverter.inverter_id diff --git a/dbt_jardiner/models/legacy/operational/string/stringregistry_latest_hour.sql b/dbt_jardiner/models/legacy/operational/string/stringregistry_latest_hour.sql index 9a999f45..1d0ef391 100644 --- a/dbt_jardiner/models/legacy/operational/string/stringregistry_latest_hour.sql +++ b/dbt_jardiner/models/legacy/operational/string/stringregistry_latest_hour.sql @@ -1,4 +1,4 @@ {{ config(materialized='view') }} select * from {{ ref('stringregistry_denormalized') }} - where now() - interval '1 hour' < time +where now() - interval '1 hour' < time diff --git a/dbt_jardiner/models/legacy/operational/string/stringregistry_multisource.sql b/dbt_jardiner/models/legacy/operational/string/stringregistry_multisource.sql index de12b987..8fe92239 100644 --- a/dbt_jardiner/models/legacy/operational/string/stringregistry_multisource.sql +++ b/dbt_jardiner/models/legacy/operational/string/stringregistry_multisource.sql @@ -2,19 +2,19 @@ with stringregistry as ( - SELECT - time, - string_id, - intensity_ma - FROM {{ ref('stringregistry_raw') }} + select + time, + string_id, + intensity_ma + from {{ ref('stringregistry_raw') }} -UNION ALL + union all - SELECT - time, - string_id, - intensity_ma - FROM {{ ref('lake_modbusreadings_stringboxes_standard') }} + select + time, + string_id, + intensity_ma + from {{ ref('lake_modbusreadings_stringboxes_standard') }} ) diff --git a/dbt_jardiner/models/legacy/operational/string/stringregistry_raw.sql b/dbt_jardiner/models/legacy/operational/string/stringregistry_raw.sql index 096fd134..accdd65f 100644 --- a/dbt_jardiner/models/legacy/operational/string/stringregistry_raw.sql +++ b/dbt_jardiner/models/legacy/operational/string/stringregistry_raw.sql @@ -2,11 +2,11 @@ {# wrong unit temperature_dc is temperature_mc #} -SELECT - sr.time, - sr.string as string_id, - sr.intensity_ma -FROM {{ source('plantmonitor_legacy','stringregistry') }} as sr +select + sr.time, + sr.string as string_id, + sr.intensity_ma +from {{ source('plantmonitor_legacy','stringregistry') }} as sr {% if target.name == 'pre' %} where sr.time >= current_date - interval '3 days' diff --git a/dbt_jardiner/models/legacy/operational/string/strings_raw.sql b/dbt_jardiner/models/legacy/operational/string/strings_raw.sql index c3a2ab0c..bf846dc3 100644 --- a/dbt_jardiner/models/legacy/operational/string/strings_raw.sql +++ b/dbt_jardiner/models/legacy/operational/string/strings_raw.sql @@ -1,6 +1,6 @@ select - id as string_id, - name as string_name, - inverter as inverter_id, - stringbox_name -from {{ source('plantmonitor_legacy','string') }} \ No newline at end of file + id as string_id, + name as string_name, + inverter as inverter_id, + stringbox_name +from {{ source('plantmonitor_legacy','string') }} diff --git a/dbt_jardiner/models/legacy/operational/tecnics/plants_energes.sql b/dbt_jardiner/models/legacy/operational/tecnics/plants_energes.sql index 1c4830a7..929ff01f 100644 --- a/dbt_jardiner/models/legacy/operational/tecnics/plants_energes.sql +++ b/dbt_jardiner/models/legacy/operational/tecnics/plants_energes.sql @@ -1,9 +1,10 @@ {{ config(post_hook="grant select on {{ this }} to group energes") }} -SELECT plant.id, - plant.name, - plant.codename, - plant.municipality, - plant.description -FROM plant -WHERE plant.name in ('Florida', 'Alcolea', 'Matallana') \ No newline at end of file +select + plant.id, + plant.name, + plant.codename, + plant.municipality, + plant.description +from plant +where plant.name in ('Florida', 'Alcolea', 'Matallana') diff --git a/dbt_jardiner/models/legacy/operational/tecnics/plants_energetica.sql b/dbt_jardiner/models/legacy/operational/tecnics/plants_energetica.sql index a24431ce..bdb662c6 100644 --- a/dbt_jardiner/models/legacy/operational/tecnics/plants_energetica.sql +++ b/dbt_jardiner/models/legacy/operational/tecnics/plants_energetica.sql @@ -1,9 +1,10 @@ {{ config(post_hook="grant select on {{ this }} to group energetica") }} -SELECT plant.id, - plant.name, - plant.codename, - plant.municipality, - plant.description -FROM plant -WHERE plant.name in ('Valteina') \ No newline at end of file +select + plant.id, + plant.name, + plant.codename, + plant.municipality, + plant.description +from plant +where plant.name in ('Valteina') diff --git a/dbt_jardiner/models/legacy/operational/tecnics/plants_ercam.sql b/dbt_jardiner/models/legacy/operational/tecnics/plants_ercam.sql index 1dc9ce3a..203ef447 100644 --- a/dbt_jardiner/models/legacy/operational/tecnics/plants_ercam.sql +++ b/dbt_jardiner/models/legacy/operational/tecnics/plants_ercam.sql @@ -1,9 +1,10 @@ {{ config(post_hook="grant select on {{ this }} to group ercam") }} -SELECT plant.id, - plant.name, - plant.codename, - plant.municipality, - plant.description -FROM plant -WHERE plant.name in ('Fontivsolar') \ No newline at end of file +select + plant.id, + plant.name, + plant.codename, + plant.municipality, + plant.description +from plant +where plant.name in ('Fontivsolar') diff --git a/dbt_jardiner/models/legacy/operational/tecnics/plants_exiom.sql b/dbt_jardiner/models/legacy/operational/tecnics/plants_exiom.sql index c34ba7a6..5fcb487e 100644 --- a/dbt_jardiner/models/legacy/operational/tecnics/plants_exiom.sql +++ b/dbt_jardiner/models/legacy/operational/tecnics/plants_exiom.sql @@ -1,9 +1,10 @@ {{ config(post_hook="grant select on {{ this }} to group exiom") }} -SELECT plant.id, - plant.name, - plant.codename, - plant.municipality, - plant.description -FROM plant -WHERE plant.name in ('Llanillos', 'Asomada') \ No newline at end of file +select + plant.id, + plant.name, + plant.codename, + plant.municipality, + plant.description +from plant +where plant.name in ('Llanillos', 'Asomada') diff --git a/dbt_jardiner/models/legacy/operational/view_target_energy.sql b/dbt_jardiner/models/legacy/operational/view_target_energy.sql index 92b90a0b..120670fb 100644 --- a/dbt_jardiner/models/legacy/operational/view_target_energy.sql +++ b/dbt_jardiner/models/legacy/operational/view_target_energy.sql @@ -1,17 +1,17 @@ -{{config(materialized='view')}} +{{ config(materialized='view') }} -- Alcolea. Objectiu mensual d'energia produïda (independentment de l'any) -SELECT * FROM (values -('2020-01-01'::timestamptz,221730), -('2020-02-01',228929), -('2020-03-01',320482), -('2020-04-01',300308), -('2020-05-01',342791), -('2020-06-01',333266), -('2020-07-01',361993), -('2020-08-01',344381), -('2020-09-01',318789), -('2020-10-01',279877), -('2020-11-01',200209), -('2020-12-01',199275) -) -AS t(time,target_energy_kwh) \ No newline at end of file +select energy_target.* from ( + values + ('2020-01-01'::timestamptz, 221730), + ('2020-02-01', 228929), + ('2020-03-01', 320482), + ('2020-04-01', 300308), + ('2020-05-01', 342791), + ('2020-06-01', 333266), + ('2020-07-01', 361993), + ('2020-08-01', 344381), + ('2020-09-01', 318789), + ('2020-10-01', 279877), + ('2020-11-01', 200209), + ('2020-12-01', 199275) +) as energy_target (month, energy) diff --git a/dbt_jardiner/models/legacy/utils/som_big_plants.sql b/dbt_jardiner/models/legacy/utils/som_big_plants.sql index 81b7633a..fab6bf47 100644 --- a/dbt_jardiner/models/legacy/utils/som_big_plants.sql +++ b/dbt_jardiner/models/legacy/utils/som_big_plants.sql @@ -1,5 +1,9 @@ {{ config(materialized='view') }} -select plant.id as plant_id, plant.name as plant_name, pp.* from plant -left join plantparameters as pp on plant.id = pp.plant -where pp.peak_power_w > 300000 \ No newline at end of file +select + pp.*, + plant.id as plant_id, + plant.name as plant_name +from plant + left join {{ source('plantmonitor_legacy', 'plantparameters') }} as pp on plant.id = pp.plant +where pp.peak_power_w > 300000 diff --git a/dbt_jardiner/models/legacy/utils/som_plants.sql b/dbt_jardiner/models/legacy/utils/som_plants.sql index 5b4f9cc8..3e2e1c1e 100644 --- a/dbt_jardiner/models/legacy/utils/som_plants.sql +++ b/dbt_jardiner/models/legacy/utils/som_plants.sql @@ -1,11 +1,12 @@ {{ config(materialized='view') }} -SELECT - plant.id as plant_id, - plant.name as plant_name, - plant.codename as plant_codename, - plantparameters.peak_power_w as peak_power_w -FROM {{source('plantmonitor_legacy','plant')}} as plant -LEFT JOIN {{source ('plantmonitor_legacy', 'plantparameters')}} as plantparameters +select + plant.id as plant_id, + plant.name as plant_name, + plant.codename as plant_codename, + plantparameters.peak_power_w as peak_power_w +from {{ source('plantmonitor_legacy','plant') }} as plant + left join + {{ source ('plantmonitor_legacy', 'plantparameters') }} as plantparameters on plant.id = plantparameters.plant -where description != 'SomRenovables' +where plant.description != 'SomRenovables' diff --git a/dbt_jardiner/models/legacy/utils/spine_days.sql b/dbt_jardiner/models/legacy/utils/spine_days.sql index 6e74bf05..9584de04 100644 --- a/dbt_jardiner/models/legacy/utils/spine_days.sql +++ b/dbt_jardiner/models/legacy/utils/spine_days.sql @@ -1,4 +1,3 @@ - {{ config(materialized = 'table') }} @@ -6,4 +5,4 @@ datepart="day", start_date="to_date('01/01/2015', 'mm/dd/yyyy')", end_date="to_date('01/01/2030', 'mm/dd/yyyy')" -) }} \ No newline at end of file +) }} diff --git a/dbt_jardiner/models/legacy/utils/spine_days_localized.sql b/dbt_jardiner/models/legacy/utils/spine_days_localized.sql index d2a897ba..2f8e72bb 100644 --- a/dbt_jardiner/models/legacy/utils/spine_days_localized.sql +++ b/dbt_jardiner/models/legacy/utils/spine_days_localized.sql @@ -1,5 +1,4 @@ - {{ config(materialized = 'table') }} select date_day at time zone 'Europe/Madrid' as day -from {{ ref('spine_days')}} \ No newline at end of file +from {{ ref('spine_days') }} diff --git a/dbt_jardiner/models/legacy/utils/spine_hours.sql b/dbt_jardiner/models/legacy/utils/spine_hours.sql index c9b55927..a0391aeb 100644 --- a/dbt_jardiner/models/legacy/utils/spine_hours.sql +++ b/dbt_jardiner/models/legacy/utils/spine_hours.sql @@ -1,4 +1,3 @@ - {{ config(materialized = 'table') }} @@ -6,4 +5,4 @@ datepart="hour", start_date="to_date('01/01/2015', 'mm/dd/yyyy')", end_date="to_date('01/01/2030', 'mm/dd/yyyy')" -) }} \ No newline at end of file +) }} diff --git a/dbt_jardiner/models/legacy/utils/spine_hours_localized.sql b/dbt_jardiner/models/legacy/utils/spine_hours_localized.sql index bb8fd00f..d6094346 100644 --- a/dbt_jardiner/models/legacy/utils/spine_hours_localized.sql +++ b/dbt_jardiner/models/legacy/utils/spine_hours_localized.sql @@ -1,5 +1,4 @@ - {{ config(materialized = 'table') }} select date_hour at time zone 'Europe/Madrid' as start_hour -from {{ ref('spine_hours')}} \ No newline at end of file +from {{ ref('spine_hours') }} diff --git a/dbt_jardiner/models/legacy/utils/spine_hours_localized_until_now.sql b/dbt_jardiner/models/legacy/utils/spine_hours_localized_until_now.sql index c7ab36fd..054ec7c2 100644 --- a/dbt_jardiner/models/legacy/utils/spine_hours_localized_until_now.sql +++ b/dbt_jardiner/models/legacy/utils/spine_hours_localized_until_now.sql @@ -1,6 +1,5 @@ - {{ config(materialized = 'table') }} select * -from {{ ref('spine_hours_localized')}} -where start_hour <= NOW() +from {{ ref('spine_hours_localized') }} +where start_hour <= now() diff --git a/dbt_jardiner/models/legacy/utils/spine_plant_meter_daily.sql b/dbt_jardiner/models/legacy/utils/spine_plant_meter_daily.sql index 59565537..7c488379 100644 --- a/dbt_jardiner/models/legacy/utils/spine_plant_meter_daily.sql +++ b/dbt_jardiner/models/legacy/utils/spine_plant_meter_daily.sql @@ -1,4 +1,3 @@ - {{ config(materialized = 'table') }} select @@ -10,6 +9,8 @@ select meter.id as meter_id, meter.name as meter_name, meter.connection_protocol as meter_connection_protocol -from {{ ref('spine_days_localized')}} as spine -left join {{ref('som_plants')}} as plant ON TRUE -left join {{source('plantmonitor_legacy','meter')}} on meter.plant = plant.plant_id +from {{ ref('spine_days_localized') }} as spine + left join {{ ref('som_plants') }} as plant on true + left join + {{ source('plantmonitor_legacy','meter') }} as meter + on meter.plant = plant.plant_id diff --git a/dbt_jardiner/models/legacy/utils/spine_plant_meter_hourly.sql b/dbt_jardiner/models/legacy/utils/spine_plant_meter_hourly.sql index a0ed0e98..bf1eb873 100644 --- a/dbt_jardiner/models/legacy/utils/spine_plant_meter_hourly.sql +++ b/dbt_jardiner/models/legacy/utils/spine_plant_meter_hourly.sql @@ -1,5 +1,4 @@ - -{{ config(materialized = 'table') }} +{{ config(materialized = 'view') }} select spine.start_hour as start_hour, @@ -10,9 +9,8 @@ select meter.id as meter_id, meter.name as meter_name, meter.connection_protocol as meter_connection_protocol -from {{ ref('spine_hours_localized')}} as spine -left join {{ref('som_plants')}} as plant ON TRUE -left join {{source('plantmonitor_legacy','meter')}} on meter.plant = plant.plant_id - - -{{ config(materialized='view') }} \ No newline at end of file +from {{ ref('spine_hours_localized') }} as spine + left join {{ ref('som_plants') }} as plant on true + left join + {{ source('plantmonitor_legacy','meter') }} as meter + on meter.plant = plant.plant_id diff --git a/dbt_jardiner/models/legacy/utils/spine_plant_meter_today.sql b/dbt_jardiner/models/legacy/utils/spine_plant_meter_today.sql index c533bcf0..a56a4210 100644 --- a/dbt_jardiner/models/legacy/utils/spine_plant_meter_today.sql +++ b/dbt_jardiner/models/legacy/utils/spine_plant_meter_today.sql @@ -1,7 +1,5 @@ - {{ config(materialized = 'table') }} -select -* -from {{ ref('spine_plant_meter_daily')}} as spine -where day <= NOW() \ No newline at end of file +select * +from {{ ref('spine_plant_meter_daily') }} +where day <= now() diff --git a/dbt_jardiner/models/legacy/utils/spine_plant_meter_until_last_hour.sql b/dbt_jardiner/models/legacy/utils/spine_plant_meter_until_last_hour.sql index 37cea5c3..2002c864 100644 --- a/dbt_jardiner/models/legacy/utils/spine_plant_meter_until_last_hour.sql +++ b/dbt_jardiner/models/legacy/utils/spine_plant_meter_until_last_hour.sql @@ -1,7 +1,5 @@ - {{ config(materialized = 'table') }} -select -* -from {{ ref('spine_plant_meter_hourly')}} as spine -where start_hour <= NOW() - INTERVAL '1 hour' \ No newline at end of file +select * +from {{ ref('spine_plant_meter_hourly') }} +where start_hour <= now() - interval '1 hour' diff --git a/dbt_jardiner/snapshots/snapshot_plant_parameters.sql b/dbt_jardiner/snapshots/snapshot_plant_parameters.sql index 49e120f9..31ddbd41 100644 --- a/dbt_jardiner/snapshots/snapshot_plant_parameters.sql +++ b/dbt_jardiner/snapshots/snapshot_plant_parameters.sql @@ -9,6 +9,6 @@ ) }} -select * from {{ source('airbyte','gestio_actius_dades_fixes') }} + select * from {{ source('airbyte','gestio_actius_dades_fixes') }} {% endsnapshot %} diff --git a/dbt_jardiner/snapshots/snapshot_production_target.sql b/dbt_jardiner/snapshots/snapshot_production_target.sql index cb9ca0bf..bf09d6c1 100644 --- a/dbt_jardiner/snapshots/snapshot_production_target.sql +++ b/dbt_jardiner/snapshots/snapshot_production_target.sql @@ -9,6 +9,6 @@ ) }} -select * from {{ source('airbyte','gestio_actius_objectius_de_produccio') }} + select * from {{ source('airbyte','gestio_actius_objectius_de_produccio') }} {% endsnapshot %} diff --git a/dbt_jardiner/snapshots/snapshot_signal_denormalized.sql b/dbt_jardiner/snapshots/snapshot_signal_denormalized.sql index cbe44d2a..8df62bde 100644 --- a/dbt_jardiner/snapshots/snapshot_signal_denormalized.sql +++ b/dbt_jardiner/snapshots/snapshot_signal_denormalized.sql @@ -9,6 +9,6 @@ ) }} -select * from {{ source('airbyte','gestio_actius_signal_denormalized') }} + select * from {{ source('airbyte','gestio_actius_signal_denormalized') }} {% endsnapshot %} diff --git a/dbt_jardiner/tests/dset/test_dset_active_power__missing_phase_last_day.sql b/dbt_jardiner/tests/dset/test_dset_active_power__missing_phase_last_day.sql index fe4bf567..c44fb518 100644 --- a/dbt_jardiner/tests/dset/test_dset_active_power__missing_phase_last_day.sql +++ b/dbt_jardiner/tests/dset/test_dset_active_power__missing_phase_last_day.sql @@ -7,32 +7,39 @@ {{ config(severity="warn") }} with - last_week as ( - select ts, plant_name, metric_name, device_uuid, signal_id, signal_code, signal_uuid - from {{ ref("obs_dset_responses__with_signal_metadata") }} - where ts > now() - interval '1 day' - ), +last_week as ( + select + ts, + plant_name, + metric_name, + device_uuid, + signal_id, + signal_code, + signal_uuid + from {{ ref("obs_dset_responses__with_signal_metadata") }} + where ts > now() - interval '1 day' +), - three_phase as ( - select - ts, - plant_name, - device_uuid, - count(*) as phase_count, - array_agg(signal_id) as signal_ids, - array_agg(signal_code) as signal_codes, - array_agg(signal_uuid) as signal_uuids - from last_week - where metric_name ilike 'potencia_activa_fase_%' - group by ts, plant_name, device_uuid - order by ts, plant_name, device_uuid - ), +three_phase as ( + select + ts, + plant_name, + device_uuid, + count(*) as phase_count, + array_agg(signal_id) as signal_ids, + array_agg(signal_code) as signal_codes, + array_agg(signal_uuid) as signal_uuids + from last_week + where metric_name ilike 'potencia_activa_fase_%' + group by ts, plant_name, device_uuid + order by ts, plant_name, device_uuid +), - missing_threephase_powers as ( - select * - from three_phase - where phase_count != 3 - ) +missing_threephase_powers as ( + select * + from three_phase + where phase_count != 3 +) select * from missing_threephase_powers diff --git a/dbt_jardiner/tests/dset/test_dset_gaps_per_day_and_signal_last_month.sql b/dbt_jardiner/tests/dset/test_dset_gaps_per_day_and_signal_last_month.sql index 4b610138..2b177ae9 100644 --- a/dbt_jardiner/tests/dset/test_dset_gaps_per_day_and_signal_last_month.sql +++ b/dbt_jardiner/tests/dset/test_dset_gaps_per_day_and_signal_last_month.sql @@ -1,88 +1,94 @@ {{ config(severity="warn") }} with - window_observed as ( - select - signal_value, - group_name, - signal_code, - signal_id, - signal_device_type, - signal_uuid, - queried_at, - ts as current_ts, - signal_frequency::interval as signal_frequency - from {{ ref("int_dset_responses__materialized") }} - where current_date - interval '1 month' < ts - and signal_value is not null - ), +window_observed as ( + select + signal_value, + group_name, + signal_code, + signal_id, + signal_device_type, + signal_uuid, + queried_at, + ts as current_ts, + signal_frequency::interval as signal_frequency + from {{ ref("int_dset_responses__materialized") }} + where + current_date - interval '1 month' < ts + and signal_value is not null +), - window_lagged as ( - select - *, - lag(current_ts) over ( - partition by signal_uuid - order by current_ts asc) as previous_ts - from window_observed - ), +window_lagged as ( + select + *, + lag(current_ts) over ( + partition by signal_uuid + order by current_ts asc + ) as previous_ts + from window_observed +), - gaps_observed as ( - select - *, - current_ts::date as "date", - current_ts - previous_ts as gap - from window_lagged - where current_ts - previous_ts > signal_frequency - ), +gaps_observed as ( + select + *, + current_ts::date as "date", + current_ts - previous_ts as gap + from window_lagged + where current_ts - previous_ts > signal_frequency +), - gaps_summarized as ( - select - "date", - group_name, - signal_code, - signal_id, - signal_device_type, - signal_uuid, - gap, - signal_frequency, - count(signal_uuid) as n_gaps - from gaps_observed - group by - "date", - group_name, - signal_uuid, - signal_code, - signal_id, - signal_device_type, - gap, - signal_frequency - order by - "date" desc, - count(signal_uuid) desc, - gap desc, - group_name asc, - signal_code asc, - signal_id asc, - signal_device_type asc - ), +gaps_summarized as ( + select + "date", + group_name, + signal_code, + signal_id, + signal_device_type, + signal_uuid, + gap, + signal_frequency, + count(signal_uuid) as n_gaps + from gaps_observed + group by + "date", + group_name, + signal_uuid, + signal_code, + signal_id, + signal_device_type, + gap, + signal_frequency + order by + "date" desc, + count(signal_uuid) desc, + gap desc, + group_name asc, + signal_code asc, + signal_id asc, + signal_device_type asc +), - gaps_converted_to_n_missing_samples as ( - select - *, - -- n_gaps * ceiling(gap/frequency - 1) as n_missing_samples. - -- The -1 is because the starting point in the gap can't be counted as missing - n_gaps * ceil(extract(epoch from gap) / extract(epoch from signal_frequency) - 1) as n_missing_samples, - -- 24*60 are the minutes in a day - (24 * 60) / (extract(epoch from signal_frequency) / 60) as n_samples_per_day - from gaps_summarized - ), +gaps_converted_to_n_missing_samples as ( + select + *, + -- n_gaps * ceiling(gap/frequency - 1) as n_missing_samples. + -- The -1 is because the starting point in the gap can't be counted as missing + n_gaps + * ceil( + extract(epoch from gap) / extract(epoch from signal_frequency) - 1 + ) as n_missing_samples, + -- 24*60 are the minutes in a day + (24 * 60) + / (extract(epoch from signal_frequency) / 60) as n_samples_per_day + from gaps_summarized +), - gaps_ratio as ( - select - *, - n_missing_samples / n_samples_per_day as ratio_missing_samples - from gaps_converted_to_n_missing_samples - ) +gaps_ratio as ( + select + *, + n_missing_samples / n_samples_per_day as ratio_missing_samples + from gaps_converted_to_n_missing_samples +) select * from gaps_ratio diff --git a/dbt_jardiner/tests/dset/test_dset_incoming_uuids_have_references_in_signal_denormalized.sql b/dbt_jardiner/tests/dset/test_dset_incoming_uuids_have_references_in_signal_denormalized.sql index a1be163b..c955df28 100644 --- a/dbt_jardiner/tests/dset/test_dset_incoming_uuids_have_references_in_signal_denormalized.sql +++ b/dbt_jardiner/tests/dset/test_dset_incoming_uuids_have_references_in_signal_denormalized.sql @@ -2,7 +2,9 @@ with child as ( - select *, signal_uuid as signal_uuid_dset + select + *, + signal_uuid as signal_uuid_dset from {{ ref("int_dset_responses__materialized") }} where signal_uuid is not null @@ -16,7 +18,9 @@ parent as ( ), joined as ( - select child.*, parent.* + select + child.*, + parent.* from child left join parent on child.signal_uuid_dset = parent.signal_uuid_som ), diff --git a/dbt_jardiner/tests/dset/test_dset_raw_uuids_not_empty_or_null_3_days.sql b/dbt_jardiner/tests/dset/test_dset_raw_uuids_not_empty_or_null_3_days.sql index 5774b2d7..a85987ea 100644 --- a/dbt_jardiner/tests/dset/test_dset_raw_uuids_not_empty_or_null_3_days.sql +++ b/dbt_jardiner/tests/dset/test_dset_raw_uuids_not_empty_or_null_3_days.sql @@ -1,25 +1,27 @@ {{ config(severity="warn") }} with - dbt_subquery as ( - select * - from {{ ref("raw_dset_responses__api_response") }} - where current_date - interval '3 days' < queried_at - ), +dbt_subquery as ( + select * + from {{ ref("raw_dset_responses__api_response") }} + where current_date - interval '3 days' < queried_at +), - all_values as ( - select *, - trim(signal_uuid) = '' as signal_uuid_is_empty, - signal_uuid is null as signal_uuid_is_null - from dbt_subquery - ), +all_values as ( + select + *, + trim(signal_uuid) = '' as signal_uuid_is_empty, + signal_uuid is null as signal_uuid_is_null + from dbt_subquery +), - errors as ( - select * - from all_values - where signal_uuid_is_empty is true - or signal_uuid_is_null is true - ) +errors as ( + select * + from all_values + where + signal_uuid_is_empty is true + or signal_uuid_is_null is true +) select * from errors diff --git a/dbt_jardiner/tests/dset/test_dset_signals_receiver_all_time.sql b/dbt_jardiner/tests/dset/test_dset_signals_receiver_all_time.sql index ac10f316..ae4af552 100644 --- a/dbt_jardiner/tests/dset/test_dset_signals_receiver_all_time.sql +++ b/dbt_jardiner/tests/dset/test_dset_signals_receiver_all_time.sql @@ -1,33 +1,34 @@ {{ config(error_if=">1000") }} with - valors as ( - select - signal_uuid, - true as received_from_dset_pre - from {{ ref("int_dset_responses__materialized") }} - where ts = (select max(ts) from {{ ref("int_dset_responses__materialized") }}) - ), +valors as ( + select + signal_uuid, + true as received_from_dset_pre + from {{ ref("int_dset_responses__materialized") }} + where + ts = (select max(ts) from {{ ref("int_dset_responses__materialized") }}) +), - joined as ( - select - signals.plant_uuid, - signals.plant_name, - signals.signal_name, - signals.signal_uuid, - signals.device_name, - signals.device_type, - signals.device_uuid, - coalesce(valors.received_from_dset_pre, false) as received_from_dset - from {{ ref("raw_gestio_actius__signal_denormalized") }} as signals - left join valors on signals.signal_uuid = valors.signal_uuid - order by signals.plant_name - ), +joined as ( + select + signals.plant_uuid, + signals.plant_name, + signals.signal_name, + signals.signal_uuid, + signals.device_name, + signals.device_type, + signals.device_uuid, + coalesce(valors.received_from_dset_pre, false) as received_from_dset + from {{ ref("raw_gestio_actius__signal_denormalized") }} as signals + left join valors on signals.signal_uuid = valors.signal_uuid + order by signals.plant_name +), - filtered as ( - select * - from joined - where received_from_dset is false - ) +filtered as ( + select * + from joined + where received_from_dset is false +) select * from filtered diff --git a/dbt_jardiner/tests/dset/test_dset_signals_receiver_last_hour.sql b/dbt_jardiner/tests/dset/test_dset_signals_receiver_last_hour.sql index ea97c8e3..fc430df3 100644 --- a/dbt_jardiner/tests/dset/test_dset_signals_receiver_last_hour.sql +++ b/dbt_jardiner/tests/dset/test_dset_signals_receiver_last_hour.sql @@ -3,36 +3,39 @@ with uuids_received_recently as ( - select - signal_uuid, - true as is_received_recently, - max(ts) as last_received_ts - from {{ ref("int_dset_responses__materialized") }} - where ts >= (now() - interval '2 hours') - group by signal_uuid - {# interval used of two hours is depending on the natural delay of dset data + materialization cycle -#} + select + signal_uuid, + true as is_received_recently, + max(ts) as last_received_ts + from {{ ref("int_dset_responses__materialized") }} + where ts >= (now() - interval '2 hours') + group by signal_uuid + {# interval used of two hours is depending on the natural delay of dset data + materialization cycle -#} + + + ), uuids_expected as ( - select - s.plant_uuid, - s.plant_name, - s.signal_name, - s.signal_uuid, - s.device_name, - s.device_type, - s.device_uuid, - coalesce(r.is_received_recently, false) as received_from_dset - from {{ ref("raw_gestio_actius__signal_denormalized") }} as s + select + s.plant_uuid, + s.plant_name, + s.signal_name, + s.signal_uuid, + s.device_name, + s.device_type, + s.device_uuid, + coalesce(r.is_received_recently, false) as received_from_dset + from {{ ref("raw_gestio_actius__signal_denormalized") }} as s left join uuids_received_recently as r on s.signal_uuid = r.signal_uuid - order by s.plant_name + order by s.plant_name ), uuids_not_received as ( - select * - from uuids_expected - where received_from_dset is false + select * + from uuids_expected + where received_from_dset is false ) select * from uuids_not_received diff --git a/dbt_jardiner/tests/dset/test_gapfill_5min_nrows.sql b/dbt_jardiner/tests/dset/test_gapfill_5min_nrows.sql index 30e1ef53..3f469eaa 100644 --- a/dbt_jardiner/tests/dset/test_gapfill_5min_nrows.sql +++ b/dbt_jardiner/tests/dset/test_gapfill_5min_nrows.sql @@ -1,13 +1,14 @@ -- Test 2: Assregurem que efectgivament hi ha 288 rows per dia (5 minuts) -- TODO: Treure dia d'avui + tenir en compte els dies cavni d'hora -SELECT * -FROM ( - SELECT - (time at time zone 'UTC')::date as day, - inverter_id, - count(*) as n - FROM {{ ref('inverterregistry_clean') }} as ir - group by (time at time zone 'UTC')::date, inverter_id -) AS TAL -where n != 288 -and day between '2020-01-01' and current_date - interval '1 day' \ No newline at end of file +select * +from ( + select + (time at time zone 'UTC')::date as day, + inverter_id, + count(*) as n + from {{ ref('inverterregistry_clean') }} + group by (time at time zone 'UTC')::date, inverter_id +) as tal +where + n != 288 + and day between '2020-01-01' and current_date - interval '1 day' diff --git a/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql b/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql index 3ea9c7f9..33ab6445 100644 --- a/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql +++ b/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql @@ -1,22 +1,35 @@ -- tests pairs device_uuid, plant, device, device_type, device_parent as unique with unique_device_per_plant as ( - select distinct device_uuid, plant_uuid, plant_name, device_name, device_type, device_parent, device_parent_uuid + select distinct + device_uuid, + plant_uuid, + plant_name, + device_name, + device_type, + device_parent, + device_parent_uuid from {{ ref("raw_gestio_actius__signal_denormalized") }} order by plant_name, device_name ), duplicated_devices as ( - select udpp.device_uuid, count(*) as n_duplicates + select + udpp.device_uuid, + count(*) as n_duplicates from unique_device_per_plant as udpp group by udpp.device_uuid having count(*) > 1 ), joined as ( - select expected.*, duplicated_devices.n_duplicates + select + expected.*, + duplicated_devices.n_duplicates from {{ ref("raw_gestio_actius__signal_denormalized") }} as expected - left join duplicated_devices on expected.device_uuid = duplicated_devices.device_uuid + left join + duplicated_devices + on expected.device_uuid = duplicated_devices.device_uuid ), filtered as ( diff --git a/dbt_jardiner/tests/dset/test_signal_is_correctly_converted_to_numeric.sql b/dbt_jardiner/tests/dset/test_signal_is_correctly_converted_to_numeric.sql index 35c1ee3e..6db85e62 100644 --- a/dbt_jardiner/tests/dset/test_signal_is_correctly_converted_to_numeric.sql +++ b/dbt_jardiner/tests/dset/test_signal_is_correctly_converted_to_numeric.sql @@ -1,7 +1,6 @@ {{ config(enabled=true, severity='warn', warn_if = '>0') }} -select distinct on (plant_name, device_name, signal_uuid) - * +select distinct on (plant_name, device_name, signal_uuid) * from {{ ref('obs_dset_responses__with_signal_metadata') }} where signal_value between 65500 and 65599 order by plant_name asc, device_name asc, signal_uuid asc, ts desc diff --git a/dbt_jardiner/tests/dset/test_solargis_temperature_registry.sql b/dbt_jardiner/tests/dset/test_solargis_temperature_registry.sql index c97e33c7..b2046c93 100644 --- a/dbt_jardiner/tests/dset/test_solargis_temperature_registry.sql +++ b/dbt_jardiner/tests/dset/test_solargis_temperature_registry.sql @@ -7,13 +7,18 @@ with validation as ( select production.nom_planta, date_trunc('day', production.hora_inici, 'Europe/Madrid') as day_inici, - sum((production.temperatura_modul_avg_c is null)::integer) / count(*)::numeric as not_null_proportion + sum((production.temperatura_modul_avg_c is null)::integer) + / count(*)::numeric as not_null_proportion from {{ ref('dm_plant_production_hourly') }} as production where - date_trunc('day', production.hora_inici, 'Europe/Madrid') > (now() at time zone 'Europe/Madrid')::date - interval '5 days' + date_trunc('day', production.hora_inici, 'Europe/Madrid') + > (now() at time zone 'Europe/Madrid')::date - interval '5 days' and production.te_plantmonitor - group by production.nom_planta, date_trunc('day', production.hora_inici, 'Europe/Madrid') + group by + production.nom_planta, + date_trunc('day', production.hora_inici, 'Europe/Madrid') ), + validation_errors as ( select nom_planta, @@ -22,6 +27,6 @@ validation_errors as ( from validation where not_null_proportion < 0.4 or not_null_proportion > 1 ) -select - * + +select * from validation_errors diff --git a/dbt_jardiner/tests/dset/test_spine_hourly.sql b/dbt_jardiner/tests/dset/test_spine_hourly.sql index bef8052d..629a8274 100644 --- a/dbt_jardiner/tests/dset/test_spine_hourly.sql +++ b/dbt_jardiner/tests/dset/test_spine_hourly.sql @@ -4,19 +4,31 @@ with dst_change as ( select * from {{ ref('spine_hourly') }} - where start_hour between '2022-03-27 01:00+02'::timestamptz and '2022-03-27 03:00+02'::timestamptz - or start_hour between '2022-10-30 01:00+02'::timestamptz and '2022-10-30 03:00+02'::timestamptz + where + start_hour between '2022-03-27 01:00+02'::timestamptz and '2022-03-27 03:00+02'::timestamptz + or start_hour between '2022-10-30 01:00+02'::timestamptz and '2022-10-30 03:00+02'::timestamptz ), + dst_count_summer as ( - select 'summer' as dst, count(*) as num_hours from dst_change - where start_hour between '2022-03-27 01:00+02'::timestamptz and '2022-03-27 03:00+02'::timestamptz + select + 'summer' as dst, + count(*) as num_hours + from dst_change + where + start_hour between '2022-03-27 01:00+02'::timestamptz and '2022-03-27 03:00+02'::timestamptz ), + dst_count_winter as ( - select 'winter' as dst, count(*) as num_hours from dst_change - where start_hour between '2022-10-30 01:00+02'::timestamptz and '2022-10-30 03:00+02'::timestamptz + select + 'winter' as dst, + count(*) as num_hours + from dst_change + where + start_hour between '2022-10-30 01:00+02'::timestamptz and '2022-10-30 03:00+02'::timestamptz ) + select * from dst_count_summer where num_hours != 4 -UNION +union select * from dst_count_winter where num_hours != 2