WIP

earthobservations · Nov 24, 2024 · b1da0ef · b1da0ef
1 parent 9938df5
commit b1da0ef
Show file tree

Hide file tree

Showing 24 changed files with 122 additions and 145 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -54,11 +54,11 @@ jobs:
       - name: Install project
         run: uv sync
 
-      - name: Install eccodes (Mac only)
-        if: ${{ startsWith(matrix.os, 'macos-') }}
-        run: |
-          brew install eccodes
-          export WD_ECCODES_DIR=$(brew --prefix eccodes)
+#      - name: Install eccodes (Mac only)
+#        if: ${{ startsWith(matrix.os, 'macos-') }}
+#        run: |
+#          brew install eccodes
+#          export WD_ECCODES_DIR=$(brew --prefix eccodes)
 
       - name: Install project
         run: .github/workflows/install.sh testing

diff --git a/tests/core/timeseries/test_api.py b/tests/core/timeseries/test_api.py
@@ -4,9 +4,8 @@
 
 from wetterdienst.exceptions import NoParametersFoundError
 from wetterdienst.provider.dwd.observation import (
-    DwdObservationDataset,
+    DwdObservationMetadata,
     DwdObservationRequest,
-    DwdObservationResolution,
 )
 
 
@@ -20,8 +19,7 @@ def test_api_skip_empty_stations(settings_skip_empty_true, ts_skip_criteria, exp
     settings_skip_empty_true.ts_skip_criteria = ts_skip_criteria
     settings_skip_empty_true.ts_skip_threshold = 0.6
     request = DwdObservationRequest(
-        parameter=["kl", "solar"],
-        resolution="daily",
+        parameter=[("daily", "kl"), ("daily", "solar")],
         start_date="2021-01-01",
         end_date="2021-12-31",
         settings=settings_skip_empty_true,
@@ -42,8 +40,7 @@ def test_api_skip_empty_stations_equal_on_any_skip_criteria_with_one_parameter(s
     def _get_values(settings):
         return (
             DwdObservationRequest(
-                parameter=["sunshine_duration"],
-                resolution="daily",
+                parameter=[("daily", "climate_summary", "sunshine_duration")],
                 start_date="1990-01-01",
                 end_date="2021-12-31",
                 settings=settings,
@@ -75,10 +72,9 @@ def _get_values(settings):
 def test_api_dropna(settings_dropna_true):
     request = DwdObservationRequest(
         parameter=[
-            "temperature_air",
-            "precipitation",
+            ("minute_10", "temperature_air"),
+            ("minute_10", "precipitation"),
         ],
-        resolution="minute_10",
         start_date="2021-01-01",
         end_date="2021-12-31",
         settings=settings_dropna_true,
@@ -91,29 +87,22 @@ def test_api_no_valid_parameters(default_settings):
     with pytest.raises(NoParametersFoundError):
         DwdObservationRequest(
             parameter=[
-                DwdObservationDataset.TEMPERATURE_AIR,
+                ("daily", "abc"),
             ],
-            resolution=DwdObservationResolution.DAILY,
             settings=default_settings,
         )
 
 
 def test_api_partly_valid_parameters(default_settings, caplog):
     request = DwdObservationRequest(
         parameter=[
-            DwdObservationDataset.TEMPERATURE_AIR,
-            DwdObservationDataset.WIND,
-            DwdObservationDataset.PRECIPITATION,
-            DwdObservationDataset.SOLAR,
+            ("daily", "temperature_air"),
+            ("daily", "wind"),
+            ("daily", "precipitation"),
+            ("daily", "solar"),
         ],
-        resolution=DwdObservationResolution.DAILY,
         settings=default_settings,
     )
-    assert "dataset WIND is not a valid dataset for resolution DAILY" in caplog.text
-    assert "dataset PRECIPITATION is not a valid dataset for resolution DAILY" in caplog.text
-    assert request.parameter == [
-        (
-            DwdObservationDataset.SOLAR,
-            DwdObservationDataset.SOLAR,
-        ),
-    ]
+    assert "daily/wind not found in DwdObservationMetadata" in caplog.text
+    assert "daily/precipitation not found in DwdObservationMetadata" in caplog.text
+    assert request.parameter == [*DwdObservationMetadata.daily.solar]
diff --git a/tests/core/timeseries/test_interpolation.py b/tests/core/timeseries/test_interpolation.py
@@ -133,7 +133,7 @@ def test_not_interpolatable_parameter(default_settings, df_interpolated_empty):
 
 def test_not_interpolatable_dataset(default_settings, df_interpolated_empty):
     request = DwdObservationRequest(
-        parameter=[("hourly", "temperature", "temperature_air")],
+        parameter=[("daily", "climate_summary", "precipitation_form")],
         start_date=dt.datetime(2022, 1, 1),
         end_date=dt.datetime(2022, 1, 2),
         settings=default_settings,

diff --git a/tests/provider/dwd/road/test_api.py b/tests/provider/dwd/road/test_api.py
@@ -10,7 +10,7 @@
 @pytest.mark.skipif(not ensure_eccodes() or not ensure_pdbufr(), reason="eccodes and/or pdbufr not installed")
 @pytest.mark.remote
 def test_dwd_road_weather():
-    request = DwdRoadRequest(parameter="temperature_air_mean_2m").filter_by_station_id("A006")
+    request = DwdRoadRequest(parameter=[("15_minutes", "data", "temperature_air_mean_2m")]).filter_by_station_id("A006")
     item = request.to_dict()["stations"][0]
     assert item == {
         "station_id": "A006",

diff --git a/tests/ui/cli/test_cli_values.py b/tests/ui/cli/test_cli_values.py
@@ -121,7 +121,7 @@ def test_cli_values_json_multiple_datasets():
     result = invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl,more_precip --date=2020-06-30",
+        setting="--parameter=daily/kl,daily/more_precip --date=2020-06-30",
         station="01048",
         fmt="json",
     )
@@ -172,7 +172,7 @@ def test_cli_values_json_with_metadata_with_stations(metadata):
     result = invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=historical",
+        setting="--parameter=daily/kl --period=historical",
         station="01047,01048",
         fmt="json",
         additional="--with-metadata=true --with-stations=true",
@@ -210,7 +210,7 @@ def test_cli_values_json_indent_false(json_dumps_mock):
     invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=recent",
+        setting="--parameter=daily/kl --period=recent",
         station="01048",
         fmt="json",
         additional="--pretty=false",
@@ -224,7 +224,7 @@ def test_cli_values_json_indent_true(json_dumps_mock):
     invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=recent",
+        setting="--parameter=daily/kl --period=recent",
         station="01048",
         fmt="json",
         additional="--pretty=true",
@@ -237,7 +237,7 @@ def test_cli_values_geojson():
     result = invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=recent",
+        setting="--parameter=daily/kl --period=recent",
         station="01048",
         fmt="geojson",
     )
@@ -268,7 +268,7 @@ def test_cli_values_geojson_with_metadata(metadata):
     result = invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=recent",
+        setting="--parameter=daily/kl --period=recent",
         station="01048",
         fmt="geojson",
         additional="--with-metadata=true",
@@ -284,7 +284,7 @@ def test_cli_values_geojson_pretty_false(json_dumps_mock):
     invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=recent",
+        setting="--parameter=daily/kl --period=recent",
         station="01048",
         fmt="geojson",
         additional="--pretty=false",
@@ -298,7 +298,7 @@ def test_cli_values_geojson_pretty_true(json_dumps_mock):
     invoke_wetterdienst_values_static(
         provider="dwd",
         network="observation",
-        setting="--resolution=daily --parameter=kl --period=recent",
+        setting="--parameter=daily/kl --period=recent",
         station="01048",
         fmt="geojson",
         additional="--pretty=true",

diff --git a/tests/ui/test_restapi.py b/tests/ui/test_restapi.py
@@ -86,8 +86,7 @@ def test_dwd_stations_basic(client):
         params={
             "provider": "dwd",
             "network": "observation",
-            "parameter": "kl",
-            "resolution": "daily",
+            "parameter": "daily/kl",
             "period": "recent",
             "all": "true",
         },
@@ -113,8 +112,7 @@ def test_dwd_stations_geo(client):
         params={
             "provider": "dwd",
             "network": "observation",
-            "parameter": "kl",
-            "resolution": "daily",
+            "parameter": "daily/kl",
             "period": "recent",
             "coordinates": "45.54,10.10",
             "rank": 5,
@@ -142,8 +140,7 @@ def test_dwd_stations_sql(client):
         params={
             "provider": "dwd",
             "network": "observation",
-            "parameter": "kl",
-            "resolution": "daily",
+            "parameter": "daily/kl",
             "period": "recent",
             "sql": "SELECT * FROM data WHERE lower(name) LIKE '%dresden%';",
         },
@@ -170,8 +167,7 @@ def test_dwd_values_success(client):
             "provider": "dwd",
             "network": "observation",
             "station": "01359",
-            "parameter": "kl",
-            "resolution": "daily",
+            "parameter": "daily/kl",
             "period": "historical",
             "date": "1982-01-01",
         },
@@ -215,26 +211,11 @@ def test_dwd_values_no_parameter(client):
             "provider": "dwd",
             "network": "observation",
             "station": "01048,4411",
-            "resolution": "daily",
             "period": "recent",
         },
     )
     assert response.status_code == 400
-    assert response.json() == {"detail": "Query arguments 'parameter', 'resolution' and 'date' are required"}
-
-
-def test_dwd_values_no_resolution(client):
-    response = client.get(
-        "/api/values",
-        params={
-            "provider": "dwd",
-            "network": "observation",
-            "parameter": "kl",
-            "period": "recent",
-        },
-    )
-    assert response.status_code == 400
-    assert response.json() == {"detail": "Query arguments 'parameter', 'resolution' and 'date' are required"}
+    assert response.json() == {"detail": "Query argument 'parameter' is required"}
 
 
 @pytest.mark.remote
@@ -246,8 +227,7 @@ def test_dwd_values_sql_tabular(client):
             "provider": "dwd",
             "network": "observation",
             "station": "01048,4411",
-            "parameter": "kl",
-            "resolution": "daily",
+            "parameter": "daily/kl",
             "period": "historical",
             "date": "2020/2021",
             "sql-values": "SELECT * FROM data WHERE temperature_air_max_2m < 2.0",
@@ -303,8 +283,7 @@ def test_dwd_values_sql_long(client):
             "provider": "dwd",
             "network": "observation",
             "station": "01048,4411",
-            "parameter": "kl",
-            "resolution": "daily",
+            "parameter": "daily/kl",
             "date": "2019-12-01/2019-12-31",
             "sql-values": "SELECT * FROM data WHERE parameter='temperature_air_max_2m' AND value < 1.5",
             "si-units": False,
@@ -329,8 +308,7 @@ def test_dwd_interpolate(client):
         params={
             "provider": "dwd",
             "network": "observation",
-            "parameter": "temperature_air_mean_2m",
-            "resolution": "daily",
+            "parameter": "daily/kl/temperature_air_mean_2m",
             "station": "00071",
             "date": "1986-10-31/1986-11-01",
         },
@@ -398,8 +376,7 @@ def test_api_values_missing_null(client):
             "provider": "dwd",
             "network": "mosmix",
             "station": "F660",
-            "parameter": "ttt",
-            "resolution": "small",
+            "parameter": "hourly/small/ttt",
         },
     )
     assert response.status_code == 200
@@ -414,8 +391,7 @@ def test_api_values_missing_empty(client):
             "provider": "dwd",
             "network": "observation",
             "station": "00011",
-            "parameter": "precipitation_height",
-            "resolution": "1_minute",
+            "parameter": "1_minute/precipitation/precipitation_height",
             "period": "recent",
         },
     )
@@ -430,8 +406,7 @@ def test_api_stations_missing_null(client):
         params={
             "provider": "dwd",
             "network": "mosmix",
-            "parameter": "ttt",
-            "resolution": "small",
+            "parameter": "hourly/small/ttt",
             "all": True,
         },
     )
@@ -457,8 +432,7 @@ def test_dwd_mosmix(client):
         params={
             "provider": "dwd",
             "network": "mosmix",
-            "parameter": "ttt",
-            "resolution": "small",
+            "parameter": "hourly/small/ttt",
             "station": "01025",
         },
     )
@@ -481,8 +455,7 @@ def test_dwd_dmo_lead_time_long(client):
         params={
             "provider": "dwd",
             "network": "dmo",
-            "parameter": "ttt",
-            "resolution": "icon",
+            "parameter": "hourly/icon/ttt",
             "station": "01025",
             "lead-time": "long",
         },

diff --git a/wetterdienst/core/timeseries/values.py b/wetterdienst/core/timeseries/values.py
@@ -585,18 +585,15 @@ def _get_actual_percentage(self, df: pl.DataFrame) -> float:
         :param df: pandas DataFrame with values
         :return: float of actual percentage of values
         """
-        parameters = []
-        for parameter, dataset in self.sr.parameter:
-            if parameter != dataset:
-                parameters.append(parameter.value)
-            else:
-                dataset_enum = self.sr.stations._parameter_base[self.sr.resolution.name][dataset.name]
-                parameters.extend([par.value for par in dataset_enum if not par.name.lower().startswith("quality")])
         percentage = df.group_by(["parameter"]).agg(
             (pl.col("value").drop_nulls().len() / pl.col("value").len()).cast(pl.Float64).alias("perc"),
         )
         missing = pl.DataFrame(
-            [{"parameter": par, "perc": 0.0} for par in parameters if par not in percentage.get_column("parameter")],
+            [
+                {"parameter": parameter.name_original, "perc": 0.0}
+                for parameter in self.sr.parameter
+                if parameter.name_original not in percentage.get_column("parameter")
+            ],
             schema={"parameter": pl.String, "perc": pl.Float64},
         )
         percentage = pl.concat([percentage, missing])