From 319a626f2d5ae4c93afa91119888e1d3ee5b4a28 Mon Sep 17 00:00:00 2001 From: Darren Boss Date: Thu, 10 Oct 2024 10:42:34 -0700 Subject: [PATCH 1/4] Handle missing RDPS data --- api/app/weather_models/precip_rdps_model.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/app/weather_models/precip_rdps_model.py b/api/app/weather_models/precip_rdps_model.py index e29960209..6bb252118 100644 --- a/api/app/weather_models/precip_rdps_model.py +++ b/api/app/weather_models/precip_rdps_model.py @@ -36,6 +36,10 @@ async def compute_and_store_precip_rasters(model_run_timestamp: datetime): for hour in range(0, 36): accumulation_timestamp = model_run_timestamp + timedelta(hours=hour) (precip_diff_raster, geotransform, projection) = await generate_24_hour_accumulating_precip_raster(accumulation_timestamp) + if precip_diff_raster is None: + # If there is no precip_diff_raster, RDPS precip data is not available. We'll retry the cron job in one hour. + logger.warning(f"No precip raster data for hour: {hour} and model run timestamp: {model_run_timestamp.strftime('%Y-%m-%d_%H:%M:%S')}") + break key = f"weather_models/{ModelEnum.RDPS.lower()}/{accumulation_timestamp.date().isoformat()}/" + compose_computed_precip_rdps_key( accumulation_end_datetime=accumulation_timestamp ) @@ -92,7 +96,7 @@ async def generate_24_hour_accumulating_precip_raster(timestamp: datetime): (yesterday_key, today_key) = get_raster_keys_to_diff(timestamp) (day_data, day_geotransform, day_projection) = await read_into_memory(today_key) if day_data is None: - raise ValueError("No precip raster data for today_key: %s" % today_key) + return (day_data, day_geotransform, day_projection) if yesterday_key is None: return (day_data, day_geotransform, day_projection) From 32d689415ce1c4268a6e4c5eae214cd741b89319 Mon Sep 17 00:00:00 2001 From: Darren Boss Date: Thu, 10 Oct 2024 12:46:09 -0700 Subject: [PATCH 2/4] Fix test --- api/app/tests/weather_models/test_precip_rdps_model.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/app/tests/weather_models/test_precip_rdps_model.py b/api/app/tests/weather_models/test_precip_rdps_model.py index 79858b0e0..433c2e2b8 100644 --- a/api/app/tests/weather_models/test_precip_rdps_model.py +++ b/api/app/tests/weather_models/test_precip_rdps_model.py @@ -75,13 +75,15 @@ async def test_generate_24_hour_accumulating_precip_raster_model_hour_ok(mocker: ], ) @pytest.mark.anyio -async def test_generate_24_hour_accumulating_precip_raster_fail(current_time: datetime, today_raster: np.ndarray, yesterday_raster: np.ndarray, mocker: MockerFixture): +async def test_generate_24_hour_accumulating_precip_raster_no_today_raster(current_time: datetime, today_raster: np.ndarray, yesterday_raster: np.ndarray, mocker: MockerFixture): """ Verify that the appropriate rasters are diffed correctly. """ mocker.patch("app.weather_models.precip_rdps_model.read_into_memory", side_effect=[today_raster, yesterday_raster]) - with pytest.raises(ValueError): - await generate_24_hour_accumulating_precip_raster(current_time) + (day_data, day_geotransform, day_projection) = await generate_24_hour_accumulating_precip_raster(current_time) + assert day_data is None + assert day_geotransform is None + assert day_projection is None @pytest.mark.parametrize( From 4eddc12b45b06f49b12d2fc633d5f589de4fbd72 Mon Sep 17 00:00:00 2001 From: Darren Boss Date: Thu, 10 Oct 2024 16:58:13 -0700 Subject: [PATCH 3/4] Add test --- .../weather_models/test_precip_rdps_model.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/api/app/tests/weather_models/test_precip_rdps_model.py b/api/app/tests/weather_models/test_precip_rdps_model.py index 433c2e2b8..a2855bd5a 100644 --- a/api/app/tests/weather_models/test_precip_rdps_model.py +++ b/api/app/tests/weather_models/test_precip_rdps_model.py @@ -3,9 +3,16 @@ import pytest from pytest_mock import MockerFixture from datetime import datetime, timezone +from unittest.mock import patch from app.tests.utils.raster_reader import read_raster_array -from app.weather_models.precip_rdps_model import TemporalPrecip, compute_precip_difference, get_raster_keys_to_diff, generate_24_hour_accumulating_precip_raster +from app.weather_models.precip_rdps_model import ( + TemporalPrecip, + compute_and_store_precip_rasters, + compute_precip_difference, + get_raster_keys_to_diff, + generate_24_hour_accumulating_precip_raster, +) from app.weather_models.rdps_filename_marshaller import model_run_for_hour geotransform = (-4556441.403315245, 10000.0, 0.0, 920682.1411659503, 0.0, -10000.0) @@ -147,3 +154,14 @@ def test_get_raster_keys_to_diff(timestamp: datetime, expected_yesterday_key, ex (yesterday_key, today_key) = get_raster_keys_to_diff(timestamp) assert yesterday_key == expected_yesterday_key assert today_key == expected_today_key + +async def return_none_tuple(timestamp: datetime): + return (None, None, None) + + +@patch("app.weather_models.precip_rdps_model.generate_24_hour_accumulating_precip_raster", return_none_tuple) +@pytest.mark.anyio +async def test_compute_and_store_precip_rasters_no_today_data(caplog): + timestamp = datetime.fromisoformat("2024-06-10T18:42:49+00:00") + await compute_and_store_precip_rasters(timestamp) + assert "No precip raster data for hour:" in caplog.text From b20cbc00e70238d2a412e331b76beb03043a68b6 Mon Sep 17 00:00:00 2001 From: Darren Boss Date: Fri, 11 Oct 2024 09:15:14 -0700 Subject: [PATCH 4/4] Tweak test --- api/app/tests/weather_models/test_precip_rdps_model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/app/tests/weather_models/test_precip_rdps_model.py b/api/app/tests/weather_models/test_precip_rdps_model.py index a2855bd5a..1ed27c61b 100644 --- a/api/app/tests/weather_models/test_precip_rdps_model.py +++ b/api/app/tests/weather_models/test_precip_rdps_model.py @@ -161,7 +161,6 @@ async def return_none_tuple(timestamp: datetime): @patch("app.weather_models.precip_rdps_model.generate_24_hour_accumulating_precip_raster", return_none_tuple) @pytest.mark.anyio -async def test_compute_and_store_precip_rasters_no_today_data(caplog): +async def test_compute_and_store_precip_rasters_no_today_data_does_not_throw(): timestamp = datetime.fromisoformat("2024-06-10T18:42:49+00:00") await compute_and_store_precip_rasters(timestamp) - assert "No precip raster data for hour:" in caplog.text