Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle missing RDPS data #3996

Merged
merged 4 commits into from
Oct 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions api/app/tests/weather_models/test_precip_rdps_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@
import pytest
from pytest_mock import MockerFixture
from datetime import datetime, timezone
from unittest.mock import patch
from app.tests.utils.raster_reader import read_raster_array

from app.weather_models.precip_rdps_model import TemporalPrecip, compute_precip_difference, get_raster_keys_to_diff, generate_24_hour_accumulating_precip_raster
from app.weather_models.precip_rdps_model import (
TemporalPrecip,
compute_and_store_precip_rasters,
compute_precip_difference,
get_raster_keys_to_diff,
generate_24_hour_accumulating_precip_raster,
)
from app.weather_models.rdps_filename_marshaller import model_run_for_hour

geotransform = (-4556441.403315245, 10000.0, 0.0, 920682.1411659503, 0.0, -10000.0)
Expand Down Expand Up @@ -75,13 +82,15 @@ async def test_generate_24_hour_accumulating_precip_raster_model_hour_ok(mocker:
],
)
@pytest.mark.anyio
async def test_generate_24_hour_accumulating_precip_raster_fail(current_time: datetime, today_raster: np.ndarray, yesterday_raster: np.ndarray, mocker: MockerFixture):
async def test_generate_24_hour_accumulating_precip_raster_no_today_raster(current_time: datetime, today_raster: np.ndarray, yesterday_raster: np.ndarray, mocker: MockerFixture):
"""
Verify that the appropriate rasters are diffed correctly.
"""
mocker.patch("app.weather_models.precip_rdps_model.read_into_memory", side_effect=[today_raster, yesterday_raster])
with pytest.raises(ValueError):
await generate_24_hour_accumulating_precip_raster(current_time)
(day_data, day_geotransform, day_projection) = await generate_24_hour_accumulating_precip_raster(current_time)
assert day_data is None
assert day_geotransform is None
assert day_projection is None


@pytest.mark.parametrize(
Expand Down Expand Up @@ -145,3 +154,13 @@ def test_get_raster_keys_to_diff(timestamp: datetime, expected_yesterday_key, ex
(yesterday_key, today_key) = get_raster_keys_to_diff(timestamp)
assert yesterday_key == expected_yesterday_key
assert today_key == expected_today_key

async def return_none_tuple(timestamp: datetime):
return (None, None, None)


@patch("app.weather_models.precip_rdps_model.generate_24_hour_accumulating_precip_raster", return_none_tuple)
@pytest.mark.anyio
async def test_compute_and_store_precip_rasters_no_today_data_does_not_throw():
timestamp = datetime.fromisoformat("2024-06-10T18:42:49+00:00")
await compute_and_store_precip_rasters(timestamp)
6 changes: 5 additions & 1 deletion api/app/weather_models/precip_rdps_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ async def compute_and_store_precip_rasters(model_run_timestamp: datetime):
for hour in range(0, 36):
accumulation_timestamp = model_run_timestamp + timedelta(hours=hour)
(precip_diff_raster, geotransform, projection) = await generate_24_hour_accumulating_precip_raster(accumulation_timestamp)
if precip_diff_raster is None:
conbrad marked this conversation as resolved.
Show resolved Hide resolved
# If there is no precip_diff_raster, RDPS precip data is not available. We'll retry the cron job in one hour.
logger.warning(f"No precip raster data for hour: {hour} and model run timestamp: {model_run_timestamp.strftime('%Y-%m-%d_%H:%M:%S')}")
break
key = f"weather_models/{ModelEnum.RDPS.lower()}/{accumulation_timestamp.date().isoformat()}/" + compose_computed_precip_rdps_key(
accumulation_end_datetime=accumulation_timestamp
)
Expand Down Expand Up @@ -92,7 +96,7 @@ async def generate_24_hour_accumulating_precip_raster(timestamp: datetime):
(yesterday_key, today_key) = get_raster_keys_to_diff(timestamp)
(day_data, day_geotransform, day_projection) = await read_into_memory(today_key)
if day_data is None:
raise ValueError("No precip raster data for today_key: %s" % today_key)
return (day_data, day_geotransform, day_projection)
if yesterday_key is None:
return (day_data, day_geotransform, day_projection)

Expand Down
Loading