diff --git a/forecastmanager/forecast_settings.py b/forecastmanager/forecast_settings.py index de57ab3..3a33141 100644 --- a/forecastmanager/forecast_settings.py +++ b/forecastmanager/forecast_settings.py @@ -62,7 +62,7 @@ def periods_as_choices(self): @property def effective_periods(self): return [ - {"label": period.label, "time": period.forecast_effective_time, "default": period.default} + {"label": period.label, "time": period.forecast_effective_time} for period in self.periods.all()] @property @@ -73,27 +73,20 @@ def weather_conditions_list(self): class ForecastPeriod(Orderable): parent = ParentalKey(ForecastSetting, on_delete=models.CASCADE, related_name="periods") - default = models.BooleanField(default=False, verbose_name=_("Is default")) - forecast_effective_time = models.TimeField(verbose_name=_("Forecast Effective Time")) + forecast_effective_time = models.TimeField(verbose_name=_("Forecast Effective Time"), unique=True) label = models.CharField(max_length=100, verbose_name=_("Label")) class Meta: - unique_together = ("default", "forecast_effective_time") + ordering = ["forecast_effective_time"] panels = [ FieldPanel('forecast_effective_time'), FieldPanel('label'), - FieldPanel('default'), ] def __str__(self): return self.label - def save(self, *args, **kwargs): - if self.default: - ForecastPeriod.objects.filter(default=True).update(default=False) - super().save(*args, **kwargs) - class ForecastDataParameters(Orderable): PARAMETER_TYPE_CHOICES = ( @@ -136,7 +129,8 @@ def parameter_info(self): return WEATHER_PARAMETERS_AS_DICT.get(self.parameter) def parse_value(self, value): - # TODO: Implement parsing for different parameter types + if self.parameter_type == "numeric": + return float(value) return value diff --git a/forecastmanager/forms.py b/forecastmanager/forms.py index c7ccdd7..2592056 100644 --- a/forecastmanager/forms.py +++ b/forecastmanager/forms.py @@ -97,7 +97,7 @@ def clean(self): # check parameters for param, value in params_data.items(): - if value: + if value is not None and value != "": param = ForecastDataParameters.objects.filter(name=param).first() if not param: self.add_error(None, f"Unknown parameter found in table data: {param}") diff --git a/forecastmanager/management/commands/clear_old_forecasts.py b/forecastmanager/management/commands/clear_old_forecasts.py new file mode 100644 index 0000000..fb5dd1a --- /dev/null +++ b/forecastmanager/management/commands/clear_old_forecasts.py @@ -0,0 +1,30 @@ +import logging + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from forecastmanager.models import Forecast + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Clear old forecasts from the database.' + + def handle(self, *args, **options): + logger.info("Clearing old forecasts from the database...") + + current_time = timezone.localtime() + + # Get all forecasts that are older than the current time + old_forecasts = Forecast.objects.filter(forecast_date__lt=current_time) + + if not old_forecasts: + logger.info("No old forecasts found.") + return + + # Delete the old forecasts + logger.info(f"Deleting {old_forecasts.count()} old forecasts...") + old_forecasts.delete() + + logger.info("Old forecasts deleted successfully.") diff --git a/forecastmanager/management/commands/generate_forecast.py b/forecastmanager/management/commands/generate_forecast.py index 1b8b4a8..23b8692 100644 --- a/forecastmanager/management/commands/generate_forecast.py +++ b/forecastmanager/management/commands/generate_forecast.py @@ -3,16 +3,42 @@ import requests from dateutil.parser import parse from django.core.management.base import BaseCommand +from django.utils import timezone from wagtail.models import Site -from forecastmanager.forecast_settings import ForecastSetting, WeatherCondition, ForecastPeriod, ForecastDataParameters -from forecastmanager.models import City, CityForecast, DataValue, Forecast from forecastmanager.constants import WEATHER_CONDITIONS_AS_DICT +from forecastmanager.forecast_settings import ( + ForecastSetting, + WeatherCondition, + ForecastPeriod, + ForecastDataParameters +) +from forecastmanager.models import ( + City, + CityForecast, + DataValue, + Forecast +) logger = logging.getLogger(__name__) # Define the base URL for the Met Norway API -BASE_URL = "https://www.yr.no/api/v0/locations" +BASE_URL = "https://api.met.no/weatherapi/locationforecast/2.0/complete" + +DEFAULT_INSTANT_DATA_PARAMETERS = [ + {"parameter": "air_pressure_at_sea_level", "name": "Air Pressure (Sea level)", "parameter_unit": "hPa"}, + {"parameter": "air_temperature", "name": "Minimum Air Temperature", "parameter_unit": "°C"}, + {"parameter": "wind_speed", "name": "Wind Speed", "parameter_unit": "m/s"}, + {"parameter": "wind_from_direction", "name": "Wind Direction ", "parameter_unit": "degrees"} +] + +DEFAULT_NEXT_HOURS_DATA_PARAMETERS = [ + {"parameter": "air_temperature_min", "name": "Minimum Air Temperature", "parameter_unit": "°C"}, + {"parameter": "air_temperature_max", "name": "Maximum Air Temperature", "parameter_unit": "°C"}, + {"parameter": "precipitation_amount", "name": "Precipitation Amount", "parameter_unit": "mm"}, +] + +DEFAULT_PARAMETERS = DEFAULT_INSTANT_DATA_PARAMETERS + DEFAULT_NEXT_HOURS_DATA_PARAMETERS class Command(BaseCommand): @@ -20,7 +46,7 @@ class Command(BaseCommand): 'for all cities in the database and save it to the database.') def handle(self, *args, **options): - print("Getting 7 Day Forecast from Yr.no...") + logger.info("Getting 7 Day Forecast from Yr.no...") cities = City.objects.all() if not cities: @@ -35,7 +61,13 @@ def handle(self, *args, **options): forecast_setting = ForecastSetting.for_site(site) - user_agent = f"{site.site_name} (WMO NMHSs Website Template) {site.root_url}" + site_name = site.site_name + root_url = site.root_url + + user_agent = f"ClimWeb {root_url}" + if site_name: + user_agent = f"{site_name}/{user_agent}" + user_agent = user_agent.strip() if not forecast_setting.enable_auto_forecast: @@ -46,83 +78,60 @@ def handle(self, *args, **options): conditions_by_symbol = {condition.symbol: condition for condition in conditions} parameters = forecast_setting.data_parameters.all() + if not parameters.exists(): # create default forecast parameters - default_parameters = [ - {"parameter": "air_temperature_max", "name": "Maximum Air Temperature", "parameter_unit": "°C"}, - {"parameter": "air_temperature_min", "name": "Minimum Air Temperature", "parameter_unit": "°C"}, - {"parameter": "wind_speed", "name": "Wind Speed", "parameter_unit": "m/s"}, - {"parameter": "precipitation_amount", "name": "Precipitation Amount", "parameter_unit": "mm"} - ] - - for default_parameter in default_parameters: + for default_parameter in DEFAULT_PARAMETERS: ForecastDataParameters.objects.create(parent=forecast_setting, **default_parameter) parameters = forecast_setting.data_parameters.all() parameters_dict = {parameter.parameter: parameter for parameter in parameters} - forecast_periods = forecast_setting.periods.all() - if not forecast_periods.exists(): - # create default forecast period - ForecastPeriod.objects.create(parent=forecast_setting, - label="Whole Day", - forecast_effective_time="00:00:00", - default=True) - - forecast_period = forecast_periods.filter(default=True).first() - if not forecast_period: - # pick first one if no default - forecast_period = forecast_periods.first() - cities_data = {} for city in cities: - print(f"Getting forecast for {city.name}...") + logger.info(f"Getting forecast for {city.name}...") lon = city.x lat = city.y - url = f"{BASE_URL}/{lat},{lon}/forecast" + url = f"{BASE_URL}?lat={lat}&lon={lon}" # Send a GET request to the API response = requests.get(url, headers={"User-Agent": user_agent}) if response.status_code >= 400: - logger.error( - f"Failed to get forecast for {city.name}. Status code: {response.status_code}") + logger.error(f"Failed to get forecast for {city.name}. Status code: {response.status_code}") continue # Get the weather data from the response data = response.json() - day_intervals = data['dayIntervals'] + # Get the timeseries data from the response + timeseries = data.get('properties', {}).get('timeseries') - for day in day_intervals[:8]: - date = parse(day.get("start")) - condition = day.get("twentyFourHourSymbol") - temperature_data = day.get("temperature") - air_temperature_max = temperature_data.get("max") - air_temperature_min = temperature_data.get("min") + # Get the first and last datetime for the forecast + first_datetime = timezone.localtime().replace(hour=0, minute=0, second=0, microsecond=0) + max_days = 6 + last_datetime = (first_datetime + timezone.timedelta(days=max_days)).replace(hour=23, minute=59, second=59) - wind_data = day.get("wind") - wind_speed = wind_data.get("max") + # Create a forecast for the city + for time_data in timeseries: + time = time_data.get("time") + utc_date = parse(time) + timezone_date = timezone.localtime(utc_date) - precipitation_data = day.get("precipitation") - precipitation = precipitation_data.get("value") + # Check if the forecast is within the next 7 days + if timezone_date < first_datetime or timezone_date > last_datetime: + continue - data_values = { - "date": date, - "condition": condition, - "parameters": { - "air_temperature_max": air_temperature_max, - "air_temperature_min": air_temperature_min, - "wind_speed": wind_speed, - "precipitation_amount": precipitation - } - } + data_values = time_data.get("data", {}) - condition = data_values.get('condition') + # Get the weather condition for the forecast + condition = data_values.get("next_1_hours", {}).get("summary", {}).get("symbol_code") + if condition is None: + condition = data_values.get("next_6_hours", {}).get("summary", {}).get("symbol_code") if conditions_by_symbol.get(condition) is None: condition_info = WEATHER_CONDITIONS_AS_DICT.get(condition) @@ -142,7 +151,10 @@ def handle(self, *args, **options): continue city_forecast = CityForecast(city=city, condition=condition_obj) - for key, value in data_values.get("parameters", {}).items(): + + instant_data = data_values.get("instant", {}).get("details", {}) + # Add the instant data values to the forecast + for key, value in instant_data.items(): if parameters_dict.get(key) is None: continue @@ -150,17 +162,47 @@ def handle(self, *args, **options): data_value = DataValue(parameter=parameter, value=value) city_forecast.data_values.add(data_value) - if date in cities_data: - cities_data[date].append(city_forecast) + # Add the next hours data values to the forecast + for param in DEFAULT_NEXT_HOURS_DATA_PARAMETERS: + param_key = param.get("parameter") + if parameters_dict.get(param_key) is None: + continue + + next_1_hours_data = data_values.get("next_1_hours", {}).get("details", {}) + next_6_hours_data = data_values.get("next_6_hours", {}).get("details", {}) + + next_data_value = None + if param_key in next_1_hours_data: + next_data_value = DataValue(parameter=parameters_dict[param_key], + value=next_1_hours_data[param_key]) + elif param_key in next_6_hours_data: + next_data_value = DataValue(parameter=parameters_dict[param_key], + value=next_6_hours_data[param_key]) + + if next_data_value is not None: + city_forecast.data_values.add(next_data_value) + + # Add the forecast to the cities data + if timezone_date in cities_data: + cities_data[timezone_date].append(city_forecast) else: - cities_data[date] = [city_forecast] + cities_data[timezone_date] = [city_forecast] + + # Create the forecast for the cities + for forecast_date, city_forecasts in cities_data.items(): + effective_time = f"{forecast_date.hour}:00" + + forecast_period = ForecastPeriod.objects.filter(forecast_effective_time=effective_time).first() + if forecast_period is None: + forecast_period = ForecastPeriod.objects.create(parent=forecast_setting, + forecast_effective_time=effective_time, + label=effective_time) - for time, city_forecasts in cities_data.items(): - forecast = Forecast.objects.filter(forecast_date=time, effective_period=forecast_period) + forecast = Forecast.objects.filter(forecast_date=forecast_date, effective_period=forecast_period) if forecast.exists(): forecast.delete() - forecast = Forecast(forecast_date=time, effective_period=forecast_period, source="yr") + forecast = Forecast(forecast_date=forecast_date, effective_period=forecast_period, source="yr") for city_forecast in city_forecasts: forecast.city_forecasts.add(city_forecast) diff --git a/forecastmanager/migrations/0025_alter_forecast_options_alter_forecastperiod_options.py b/forecastmanager/migrations/0025_alter_forecast_options_alter_forecastperiod_options.py new file mode 100644 index 0000000..2d82dda --- /dev/null +++ b/forecastmanager/migrations/0025_alter_forecast_options_alter_forecastperiod_options.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.7 on 2024-06-06 06:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('forecastmanager', '0024_alter_forecast_options_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='forecast', + options={'ordering': ['forecast_date', 'effective_period'], 'verbose_name': 'Forecast', 'verbose_name_plural': 'Forecasts'}, + ), + migrations.AlterModelOptions( + name='forecastperiod', + options={'ordering': ['forecast_effective_time']}, + ), + ] diff --git a/forecastmanager/migrations/0026_alter_forecastperiod_unique_together_and_more.py b/forecastmanager/migrations/0026_alter_forecastperiod_unique_together_and_more.py new file mode 100644 index 0000000..7f6abdd --- /dev/null +++ b/forecastmanager/migrations/0026_alter_forecastperiod_unique_together_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.3 on 2024-06-13 09:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('forecastmanager', '0025_alter_forecast_options_alter_forecastperiod_options'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='forecastperiod', + unique_together=set(), + ), + migrations.AlterField( + model_name='forecastperiod', + name='forecast_effective_time', + field=models.TimeField(unique=True, verbose_name='Forecast Effective Time'), + ), + migrations.RemoveField( + model_name='forecastperiod', + name='default', + ), + ] diff --git a/forecastmanager/models.py b/forecastmanager/models.py index 0af01e4..a65f4e0 100644 --- a/forecastmanager/models.py +++ b/forecastmanager/models.py @@ -1,6 +1,9 @@ import uuid +from datetime import datetime from django.contrib.gis.db import models +from django.utils import timezone +from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ from django_extensions.db.fields import AutoSlugField from modelcluster.fields import ParentalKey @@ -73,7 +76,7 @@ class Meta: unique_together = ("forecast_date", "effective_period") verbose_name = _("Forecast") verbose_name_plural = _("Forecasts") - ordering = ["-forecast_date", "effective_period"] + ordering = ["forecast_date", "effective_period"] panels = [ FieldPanel("forecast_date"), @@ -85,6 +88,10 @@ class Meta: def __str__(self): return f"{self.forecast_date} - {self.effective_period.label}" + @property + def datetime(self): + return datetime.combine(self.forecast_date, self.effective_period.forecast_effective_time) + def get_geojson(self, request=None): features = [] for city_forecast in self.city_forecasts.all(): @@ -92,6 +99,7 @@ def get_geojson(self, request=None): return { "type": "FeatureCollection", "date": self.forecast_date, + "datetime": self.datetime, "features": features, } @@ -122,6 +130,12 @@ def forecast_date(self): def effective_period(self): return self.parent.effective_period + @cached_property + def datetime(self): + effective_period_time = self.effective_period.forecast_effective_time + date = datetime.combine(self.forecast_date, effective_period_time) + return timezone.make_aware(date) + @property def data_values_dict(self): data_values = {} @@ -142,26 +156,6 @@ def data_values_dict(self): data_values[data_value.parameter.parameter] = val - # Group temperature values - temperature = {} - if "air_temperature_max" in data_values: - temperature["max_temp"] = data_values.get("air_temperature_max") - # remove air_temperature_max from data_values - data_values.pop("air_temperature_max") - - if "air_temperature_min" in data_values: - temperature["min_temp"] = data_values.get("air_temperature_min") - # remove air_temperature_max from data_values - data_values.pop("air_temperature_min") - - if "air_temperature" in data_values: - temperature["temp"] = data_values.get("air_temperature") - # remove air_temperature_max from data_values - data_values.pop("air_temperature") - - if temperature: - data_values["temperature"] = temperature - return data_values def get_geojson_feature(self, request=None): @@ -174,7 +168,7 @@ def get_geojson_feature(self, request=None): "coordinates": self.city.coordinates, }, "properties": { - "date": self.parent.forecast_date, + "date": self.parent.datetime, "effective_period_time": self.parent.effective_period.forecast_effective_time, "effective_period_label": self.parent.effective_period.label, "city": self.city.name, @@ -208,7 +202,7 @@ def parsed_value(self): @property def value_with_units(self): - if not self.parsed_value: + if self.parsed_value is None: return None if not self.parameter.units: diff --git a/forecastmanager/views.py b/forecastmanager/views.py index 0b17435..1cc2b45 100644 --- a/forecastmanager/views.py +++ b/forecastmanager/views.py @@ -1,11 +1,11 @@ import csv -from datetime import date from django.contrib.gis.geos import Point from django.http import HttpResponse from django.http import JsonResponse from django.shortcuts import render, redirect from django.urls import reverse +from django.utils import timezone from django_filters.rest_framework import DjangoFilterBackend from rest_framework.generics import ListAPIView from rest_framework.permissions import BasePermission, IsAuthenticated, SAFE_METHODS @@ -45,29 +45,8 @@ class ForecastListView(ListAPIView): def get_queryset(self): queryset = super().get_queryset() - forecast_date = self.request.query_params.get('forecast_date') - - start_date = self.request.query_params.get('start_date') - end_date = self.request.query_params.get('end_date') - effective_time = self.request.query_params.get('effective_time') - - if effective_time: - queryset = queryset.filter(effective_period__forecast_effective_time=effective_time) - else: - queryset = queryset.filter(effective_period__default=True) - - if start_date: - queryset = queryset.filter(forecast_date__gte=start_date) - else: - queryset = queryset.filter(forecast_date__gte=date.today()) - - if end_date: - queryset = queryset.filter(forecast_date__lte=end_date) - - if forecast_date: - queryset = queryset.filter(forecast_date=forecast_date) - - return queryset.order_by('forecast_date', 'effective_period__forecast_effective_time') + queryset.filter(forecast_date__gte=timezone.localtime().date()) + return queryset def download_forecast_template(request): diff --git a/sandbox/sandbox/settings/base.py b/sandbox/sandbox/settings/base.py index dbfa18a..365c695 100644 --- a/sandbox/sandbox/settings/base.py +++ b/sandbox/sandbox/settings/base.py @@ -165,7 +165,7 @@ # ('am', 'Amharic'), ] -TIME_ZONE = "UTC" +TIME_ZONE = "Africa/Nairobi" USE_I18N = True @@ -212,3 +212,43 @@ # e.g. in notification emails. Don't include '/admin' or a trailing slash WAGTAILADMIN_BASE_URL = "http://example.com" # FORCE_SCRIPT_NAME='/cms' + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + # Send logs with at least INFO level to the console. + "console": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + }, + "formatters": { + "verbose": { + "format": "[%(asctime)s][%(process)d][%(levelname)s][%(name)s] %(message)s" + }, + }, + "loggers": { + "": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "wagtail": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "django.request": { + "handlers": ["console"], + "level": "WARNING", + "propagate": False, + }, + "django.security": { + "handlers": ["console"], + "level": "WARNING", + "propagate": False, + }, + }, +} diff --git a/setup.cfg b/setup.cfg index 9dcd8b3..7bc863d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = forecastmanager -version = 0.4.9 +version = 0.5.0 description = Integration of Weather City Forecasts Manager in Wagtail Projects. long_description = file:README.md long_description_content_type = text/markdown