Feature/import ski trail maintenance history

config_dev.env.example

@@ -159,8 +159,8 @@ ECO_COUNTER_LOG_LEVEL=
 MOBILITY_DATA_LOG_LEVEL=
 # Bicycle networks APP, default INFO
 BICYCLE_NETWORK_LOG_LEVEL=
-# Street maintenance, default INFO
-STREET_MAINTENANCE_LOG_LEVEL=
+# Maintenance
+MAINTENANCE_LOG_LEVEL=INFO
 
 # Settings needed for enabling Turku area:
 #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv
@@ -202,3 +202,7 @@ KUNTEC_KEY=
 # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py)
 # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api
 TELRAAM_TOKEN=
+# Optionally, define the urls, from where the ski trails and maintenance histories are retrieved from.
+# The default values are in the examples.
+# SKI_TRAILS_URL=https://api.paikannuspalvelu.fi/v1/public/track/?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson
+# SKI_TRAILS_MAINTENANCE_HISTORY_URL=https://api.paikannuspalvelu.fi/v1/public/location/lastvisit/?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson&max_distance=50

street_maintenance/README.md → maintenance/README.md

@@ -1,8 +1,9 @@
-# Street Maintenance history
+# Maintenance history
 
-Django app for importing and serving street maintenance data.
+Django app for importing, processing and serving maintenance data.
 
-## Importer
+## Street maintenance history
+### Importer
 Name:
 import_street_maintenance_history 
 
@@ -32,12 +33,24 @@ Note, only the MaintenanceWorks and MaintenanceUnits for the given provider from
 To periodically import data use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#street-maintenance-history-street_maintenancetasksimport_street_maintenance_history).
 
 
-## Deleting street maintenance history for a provider
+### Deleting street maintenance history for a provider
 It is possible to delete street maintenance history for a provider.
 e.g., to delete all street maintenance history for provider 'destia':
 ```
 ./manage.py delete_street_maintenance_history destia
 ```
 
-## API
-See: specificatin.swagger.yaml
+## Ski trails
+To periodically import ski trails maintenance history use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#unit-maintenance).
+
+### Ski trails
+Before importing the ski trails maintenance history the ski trails must be imported. To import type:
+```
+./manage.py import_ski_trails
+```
+
+### Ski trails maintenance history
+To import the maintenance history, type:
+```
+./manage.py import_ski_trails_maintenance_history
+```

maintenance/admin.py

@@ -0,0 +1,13 @@
+from django.contrib import admin
+
+from maintenance.models import (
+    MaintenanceUnit,
+    MaintenanceWork,
+    UnitMaintenance,
+    UnitMaintenanceGeometry,
+)
+
+admin.site.register(MaintenanceWork)
+admin.site.register(MaintenanceUnit)
+admin.site.register(UnitMaintenance)
+admin.site.register(UnitMaintenanceGeometry)

street_maintenance/api/serializers.py → maintenance/api/serializers.py

@@ -3,7 +3,46 @@
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
-from street_maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork
+from maintenance.models import (
+    GeometryHistory,
+    MaintenanceUnit,
+    MaintenanceWork,
+    UnitMaintenance,
+    UnitMaintenanceGeometry,
+)
+
+
+class UnitMaintenanceGeometrySerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = UnitMaintenanceGeometry
+        fields = ["id", "geometry", "geometry_id", "unit_maintenance"]
+
+    def to_representation(self, instance):
+        ret = super().to_representation(instance)
+        # If nested in UnitMaintenanceSerializer
+        if (
+            self.context.get("request", False)
+            and "/maintenance/unit_maintenance/" in self.context.get("request", "").path
+        ):
+            ret.pop("unit_maintenance", None)
+        return ret
+
+
+class UnitMaintenanceSerializer(serializers.ModelSerializer):
+    geometries = UnitMaintenanceGeometrySerializer(many=True, read_only=True)
+
+    class Meta:
+        model = UnitMaintenance
+        fields = [
+            "id",
+            "unit",
+            "target",
+            "condition",
+            "maintained_at",
+            "last_imported_time",
+            "geometries",
+        ]
 
 
 class GeometryHistorySerializer(serializers.ModelSerializer):

maintenance/api/urls.py

@@ -0,0 +1,30 @@
+from django.urls import include, path
+from rest_framework import routers
+
+from . import views
+
+app_name = "maintenance"
+
+router = routers.DefaultRouter()
+router.register("active_events", views.ActiveEventsViewSet, basename="active_events")
+
+router.register(
+    "maintenance_works", views.MaintenanceWorkViewSet, basename="maintenance_works"
+)
+router.register(
+    "maintenance_units", views.MaintenanceUnitViewSet, basename="maintenance_units"
+)
+router.register(
+    "geometry_history", views.GeometryHitoryViewSet, basename="geometry_history"
+)
+router.register(
+    "unit_maintenance", views.UnitMaintenanceViewSet, basename="unit_maintenance"
+)
+router.register(
+    "unit_maintenance_geometry",
+    views.UnitMaintenanceGeometryViewSet,
+    basename="unit_maintenance_geometry",
+)
+urlpatterns = [
+    path("", include(router.urls), name="maintenance"),
+]

street_maintenance/api/urls.py → maintenance/api/urls_street_maintenance.py

@@ -1,3 +1,7 @@
+"""
+Keep backward compatibility to street_maintenance API.
+"""
+
 from django.urls import include, path
 from rest_framework import routers
 
@@ -20,6 +24,5 @@
 )
 
 urlpatterns = [
-    # re_path("^street_maintenance/active_events", )
     path("", include(router.urls), name="street_maintenance"),
 ]

street_maintenance/api/views.py → maintenance/api/views.py

@@ -1,25 +1,35 @@
 from datetime import datetime
 
+import django_filters
 from django.utils.decorators import method_decorator
 from django.utils.timezone import make_aware
 from django.views.decorators.cache import cache_page
+from django_filters.rest_framework import DjangoFilterBackend
 from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter
 from rest_framework import mixins, viewsets
 from rest_framework.exceptions import ParseError
 from rest_framework.pagination import PageNumberPagination
 
-from street_maintenance.api.serializers import (
+from maintenance.api.serializers import (
     ActiveEventSerializer,
     GeometryHistorySerializer,
     MaintenanceUnitSerializer,
     MaintenanceWorkSerializer,
+    UnitMaintenanceGeometrySerializer,
+    UnitMaintenanceSerializer,
 )
-from street_maintenance.management.commands.constants import (
+from maintenance.management.commands.constants import (
     EVENT_CHOICES,
     PROVIDERS,
     START_DATE_TIME_FORMAT,
 )
-from street_maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork
+from maintenance.models import (
+    GeometryHistory,
+    MaintenanceUnit,
+    MaintenanceWork,
+    UnitMaintenance,
+    UnitMaintenanceGeometry,
+)
 
 EXAMPLE_TIME_FORMAT = "YYYY-MM-DD HH:MM:SS"
 EXAMPLE_TIME = "2022-09-18 10:00:00"
@@ -64,6 +74,37 @@ class LargeResultsSetPagination(PageNumberPagination):
     max_page_size = 200_000
 
 
+class UnitMaintenaceFilterSet(django_filters.FilterSet):
+    maintained_at__gte = django_filters.DateTimeFilter(
+        method="filter_maintained_at__gte"
+    )
+    maintained_at__lte = django_filters.DateTimeFilter(
+        method="filter_maintained_at__lte"
+    )
+
+    class Meta:
+        model = UnitMaintenance
+        fields = {"target": ["iexact"], "unit": ["exact"]}
+
+    def filter_maintained_at__gte(self, queryset, fields, maintained_at):
+        return queryset.filter(maintained_at__gte=maintained_at)
+
+    def filter_maintained_at__lte(self, queryset, fields, maintained_at):
+        return queryset.filter(maintained_at__lte=maintained_at)
+
+
+class UnitMaintenanceViewSet(viewsets.ReadOnlyModelViewSet):
+    queryset = UnitMaintenance.objects.all()
+    serializer_class = UnitMaintenanceSerializer
+    filter_backends = [DjangoFilterBackend]
+    filterset_class = UnitMaintenaceFilterSet
+
+
+class UnitMaintenanceGeometryViewSet(viewsets.ReadOnlyModelViewSet):
+    queryset = UnitMaintenanceGeometry.objects.all()
+    serializer_class = UnitMaintenanceGeometrySerializer
+
+
 class ActiveEventsViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
     queryset = MaintenanceWork.objects.order_by().distinct("events")
     serializer_class = ActiveEventSerializer
@@ -133,7 +174,7 @@ def list(self, request):
     list=extend_schema(
         description="MaintananceUnit objets are the entities that creates the MaintenanceWorks. Every MaintenanceWork "
         "has a relation to a MaintenanceUnit. The type of the MaintenanceUnit can vary depending on the provider. It "
-        "can be a machine or a event",
+        "can be a machine or an event.",
     ),
 )
 class MaintenanceUnitViewSet(viewsets.ReadOnlyModelViewSet):

maintenance/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class MaintenanceConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "maintenance"

street_maintenance/management/commands/constants.py → maintenance/management/commands/constants.py

@@ -191,3 +191,5 @@
     KUNTEC: {HISTORY_SIZE: KUNTEC_DEFAULT_WORKS_HISTORY_SIZE},
     YIT: {HISTORY_SIZE: YIT_DEFAULT_WORKS_HISTORY_SIZE},
 }
+
+SKI_TRAILS_DATE_FIELD_FORMAT = "%Y-%m-%d %H:%M"

street_maintenance/management/commands/delete_street_maintenance_history.py → maintenance/management/commands/delete_street_maintenance_history.py

@@ -2,11 +2,11 @@
 
 from django.core.management import BaseCommand
 
-from street_maintenance.models import GeometryHistory, MaintenanceUnit
+from maintenance.models import GeometryHistory, MaintenanceUnit
 
 from .constants import PROVIDERS
 
-logger = logging.getLogger("mobility_data")
+logger = logging.getLogger("maintenance")
 
 # Add deprecated provider name 'AUTORI'
 PROVIDERS.append("AUTORI")

maintenance/management/commands/import_ski_trails.py

@@ -0,0 +1,74 @@
+import logging
+
+from django.conf import settings
+from django.contrib.gis.geos import GEOSGeometry
+from django.contrib.gis.geos.error import GEOSException
+from django.core.management.base import BaseCommand
+from django.db.utils import IntegrityError
+
+from maintenance.models import UnitMaintenance, UnitMaintenanceGeometry
+
+from .utils import get_data_layer
+
+logger = logging.getLogger(__name__)
+
+
+def save_trails(layer):
+    num_created = 0
+    num_updated = 0
+    for feature in layer:
+        is_created = False
+        is_updated = False
+        try:
+            geometry = None
+
+            try:
+                geometry = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid)
+            except GEOSException:
+                logger.error(f"Invalid geometry {feature.geom.wkt}, skipping...")
+                continue
+
+            geometry_id = feature["construction_point_id"].as_int()
+            filter = {"geometry_id": geometry_id}
+            queryset = UnitMaintenanceGeometry.objects.filter(**filter)
+
+            if queryset.count() == 0:
+                unit_maintenance_geometry = UnitMaintenanceGeometry(**filter)
+                unit_maintenance_geometry.geometry = geometry
+                is_created = True
+            else:
+                unit_maintenance_geometry = queryset.first()
+                if not unit_maintenance_geometry.geometry.equals(geometry):
+                    unit_maintenance_geometry.geometry = geometry
+                    is_updated = True
+
+            try:
+                unit_maintenance_geometry.save()
+            except IntegrityError:
+                logger.error(f"geometry id {geometry.geometry_id} exists, skipping...")
+
+        except Exception as exp:
+            logger.error(f"Could not save ski trail {feature}, reason {exp}")
+        num_created = num_created + 1 if is_created else num_created
+        num_updated = num_updated + 1 if is_updated else num_updated
+    return num_created, num_updated
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--delete",
+            action="store_true",
+            help="Delete all ski trails that have a unit maintenance relationship, before importing.",
+        )
+
+    def handle(self, *args, **options):
+        if options.get("delete", False):
+            UnitMaintenanceGeometry.objects.filter(
+                unit_maintenance__target=UnitMaintenance.SKI_TRAIL
+            ).delete()
+
+        layer = get_data_layer(settings.SKI_TRAILS_URL)
+        num_created, num_updated = save_trails(layer)
+        logger.info(f"Created {num_created}, updated {num_updated} ski trails.")