From 4eeb5018aa8f420568544bb93da2ec500c72f6db Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:12:02 +0300 Subject: [PATCH 01/66] Copied street maintenance model --- maintenance/models.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 maintenance/models.py diff --git a/maintenance/models.py b/maintenance/models.py new file mode 100644 index 000000000..3591c3117 --- /dev/null +++ b/maintenance/models.py @@ -0,0 +1,49 @@ +from django.contrib.gis.db import models +from django.contrib.postgres.fields import ArrayField + +from street_maintenance.management.commands.constants import PROVIDER_CHOICES + +DEFAULT_SRID = 4326 + + +class MaintenanceUnit(models.Model): + + unit_id = models.CharField(max_length=64, null=True) + provider = models.CharField(max_length=16, choices=PROVIDER_CHOICES, null=True) + names = ArrayField(models.CharField(max_length=64), default=list) + + def __str__(self): + return "%s" % (self.unit_id) + + +class MaintenanceWork(models.Model): + geometry = models.GeometryField(srid=DEFAULT_SRID, null=True) + events = ArrayField(models.CharField(max_length=64), default=list) + original_event_names = ArrayField(models.CharField(max_length=64), default=list) + timestamp = models.DateTimeField() + maintenance_unit = models.ForeignKey( + "MaintenanceUnit", + on_delete=models.CASCADE, + related_name="maintenance_work", + null=True, + ) + + def __str__(self): + return "%s %s" % (self.timestamp, self.events) + + class Meta: + ordering = ["-timestamp"] + + +class GeometryHistory(models.Model): + timestamp = models.DateTimeField() + geometry = models.GeometryField(srid=DEFAULT_SRID, null=True) + coordinates = ArrayField(ArrayField(models.FloatField()), default=list) + events = ArrayField(models.CharField(max_length=64), default=list) + provider = models.CharField(max_length=16, choices=PROVIDER_CHOICES, null=True) + + def __str__(self): + return "%s %s" % (self.timestamp, self.events) + + class Meta: + ordering = ["-timestamp"] From 6b8cf194cfaf6f992025b13b0c93b7b54420cfd2 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 17 Sep 2024 10:35:14 +0300 Subject: [PATCH 02/66] Add maintenance to installed apps --- smbackend/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 3fa06f3cb..7d5110859 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -134,6 +134,7 @@ "bicycle_network.apps.BicycleNetworkConfig", "iot.apps.IotConfig", "street_maintenance.apps.StreetMaintenanceConfig", + "maintenance.apps.MaintenanceConfig", "environment_data.apps.EnvironmentDataConfig", "exceptional_situations.apps.ExceptionalSituationsConfig", ] From 8bf546bf7286dccdb8fcf342184b737f600745a7 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:20:20 +0300 Subject: [PATCH 03/66] Delete as move to maintenace application --- .../delete_street_maintenance_history.py | 61 -------- .../import_street_maintenance_history.py | 132 ------------------ 2 files changed, 193 deletions(-) delete mode 100644 street_maintenance/management/commands/delete_street_maintenance_history.py delete mode 100644 street_maintenance/management/commands/import_street_maintenance_history.py diff --git a/street_maintenance/management/commands/delete_street_maintenance_history.py b/street_maintenance/management/commands/delete_street_maintenance_history.py deleted file mode 100644 index ec91b0193..000000000 --- a/street_maintenance/management/commands/delete_street_maintenance_history.py +++ /dev/null @@ -1,61 +0,0 @@ -import logging - -from django.core.management import BaseCommand - -from street_maintenance.models import GeometryHistory, MaintenanceUnit - -from .constants import PROVIDERS - -logger = logging.getLogger("mobility_data") - -# Add deprecated provider name 'AUTORI' -PROVIDERS.append("AUTORI") - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument( - "providers", - type=str, - nargs="*", - help=", ".join(PROVIDERS), - ) - - def handle(self, *args, **options): - providers = [p.upper() for p in options.get("providers", None)] - - for provider in providers: - if provider not in PROVIDERS: - logger.error( - f"Invalid providers argument {provider}, choices are: {', '.join(PROVIDERS)}" - ) - continue - - logger.info(f"Deleting street maintenance history for {provider}.") - provider = provider.upper() - deleted_units = MaintenanceUnit.objects.filter(provider=provider).delete() - deleted_histories = GeometryHistory.objects.filter( - provider=provider - ).delete() - if "street_maintenance.MaintenanceUnit" in deleted_units[1]: - num_deleted_units = deleted_units[1][ - "street_maintenance.MaintenanceUnit" - ] - else: - num_deleted_units = 0 - if "street_maintenance.MaintenanceWork" in deleted_units[1]: - num_deleted_works = deleted_units[1][ - "street_maintenance.MaintenanceWork" - ] - else: - num_deleted_works = 0 - if "street_maintenance.GeometryHistory" in deleted_histories[1]: - num_deleted_histories = deleted_histories[1][ - "street_maintenance.GeometryHistory" - ] - else: - num_deleted_histories = 0 - - logger.info(f"GeometryHistorys deleted {num_deleted_histories}.") - logger.info(f"MaintenanceUnits deleted {num_deleted_units}.") - logger.info(f"MaintenanceWorks deleted {num_deleted_works}.") diff --git a/street_maintenance/management/commands/import_street_maintenance_history.py b/street_maintenance/management/commands/import_street_maintenance_history.py deleted file mode 100644 index 2512840f1..000000000 --- a/street_maintenance/management/commands/import_street_maintenance_history.py +++ /dev/null @@ -1,132 +0,0 @@ -import logging -from datetime import datetime - -from django.core.management import BaseCommand - -from street_maintenance.models import MaintenanceUnit, MaintenanceWork - -from .constants import ( - FETCH_SIZE, - HISTORY_SIZE, - HISTORY_SIZES, - PROVIDER_TYPES, - PROVIDERS, -) -from .utils import ( - create_kuntec_maintenance_units, - create_kuntec_maintenance_works, - create_maintenance_units, - create_maintenance_works, - create_yit_maintenance_units, - create_yit_maintenance_works, - get_yit_access_token, - precalculate_geometry_history, -) - -logger = logging.getLogger("street_maintenance") - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument( - "--fetch-size", - type=int, - nargs="+", - default=False, - help=("Max number of location history items to fetch per unit."), - ) - parser.add_argument( - "--history-size", - type=int, - nargs="+", - default=False, - help=("History size in days."), - ) - parser.add_argument( - "--providers", - type=str, - nargs="+", - default=False, - help=", ".join(PROVIDERS), - ) - - def handle(self, *args, **options): - history_size = None - fetch_size = None - - if options["history_size"]: - history_size = options["history_size"] - history_size = ( - history_size[0] if type(history_size) == list else history_size - ) - if options["fetch_size"]: - fetch_size = options["fetch_size"] - fetch_size = fetch_size[0] if type(fetch_size) == list else fetch_size - - providers = [p.upper() for p in options.get("providers", None)] - for provider in providers: - if provider not in PROVIDERS: - logger.warning( - f"Provider {provider} not defined, choices are {', '.join(PROVIDERS)}" - ) - continue - start_time = datetime.now() - history_size = ( - history_size if history_size else HISTORY_SIZES[provider][HISTORY_SIZE] - ) - fetch_size = ( - fetch_size - if fetch_size - else HISTORY_SIZES[provider].get(FETCH_SIZE, None) - ) - match provider.upper(): - case PROVIDER_TYPES.DESTIA | PROVIDER_TYPES.INFRAROAD: - num_created_units, num_del_units = create_maintenance_units( - provider - ) - num_created_works, num_del_works = create_maintenance_works( - provider, history_size, fetch_size - ) - case PROVIDER_TYPES.KUNTEC: - num_created_units, num_del_units = create_kuntec_maintenance_units() - num_created_works, num_del_works = create_kuntec_maintenance_works( - history_size - ) - - case PROVIDER_TYPES.YIT: - access_token = get_yit_access_token() - num_created_units, num_del_units = create_yit_maintenance_units( - access_token - ) - num_created_works, num_del_works = create_yit_maintenance_works( - access_token, history_size - ) - - tot_num_units = MaintenanceUnit.objects.filter(provider=provider).count() - tot_num_works = MaintenanceWork.objects.filter( - maintenance_unit__provider=provider - ).count() - logger.info( - f"Deleted {num_del_units} obsolete Units for provider {provider}" - ) - logger.info( - f"Created {num_created_units} units of total {tot_num_units} units for provider {provider}" - ) - logger.info( - f"Deleted {num_del_works} obsolete Works for provider {provider}" - ) - logger.info( - f"Created {num_created_works} Works of total {tot_num_works} Works for provider {provider}" - ) - - if num_created_works > 0: - precalculate_geometry_history(provider) - else: - logger.warning( - f"No works created for {provider}, skipping geometry history population." - ) - end_time = datetime.now() - duration = end_time - start_time - logger.info( - f"Imported {provider} street maintenance history in: {duration}" - ) From 7818d819eca3986d789e1486cc83d545b83698a6 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:35:56 +0300 Subject: [PATCH 04/66] Fix import --- maintenance/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintenance/models.py b/maintenance/models.py index 3591c3117..ec4f2fc1a 100644 --- a/maintenance/models.py +++ b/maintenance/models.py @@ -1,7 +1,7 @@ from django.contrib.gis.db import models from django.contrib.postgres.fields import ArrayField -from street_maintenance.management.commands.constants import PROVIDER_CHOICES +from maintenance.management.commands.constants import PROVIDER_CHOICES DEFAULT_SRID = 4326 From 14e1c53dac25b9b92eb803872c0f21301f3ca5c7 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:57:12 +0300 Subject: [PATCH 05/66] Delete, moved to maintenance app --- street_maintenance/tests/__init__.py | 0 street_maintenance/tests/conftest.py | 103 ------ street_maintenance/tests/test_api.py | 69 ---- street_maintenance/tests/test_importers.py | 275 ---------------- street_maintenance/tests/utils.py | 360 --------------------- 5 files changed, 807 deletions(-) delete mode 100644 street_maintenance/tests/__init__.py delete mode 100644 street_maintenance/tests/conftest.py delete mode 100644 street_maintenance/tests/test_api.py delete mode 100644 street_maintenance/tests/test_importers.py delete mode 100644 street_maintenance/tests/utils.py diff --git a/street_maintenance/tests/__init__.py b/street_maintenance/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/street_maintenance/tests/conftest.py b/street_maintenance/tests/conftest.py deleted file mode 100644 index cd4b5fdc2..000000000 --- a/street_maintenance/tests/conftest.py +++ /dev/null @@ -1,103 +0,0 @@ -from datetime import datetime, timedelta - -import pytest -import pytz -from django.contrib.gis.geos import GEOSGeometry, LineString -from munigeo.models import ( - AdministrativeDivision, - AdministrativeDivisionGeometry, - AdministrativeDivisionType, -) -from rest_framework.test import APIClient - -from mobility_data.tests.conftest import TURKU_WKT -from street_maintenance.management.commands.constants import ( - AURAUS, - INFRAROAD, - KUNTEC, - LIUKKAUDENTORJUNTA, -) -from street_maintenance.models import DEFAULT_SRID, GeometryHistory - -UTC_TIMEZONE = pytz.timezone("UTC") - - -@pytest.fixture -def api_client(): - return APIClient() - - -@pytest.mark.django_db -@pytest.fixture -def geometry_historys(): - geometry_historys = [] - now = datetime.now(UTC_TIMEZONE) - geometry = LineString((0, 0), (0, 50), (50, 50), (50, 0), (0, 0), sird=DEFAULT_SRID) - obj = GeometryHistory.objects.create( - timestamp=now, - geometry=geometry, - coordinates=geometry.coords, - provider=INFRAROAD, - events=[AURAUS], - ) - geometry_historys.append(obj) - obj = GeometryHistory.objects.create( - timestamp=now - timedelta(days=1), - geometry=geometry, - coordinates=geometry.coords, - provider=INFRAROAD, - events=[AURAUS], - ) - geometry_historys.append(obj) - obj = GeometryHistory.objects.create( - timestamp=now - timedelta(days=2), - geometry=geometry, - coordinates=geometry.coords, - provider=INFRAROAD, - events=[LIUKKAUDENTORJUNTA], - ) - geometry_historys.append(obj) - obj = GeometryHistory.objects.create( - timestamp=now - timedelta(days=1), - geometry=geometry, - coordinates=geometry.coords, - provider=KUNTEC, - events=[AURAUS], - ) - geometry_historys.append(obj) - obj = GeometryHistory.objects.create( - timestamp=now - timedelta(days=2), - geometry=geometry, - coordinates=geometry.coords, - provider=KUNTEC, - events=[AURAUS, LIUKKAUDENTORJUNTA], - ) - geometry_historys.append(obj) - - -@pytest.mark.django_db -@pytest.fixture -def administrative_division_type(): - adm_div_type = AdministrativeDivisionType.objects.create( - id=1, type="muni", name="Municipality" - ) - return adm_div_type - - -@pytest.mark.django_db -@pytest.fixture -def administrative_division(administrative_division_type): - adm_div = AdministrativeDivision.objects.get_or_create( - id=1, name="Turku", origin_id=853, type_id=1 - ) - return adm_div - - -@pytest.mark.django_db -@pytest.fixture -def administrative_division_geometry(administrative_division): - turku_multipoly = GEOSGeometry(TURKU_WKT, srid=3067) - adm_div_geom = AdministrativeDivisionGeometry.objects.create( - id=1, division_id=1, boundary=turku_multipoly - ) - return adm_div_geom diff --git a/street_maintenance/tests/test_api.py b/street_maintenance/tests/test_api.py deleted file mode 100644 index b56048572..000000000 --- a/street_maintenance/tests/test_api.py +++ /dev/null @@ -1,69 +0,0 @@ -from datetime import timedelta - -import pytest -from django.utils import timezone -from rest_framework.reverse import reverse - -from street_maintenance.management.commands.constants import ( - AURAUS, - INFRAROAD, - KUNTEC, - LIUKKAUDENTORJUNTA, - START_DATE_TIME_FORMAT, -) - - -@pytest.mark.django_db -def test_geometry_history_list(api_client, geometry_historys): - url = reverse("street_maintenance:geometry_history-list") - response = api_client.get(url) - assert response.json()["count"] == 5 - - -@pytest.mark.django_db -def test_geometry_history_list_provider_parameter(api_client, geometry_historys): - url = reverse("street_maintenance:geometry_history-list") + f"?provider={KUNTEC}" - response = api_client.get(url) - # Fixture data contains 2 KUNTEC GeometryHistroy rows - assert response.json()["count"] == 2 - - -@pytest.mark.django_db -def test_geometry_history_list_event_parameter(api_client, geometry_historys): - url = reverse("street_maintenance:geometry_history-list") + f"?event={AURAUS}" - response = api_client.get(url) - # 3 INFRAROAD AURAUS events and 1 KUNTEC - assert response.json()["count"] == 4 - - -@pytest.mark.django_db -def test_geometry_history_list_event_and_provider_parameter( - api_client, geometry_historys -): - url = ( - reverse("street_maintenance:geometry_history-list") - + f"?provider={KUNTEC}&event={LIUKKAUDENTORJUNTA}" - ) - response = api_client.get(url) - assert response.json()["count"] == 1 - - -@pytest.mark.django_db -def test_geometry_history_list_start_date_time_parameter(api_client, geometry_historys): - start_date_time = timezone.now() - timedelta(hours=1) - url = ( - reverse("street_maintenance:geometry_history-list") - + f"?start_date_time={start_date_time.strftime(START_DATE_TIME_FORMAT)}" - ) - response = api_client.get(url) - assert response.json()["count"] == 1 - geometry_history = response.json()["results"][0] - assert geometry_history["geometry_type"] == "LineString" - assert geometry_history["provider"] == INFRAROAD - start_date_time = timezone.now() - timedelta(days=1, hours=2) - url = ( - reverse("street_maintenance:geometry_history-list") - + f"?start_date_time={start_date_time.strftime(START_DATE_TIME_FORMAT)}" - ) - response = api_client.get(url) - assert response.json()["count"] == 3 diff --git a/street_maintenance/tests/test_importers.py b/street_maintenance/tests/test_importers.py deleted file mode 100644 index 888c3f026..000000000 --- a/street_maintenance/tests/test_importers.py +++ /dev/null @@ -1,275 +0,0 @@ -from unittest.mock import patch - -import pytest - -from street_maintenance.management.commands.constants import DESTIA, INFRAROAD -from street_maintenance.models import MaintenanceUnit, MaintenanceWork - -from .utils import ( - get_fluentprogress_units_mock_data, - get_fluentprogress_works_mock_data, - get_kuntec_units_mock_data, - get_kuntec_works_mock_data, - get_yit_contract_mock_data, - get_yit_event_types_mock_data, - get_yit_routes_mock_data, - get_yit_vehicles_mock_data, -) - - -@pytest.mark.django_db -@patch("street_maintenance.management.commands.utils.get_yit_vehicles") -def test_yit_units( - get_yit_vehicles_mock, - administrative_division, - administrative_division_geometry, -): - from street_maintenance.management.commands.utils import ( - create_yit_maintenance_units, - ) - - get_yit_vehicles_mock.return_value = get_yit_vehicles_mock_data(2) - num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") - assert MaintenanceUnit.objects.count() == 2 - assert num_created_units == 2 - assert num_del_units == 0 - unit = MaintenanceUnit.objects.first() - unit_id = unit.id - assert unit.names == ["Huoltoauto"] - assert unit.unit_id == "82260ff7-589e-4cee-a8e0-124b615381f1" - get_yit_vehicles_mock.return_value = get_yit_vehicles_mock_data(1) - num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") - assert unit_id == MaintenanceUnit.objects.first().id - assert num_created_units == 0 - assert num_del_units == 1 - assert MaintenanceUnit.objects.count() == 1 - # Test duplicate unit - unit_dup = MaintenanceUnit.objects.first() - unit_dup.pk = 42 - unit_dup.save() - num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") - assert num_created_units == 0 - assert num_del_units == 1 - - -@pytest.mark.django_db -@patch( - "street_maintenance.management.commands.utils.get_yit_vehicles", - return_value=get_yit_vehicles_mock_data(2), -) -@patch( - "street_maintenance.management.commands.utils.get_yit_contract", - return_value=get_yit_contract_mock_data(), -) -@patch( - "street_maintenance.management.commands.utils.get_yit_event_types", - return_value=get_yit_event_types_mock_data(), -) -@patch("street_maintenance.management.commands.utils.get_yit_routes") -def test_yit_works( - get_yit_routes_mock, - get_yit_vechiles_mock, - administrative_division, - administrative_division_geometry, -): - from street_maintenance.management.commands.utils import ( - create_yit_maintenance_units, - create_yit_maintenance_works, - ) - - create_yit_maintenance_units("test_access_token") - get_yit_routes_mock.return_value = get_yit_routes_mock_data(2) - num_created_works, num_del_works = create_yit_maintenance_works( - "test_access_token", 3 - ) - assert num_created_works == 2 - assert num_del_works == 0 - assert MaintenanceWork.objects.count() == 2 - work = MaintenanceWork.objects.first() - work_id = work.id - assert work.events == ["liukkaudentorjunta"] - assert work.original_event_names == ["Suolaus"] - get_yit_routes_mock.return_value = get_yit_routes_mock_data(1) - num_created_works, num_del_works = create_yit_maintenance_works( - "test_access_token", 3 - ) - assert num_created_works == 0 - assert num_del_works == 1 - assert work_id == MaintenanceWork.objects.first().id - assert MaintenanceWork.objects.count() == 1 - # Create duplicate work - work_dup = MaintenanceWork.objects.first() - work_dup.pk = 42 - work_dup.save() - num_created_works, num_del_works = create_yit_maintenance_works( - "test_access_token", 3 - ) - assert num_created_works == 0 - assert num_del_works == 1 - - -@pytest.mark.django_db -@patch("street_maintenance.management.commands.utils.get_json_data") -def test_kuntec( - get_json_data_mock, administrative_division, administrative_division_geometry -): - from street_maintenance.management.commands.utils import ( - create_kuntec_maintenance_units, - create_kuntec_maintenance_works, - ) - - # Note, the fixture JSON contains one unit item with IO_DIN state 0(off) - # i.e., will not be included - get_json_data_mock.return_value = get_kuntec_units_mock_data(2) - num_created_units, num_del_units = create_kuntec_maintenance_units() - assert num_created_units == 2 - assert num_del_units == 0 - assert MaintenanceUnit.objects.count() == 2 - unit = MaintenanceUnit.objects.first() - unit_id = unit.id - assert unit.unit_id == "150635" - assert unit.names == ["Auraus", "Hiekoitus"] - get_json_data_mock.return_value = get_kuntec_units_mock_data(1) - num_created_units, num_del_units = create_kuntec_maintenance_units() - assert unit_id == MaintenanceUnit.objects.first().id - assert num_created_units == 0 - assert num_del_units == 1 - assert MaintenanceUnit.objects.count() == 1 - get_json_data_mock.return_value = get_kuntec_works_mock_data(2) - num_created_works, num_del_works = create_kuntec_maintenance_works(3) - assert num_created_works == 2 - assert num_del_works == 0 - assert MaintenanceWork.objects.count() == 2 - work = MaintenanceWork.objects.first() - work_id = work.id - work.events = ["auraus", "liukkaudentorjunta"] - work.original_event_names = ["Auraus", "Hiekoitus"] - get_json_data_mock.return_value = get_kuntec_works_mock_data(1) - num_created_works, num_del_works = create_kuntec_maintenance_works(3) - assert num_created_works == 0 - assert num_del_works == 1 - assert work_id == MaintenanceWork.objects.first().id - assert MaintenanceWork.objects.count() == 1 - # Test duplicate unit - unit_dup = MaintenanceUnit.objects.first() - unit_dup.pk = 42 - unit_dup.save() - get_json_data_mock.return_value = get_kuntec_units_mock_data(1) - num_created_units, num_del_units = create_kuntec_maintenance_units() - assert num_created_units == 0 - assert num_del_units == 1 - # Create duplicate work - work_dup = MaintenanceWork.objects.first() - work_dup.pk = 42 - work_dup.save() - get_json_data_mock.return_value = get_kuntec_works_mock_data(1) - num_created_works, num_del_works = create_kuntec_maintenance_works(3) - assert num_created_works == 0 - assert num_del_works == 1 - - -@pytest.mark.django_db -@patch("street_maintenance.management.commands.utils.get_json_data") -def test_infraroad( - get_json_data_mock, administrative_division, administrative_division_geometry -): - from street_maintenance.management.commands.utils import ( - create_maintenance_units, - create_maintenance_works, - ) - - # Test unit creation - get_json_data_mock.return_value = get_fluentprogress_units_mock_data(2) - num_created_units, num_del_units = create_maintenance_units(INFRAROAD) - assert MaintenanceUnit.objects.count() == 2 - assert num_created_units == 2 - assert num_del_units == 0 - unit = MaintenanceUnit.objects.first() - unit_id = unit.id - unit.unit_id = "2817625" - unit.names = ["au"] - get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) - num_created_units, num_del_units = create_maintenance_units(INFRAROAD) - assert unit_id == MaintenanceUnit.objects.first().id - assert num_created_units == 0 - assert num_del_units == 1 - assert MaintenanceUnit.objects.count() == 1 - get_json_data_mock.return_value = get_fluentprogress_works_mock_data(3) - num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) - assert num_created_works == 3 - assert num_del_works == 0 - assert MaintenanceWork.objects.count() == 3 - work = MaintenanceWork.objects.first() - work_id = work.id - work.events = ["auraus"] - work.original_event_names = ["au"] - get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) - num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) - assert num_created_works == 0 - assert num_del_works == 2 - assert work_id == MaintenanceWork.objects.first().id - assert MaintenanceWork.objects.count() == 1 - # Test duplicate Unit - unit_dup = MaintenanceUnit.objects.first() - unit_dup.pk = 42 - unit_dup.save() - get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) - num_created_units, num_del_units = create_maintenance_units(INFRAROAD) - assert num_created_units == 0 - assert num_del_units == 1 - # Test duplicate work - work_dup = MaintenanceWork.objects.first() - work_dup.pk = 42 - work_dup.save() - get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) - num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) - assert num_created_works == 0 - assert num_del_works == 1 - - -@pytest.mark.django_db -@patch("street_maintenance.management.commands.utils.get_json_data") -def test_destia( - get_json_data_mock, administrative_division, administrative_division_geometry -): - from street_maintenance.management.commands.utils import ( - create_maintenance_units, - create_maintenance_works, - ) - - # Test unit creation - get_json_data_mock.return_value = get_fluentprogress_units_mock_data(2) - num_created_units, num_del_units = create_maintenance_units(DESTIA) - assert MaintenanceUnit.objects.count() == 2 - assert num_created_units == 2 - assert num_del_units == 0 - unit = MaintenanceUnit.objects.first() - unit_id = unit.id - unit.unit_id = "2817625" - unit.names = ["au"] - get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) - num_created_units, num_del_units = create_maintenance_units(DESTIA) - assert unit_id == MaintenanceUnit.objects.first().id - assert num_created_units == 0 - assert num_del_units == 1 - assert MaintenanceUnit.objects.count() == 1 - get_json_data_mock.return_value = get_fluentprogress_works_mock_data(3) - num_created_works, num_del_works = create_maintenance_works(DESTIA, 1, 10) - assert num_created_works == 3 - assert num_del_works == 0 - assert MaintenanceWork.objects.count() == 3 - work = MaintenanceWork.objects.first() - work_id = work.id - work.events = ["auraus"] - work.original_event_names = ["au"] - work = MaintenanceWork.objects.get( - original_event_names=["au", "sivuaura", "sirotin"] - ) - # Test that duplicate events are not included, as "sivuaura" and "au" are mapped to "auraus" - assert work.events == ["auraus", "liukkaudentorjunta"] - get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) - num_created_works, num_del_works = create_maintenance_works(DESTIA, 1, 10) - assert num_created_works == 0 - assert num_del_works == 2 - assert work_id == MaintenanceWork.objects.first().id - assert MaintenanceWork.objects.count() == 1 diff --git a/street_maintenance/tests/utils.py b/street_maintenance/tests/utils.py deleted file mode 100644 index 078807e51..000000000 --- a/street_maintenance/tests/utils.py +++ /dev/null @@ -1,360 +0,0 @@ -from datetime import datetime - -from street_maintenance.management.commands.constants import ( - DATE_FORMATS, - INFRAROAD, - YIT, -) - - -def get_yit_vehicles_mock_data(num_elements): - data = [ - {"id": "82260ff7-589e-4cee-a8e0-124b615381f1", "vehicleTypeName": "Huoltoauto"}, - { - "id": "77396829-6275-4dbe-976f-f9d4f5254653", - "vehicleTypeName": "Kuorma-auto", - }, - ] - assert num_elements <= len(data) - return data[:num_elements] - - -def get_yit_event_types_mock_data(): - data = [ - { - "id": "a4f6188f-8c25-4f42-89be-f5a1bd7d833a", - "operationName": "Auraus ja sohjonpoisto", - }, - {"id": "a51e8e4c-8b16-4132-a882-70f6624c1f2b", "operationName": "Suolaus"}, - ] - return data - - -def get_yit_contract_mock_data(): - return "d73447e6-df70-4f4a-817d-3387b58aca6c" - - -def get_yit_routes_mock_data(num_elements): - current_date = datetime.now().date().strftime(DATE_FORMATS[YIT]) - data = [ - { - "vehicleType": "82260ff7-589e-4cee-a8e0-124b615381f1", - "length": 0.0, - "geography": { - "crs": None, - "features": [ - { - "geometry": { - "coordinates": [ - [22.315554108363656, 60.47901418729062], - [22.31555399713308, 60.47901429688299], - ], - "type": "LineString", - }, - "properties": { - "streetAddress": "Polttolaitoksenkatu 13, Turku", - "featureType": "StreetAddress", - }, - "type": "Feature", - } - ], - "type": "FeatureCollection", - }, - "created": f"{current_date}T11:52:00.5136066Z", - "updated": f"{current_date}T11:52:00.5136066Z", - "deleted": False, - "id": "9c566b34-2bb5-46b0-9c0a-99f53eada2d2", - "user": "442a5ab2-d58c-4c22-bae2-bcf55327cde7", - "contract": "d73447e6-df70-4f4a-817d-3387b58aca6c", - "startTime": f"{current_date}T11:50:21.708Z", - "endTime": f"{current_date}T11:50:21.709Z", - "operations": ["a51e8e4c-8b16-4132-a882-70f6624c1f2b"], - }, - { - # Note, the MaintenanceUnit is retrieved by the vehicleType - "vehicleType": "82260ff7-589e-4cee-a8e0-124b615381f1", - "length": 21.0, - "geography": { - "crs": None, - "features": [ - { - "geometry": { - "coordinates": [ - [22.308685, 60.471465], - [22.308758333333337, 60.47151166666666], - [22.308844999999998, 60.47154999999999], - [22.30887476486449, 60.471557320473416], - ], - "type": "LineString", - }, - "properties": { - "streetAddress": "Koroistenkaari, Turku", - "featureType": "StreetAddress", - }, - "type": "Feature", - } - ], - "type": "FeatureCollection", - }, - "created": f"{current_date}T11:50:58.6173037Z", - "updated": f"{current_date}T11:50:58.6173037Z", - "deleted": False, - "id": "aaee2c3b-4296-44b3-aba0-82b859d4eea8", - "user": "442a5ab2-d58c-4c22-bae2-bcf55327cde7", - "contract": "d73447e6-df70-4f4a-817d-3387b58aca6c", - "startTime": f"{current_date}T11:48:44.694Z", - "endTime": f"{current_date}T11:48:46.984Z", - "operations": ["a51e8e4c-8b16-4132-a882-70f6624c1f2b"], - }, - ] - assert num_elements <= len(data) - return data[:num_elements] - - -# Both Destia and Infraroad uses the fluentprogress API -def get_fluentprogress_works_mock_data(num_elements): - current_date = datetime.now().date().strftime(DATE_FORMATS[INFRAROAD]) - location_history = [ - { - "timestamp": f"{current_date} 08:29:49", - "coords": "(22.24957474 60.49515401)", - "events": ["au"], - }, - { - "timestamp": f"{current_date} 08:29:28", - "coords": "(22.24946401 60.49515848)", - "events": ["au", "sivuaura", "sirotin"], - }, - { - "timestamp": f"{current_date} 08:28:32", - "coords": "(22.24944127 60.49519463)", - "events": ["hiekoitus"], - }, - ] - assert num_elements <= len(location_history) - data = {"location_history": location_history[:num_elements]} - return data - - -def get_fluentprogress_units_mock_data(num_elements): - current_date = datetime.now().date().strftime(DATE_FORMATS[INFRAROAD]) - data = [ - { - "id": 2817625, - "last_location": { - "timestamp": f"{current_date} 06:31:34", - "coords": "(22.249642023816705 60.49569119699299)", - "events": ["au"], - }, - }, - { - "id": 12891825, - "last_location": { - "timestamp": f"{current_date} 08:29:49", - "coords": "(22.24957474 60.49515401)", - "events": ["Kenttien hoito"], - }, - }, - ] - assert num_elements <= len(data) - return data[:num_elements] - - -def get_kuntec_works_mock_data(num_elements): - current_date = datetime.now().date().strftime(DATE_FORMATS[INFRAROAD]) - routes = [ - { - "route_id": 3980827390, - "type": "route", - "start": { - "time": f"{current_date}T14:54:31Z", - "address": "M\u00e4likk\u00e4l\u00e4, 21280 Turku, Suomi", - "lat": 60.47185, - "lng": 22.21618, - }, - "avg_speed": 18, - "max_speed": 43, - "end": { - "time": f"{current_date}T15:05:56Z", - "address": "Ihalantie 7, 21200 Raisio, Suomi", - "lat": 60.47945, - "lng": 22.18221, - }, - "distance": 3565, - "polyline": "a|apJcbrfC@HEpQYdEi@rDeAnE{Sji@eOjp@qEpb@uUlkA?T@TFRHLNDb@@b@Cd@Sb@i@XeArCeJv@{@z@c@h@E~@JdC" - + "n@FCBE@C@G@G@K?KEsA@cBAI?K?}BBWF[DOBMJQBCB?D?JHFHDN@FBNBLBD@BDBES?EAEAEI[CM@EXt@Uw@Tz@BHA?EQEMO_@@HIII" - + "IAE?QAc@EXCECBEDAHWdBENAF?D?D?v@DhE@JCQ?M@M?_@@oFBKF]", - }, - { - "route_id": 3984019243, - "type": "route", - "start": { - "time": f"{current_date}T11:16:00Z", - "address": "Tuontikatu 180, 20200 Turku, Suomi", - "lat": 60.44447, - "lng": 22.21462, - }, - "avg_speed": 18, - "max_speed": 43, - "end": { - "time": f"{current_date}T11:21:55Z", - "address": "Tuontikatu, 20200 Turku, Suomi", - "lat": 60.44754, - "lng": 22.21391, - }, - "distance": 1799, - "polyline": "}p|oJkxqfCt@|AtAtELT@JFd@d@nBX`AfAlFLl@Rv@xHv[T^p@~@DJ@B?DDPD@KUACCIQUg@i@]q@m@_C{Loi@AUIKCEU" - + "_@Oc@{@yCw@aB_BaB_A_@_CI{F[IFGLI^A`@?d@IfCGxA", - }, - ] - - assert num_elements <= len(routes) - data = {"data": {"units": [{"routes": routes[:num_elements]}]}} - return data - - -def get_kuntec_units_mock_data(num_elements): - null = None - units = [ - { - "unit_id": 150635, - "box_id": 27953713, - "company_id": 5495, - "country_code": "FI", - "label": "1100781186", - "number": "HAKA 2", - "shortcut": "", - "vehicle_title": null, - "car_reg_certificate": "", - "vin": null, - "type": "car", - "icon": "tractor", - "lat": 60.46423, - "lng": 22.43703, - "direction": 161, - "speed": null, - "mileage": 11779869, - "last_update": "2023-02-25T13:20:56Z", - "ignition_total_time": 7683589, - "state": { - "name": "nodata", - "start": "2023-02-25T12:40:53Z", - "duration": 148756, - "debug_info": { - "boxId": 27953713, - "carId": 150635, - "msg": "POWEROFF", - "lastUpdate": 1677331256, - "lastValues": null, - }, - }, - "movement_state": { - "name": "nodata", - "start": "2023-02-25T12:40:53Z", - "duration": 151159, - }, - "fuel_type": "", - "avg_fuel_consumption": {"norm": 0, "measurement": "l/100km"}, - "created_at": "2019-11-05T10:10:38Z", - "io_din": [ - {"no": 1, "label": "Auraus", "state": 1}, - {"no": 2, "label": "Hiekoitus", "state": 1}, - {"no": 3, "label": "Muu ty\u00f6", "state": 0}, - ], - }, - { - "unit_id": 150662, - "box_id": 27953746, - "company_id": 5495, - "country_code": "FI", - "label": "1101049692", - "number": "TAKKU 1", - "shortcut": "", - "vehicle_title": null, - "car_reg_certificate": "", - "vin": null, - "type": "car", - "icon": "tractor", - "lat": 60.55185, - "lng": 22.20567, - "direction": 213, - "speed": null, - "mileage": 10537795, - "last_update": "2023-02-26T10:02:03Z", - "ignition_total_time": 0, - "state": { - "name": "nodata", - "start": "2023-02-26T09:33:37Z", - "duration": 74289, - "debug_info": { - "boxId": 27953746, - "carId": 150662, - "msg": "OTHER", - "lastUpdate": 1677405723, - "lastValues": {"VOLTAGE": "14289"}, - }, - }, - "movement_state": { - "name": "nodata", - "start": "2023-02-26T09:33:37Z", - "duration": 75995, - }, - "fuel_type": "", - "avg_fuel_consumption": {"norm": 0, "measurement": "l/100km"}, - "created_at": "2019-11-05T10:39:46Z", - "io_din": [ - {"no": 1, "label": "Auraus", "state": 1}, - {"no": 2, "label": "Hiekoitus", "state": 0}, - {"no": 3, "label": "Muu ty\u00f6", "state": 0}, - ], - }, - { - "unit_id": 150662, - "box_id": 27953746, - "company_id": 5495, - "country_code": "FI", - "label": "1101049692", - "number": "TAKKU 1", - "shortcut": "", - "vehicle_title": null, - "car_reg_certificate": "", - "vin": null, - "type": "car", - "icon": "tractor", - "lat": 60.55185, - "lng": 22.20567, - "direction": 213, - "speed": null, - "mileage": 10537795, - "last_update": "2023-02-26T10:02:03Z", - "ignition_total_time": 0, - "state": { - "name": "nodata", - "start": "2023-02-26T09:33:37Z", - "duration": 74289, - "debug_info": { - "boxId": 27953746, - "carId": 150662, - "msg": "OTHER", - "lastUpdate": 1677405723, - "lastValues": {"VOLTAGE": "14289"}, - }, - }, - "movement_state": { - "name": "nodata", - "start": "2023-02-26T09:33:37Z", - "duration": 75995, - }, - "fuel_type": "", - "avg_fuel_consumption": {"norm": 0, "measurement": "l/100km"}, - "created_at": "2019-11-05T10:39:46Z", - "io_din": [ - {"no": 1, "label": "Auraus", "state": 1}, - {"no": 2, "label": "Hiekoitus", "state": 0}, - {"no": 3, "label": "Muu ty\u00f6", "state": 0}, - ], - }, - ] - assert num_elements <= len(units) - data = {"data": {"units": units[:num_elements]}} - return data From 2a8ae446321720e7a53bb1390574501944b0ea1b Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:57:43 +0300 Subject: [PATCH 06/66] Copy tests from street_maintenance app --- maintenance/tests/__init__.py | 0 maintenance/tests/conftest.py | 97 ++++++++ maintenance/tests/test_api.py | 70 ++++++ maintenance/tests/test_importers.py | 275 +++++++++++++++++++++ maintenance/tests/utils.py | 360 ++++++++++++++++++++++++++++ 5 files changed, 802 insertions(+) create mode 100644 maintenance/tests/__init__.py create mode 100644 maintenance/tests/conftest.py create mode 100644 maintenance/tests/test_api.py create mode 100644 maintenance/tests/test_importers.py create mode 100644 maintenance/tests/utils.py diff --git a/maintenance/tests/__init__.py b/maintenance/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py new file mode 100644 index 000000000..e7b138226 --- /dev/null +++ b/maintenance/tests/conftest.py @@ -0,0 +1,97 @@ +from datetime import datetime, timedelta + +import pytest +import pytz +from django.contrib.gis.geos import GEOSGeometry, LineString +from munigeo.models import ( + AdministrativeDivision, + AdministrativeDivisionGeometry, + AdministrativeDivisionType, +) +from rest_framework.test import APIClient + +from mobility_data.tests.conftest import TURKU_WKT +from maintenance.management.commands.constants import ( + AURAUS, + INFRAROAD, + KUNTEC, + LIUKKAUDENTORJUNTA, +) +from maintenance.models import DEFAULT_SRID, GeometryHistory + +UTC_TIMEZONE = pytz.timezone("UTC") + + +@pytest.fixture +def api_client(): + return APIClient() + + +@pytest.mark.django_db +@pytest.fixture +def geometry_historys(): + now = datetime.now(UTC_TIMEZONE) + geometry = LineString((0, 0), (0, 50), (50, 50), (50, 0), (0, 0), sird=DEFAULT_SRID) + GeometryHistory.objects.create( + timestamp=now, + geometry=geometry, + coordinates=geometry.coords, + provider=INFRAROAD, + events=[AURAUS], + ) + GeometryHistory.objects.create( + timestamp=now - timedelta(days=1), + geometry=geometry, + coordinates=geometry.coords, + provider=INFRAROAD, + events=[AURAUS], + ) + GeometryHistory.objects.create( + timestamp=now - timedelta(days=2), + geometry=geometry, + coordinates=geometry.coords, + provider=INFRAROAD, + events=[LIUKKAUDENTORJUNTA], + ) + GeometryHistory.objects.create( + timestamp=now - timedelta(days=1), + geometry=geometry, + coordinates=geometry.coords, + provider=KUNTEC, + events=[AURAUS], + ) + GeometryHistory.objects.create( + timestamp=now - timedelta(days=2), + geometry=geometry, + coordinates=geometry.coords, + provider=KUNTEC, + events=[AURAUS, LIUKKAUDENTORJUNTA], + ) + return GeometryHistory.objects.all() + +@pytest.mark.django_db +@pytest.fixture +def administrative_division_type(): + adm_div_type = AdministrativeDivisionType.objects.create( + id=1, type="muni", name="Municipality" + ) + return adm_div_type + + +@pytest.mark.django_db +@pytest.fixture +def administrative_division(administrative_division_type): + adm_div = AdministrativeDivision.objects.get_or_create( + id=1, name="Turku", origin_id=853, type_id=1 + ) + return adm_div + + +@pytest.mark.django_db +@pytest.fixture +def administrative_division_geometry(administrative_division): + turku_multipoly = GEOSGeometry(TURKU_WKT, srid=3067) + adm_div_geom = AdministrativeDivisionGeometry.objects.create( + id=1, division_id=1, boundary=turku_multipoly + ) + return adm_div_geom diff --git a/maintenance/tests/test_api.py b/maintenance/tests/test_api.py new file mode 100644 index 000000000..a60725b0a --- /dev/null +++ b/maintenance/tests/test_api.py @@ -0,0 +1,70 @@ +from datetime import timedelta + +import pytest +from django.utils import timezone +from rest_framework.reverse import reverse + +from maintenance.management.commands.constants import ( + AURAUS, + INFRAROAD, + KUNTEC, + LIUKKAUDENTORJUNTA, + START_DATE_TIME_FORMAT, +) + + +@pytest.mark.django_db +def test_geometry_history_list(api_client, geometry_historys): + url = reverse("maintenance:geometry_history-list") + response = api_client.get(url) + breakpoint() + assert response.json()["count"] == 5 + + +@pytest.mark.django_db +def test_geometry_history_list_provider_parameter(api_client, geometry_historys): + url = reverse("maintenance:geometry_history-list") + f"?provider={KUNTEC}" + response = api_client.get(url) + # Fixture data contains 2 KUNTEC GeometryHistroy rows + assert response.json()["count"] == 2 + + +@pytest.mark.django_db +def test_geometry_history_list_event_parameter(api_client, geometry_historys): + url = reverse("maintenance:geometry_history-list") + f"?event={AURAUS}" + response = api_client.get(url) + # 3 INFRAROAD AURAUS events and 1 KUNTEC + assert response.json()["count"] == 4 + + +@pytest.mark.django_db +def test_geometry_history_list_event_and_provider_parameter( + api_client, geometry_historys +): + url = ( + reverse("maintenance:geometry_history-list") + + f"?provider={KUNTEC}&event={LIUKKAUDENTORJUNTA}" + ) + response = api_client.get(url) + assert response.json()["count"] == 1 + + +@pytest.mark.django_db +def test_geometry_history_list_start_date_time_parameter(api_client, geometry_historys): + start_date_time = timezone.now() - timedelta(hours=1) + url = ( + reverse("maintenance:geometry_history-list") + + f"?start_date_time={start_date_time.strftime(START_DATE_TIME_FORMAT)}" + ) + response = api_client.get(url) + assert response.json()["count"] == 1 + geometry_history = response.json()["results"][0] + assert geometry_history["geometry_type"] == "LineString" + assert geometry_history["provider"] == INFRAROAD + start_date_time = timezone.now() - timedelta(days=1, hours=2) + url = ( + reverse("maintenance:geometry_history-list") + + f"?start_date_time={start_date_time.strftime(START_DATE_TIME_FORMAT)}" + ) + response = api_client.get(url) + assert response.json()["count"] == 3 diff --git a/maintenance/tests/test_importers.py b/maintenance/tests/test_importers.py new file mode 100644 index 000000000..6cfebb169 --- /dev/null +++ b/maintenance/tests/test_importers.py @@ -0,0 +1,275 @@ +from unittest.mock import patch + +import pytest + +from maintenance.management.commands.constants import DESTIA, INFRAROAD +from maintenance.models import MaintenanceUnit, MaintenanceWork + +from .utils import ( + get_fluentprogress_units_mock_data, + get_fluentprogress_works_mock_data, + get_kuntec_units_mock_data, + get_kuntec_works_mock_data, + get_yit_contract_mock_data, + get_yit_event_types_mock_data, + get_yit_routes_mock_data, + get_yit_vehicles_mock_data, +) + + +@pytest.mark.django_db +@patch("maintenance.management.commands.utils.get_yit_vehicles") +def test_yit_units( + get_yit_vehicles_mock, + administrative_division, + administrative_division_geometry, +): + from maintenance.management.commands.utils import ( + create_yit_maintenance_units, + ) + + get_yit_vehicles_mock.return_value = get_yit_vehicles_mock_data(2) + num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") + assert MaintenanceUnit.objects.count() == 2 + assert num_created_units == 2 + assert num_del_units == 0 + unit = MaintenanceUnit.objects.first() + unit_id = unit.id + assert unit.names == ["Huoltoauto"] + assert unit.unit_id == "82260ff7-589e-4cee-a8e0-124b615381f1" + get_yit_vehicles_mock.return_value = get_yit_vehicles_mock_data(1) + num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") + assert unit_id == MaintenanceUnit.objects.first().id + assert num_created_units == 0 + assert num_del_units == 1 + assert MaintenanceUnit.objects.count() == 1 + # Test duplicate unit + unit_dup = MaintenanceUnit.objects.first() + unit_dup.pk = 42 + unit_dup.save() + num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") + assert num_created_units == 0 + assert num_del_units == 1 + + +@pytest.mark.django_db +@patch( + "maintenance.management.commands.utils.get_yit_vehicles", + return_value=get_yit_vehicles_mock_data(2), +) +@patch( + "maintenance.management.commands.utils.get_yit_contract", + return_value=get_yit_contract_mock_data(), +) +@patch( + "maintenance.management.commands.utils.get_yit_event_types", + return_value=get_yit_event_types_mock_data(), +) +@patch("maintenance.management.commands.utils.get_yit_routes") +def test_yit_works( + get_yit_routes_mock, + get_yit_vechiles_mock, + administrative_division, + administrative_division_geometry, +): + from maintenance.management.commands.utils import ( + create_yit_maintenance_units, + create_yit_maintenance_works, + ) + + create_yit_maintenance_units("test_access_token") + get_yit_routes_mock.return_value = get_yit_routes_mock_data(2) + num_created_works, num_del_works = create_yit_maintenance_works( + "test_access_token", 3 + ) + assert num_created_works == 2 + assert num_del_works == 0 + assert MaintenanceWork.objects.count() == 2 + work = MaintenanceWork.objects.first() + work_id = work.id + assert work.events == ["liukkaudentorjunta"] + assert work.original_event_names == ["Suolaus"] + get_yit_routes_mock.return_value = get_yit_routes_mock_data(1) + num_created_works, num_del_works = create_yit_maintenance_works( + "test_access_token", 3 + ) + assert num_created_works == 0 + assert num_del_works == 1 + assert work_id == MaintenanceWork.objects.first().id + assert MaintenanceWork.objects.count() == 1 + # Create duplicate work + work_dup = MaintenanceWork.objects.first() + work_dup.pk = 42 + work_dup.save() + num_created_works, num_del_works = create_yit_maintenance_works( + "test_access_token", 3 + ) + assert num_created_works == 0 + assert num_del_works == 1 + + +@pytest.mark.django_db +@patch("maintenance.management.commands.utils.get_json_data") +def test_kuntec( + get_json_data_mock, administrative_division, administrative_division_geometry +): + from maintenance.management.commands.utils import ( + create_kuntec_maintenance_units, + create_kuntec_maintenance_works, + ) + + # Note, the fixture JSON contains one unit item with IO_DIN state 0(off) + # i.e., will not be included + get_json_data_mock.return_value = get_kuntec_units_mock_data(2) + num_created_units, num_del_units = create_kuntec_maintenance_units() + assert num_created_units == 2 + assert num_del_units == 0 + assert MaintenanceUnit.objects.count() == 2 + unit = MaintenanceUnit.objects.first() + unit_id = unit.id + assert unit.unit_id == "150635" + assert unit.names == ["Auraus", "Hiekoitus"] + get_json_data_mock.return_value = get_kuntec_units_mock_data(1) + num_created_units, num_del_units = create_kuntec_maintenance_units() + assert unit_id == MaintenanceUnit.objects.first().id + assert num_created_units == 0 + assert num_del_units == 1 + assert MaintenanceUnit.objects.count() == 1 + get_json_data_mock.return_value = get_kuntec_works_mock_data(2) + num_created_works, num_del_works = create_kuntec_maintenance_works(3) + assert num_created_works == 2 + assert num_del_works == 0 + assert MaintenanceWork.objects.count() == 2 + work = MaintenanceWork.objects.first() + work_id = work.id + work.events = ["auraus", "liukkaudentorjunta"] + work.original_event_names = ["Auraus", "Hiekoitus"] + get_json_data_mock.return_value = get_kuntec_works_mock_data(1) + num_created_works, num_del_works = create_kuntec_maintenance_works(3) + assert num_created_works == 0 + assert num_del_works == 1 + assert work_id == MaintenanceWork.objects.first().id + assert MaintenanceWork.objects.count() == 1 + # Test duplicate unit + unit_dup = MaintenanceUnit.objects.first() + unit_dup.pk = 42 + unit_dup.save() + get_json_data_mock.return_value = get_kuntec_units_mock_data(1) + num_created_units, num_del_units = create_kuntec_maintenance_units() + assert num_created_units == 0 + assert num_del_units == 1 + # Create duplicate work + work_dup = MaintenanceWork.objects.first() + work_dup.pk = 42 + work_dup.save() + get_json_data_mock.return_value = get_kuntec_works_mock_data(1) + num_created_works, num_del_works = create_kuntec_maintenance_works(3) + assert num_created_works == 0 + assert num_del_works == 1 + + +@pytest.mark.django_db +@patch("maintenance.management.commands.utils.get_json_data") +def test_infraroad( + get_json_data_mock, administrative_division, administrative_division_geometry +): + from maintenance.management.commands.utils import ( + create_maintenance_units, + create_maintenance_works, + ) + + # Test unit creation + get_json_data_mock.return_value = get_fluentprogress_units_mock_data(2) + num_created_units, num_del_units = create_maintenance_units(INFRAROAD) + assert MaintenanceUnit.objects.count() == 2 + assert num_created_units == 2 + assert num_del_units == 0 + unit = MaintenanceUnit.objects.first() + unit_id = unit.id + unit.unit_id = "2817625" + unit.names = ["au"] + get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) + num_created_units, num_del_units = create_maintenance_units(INFRAROAD) + assert unit_id == MaintenanceUnit.objects.first().id + assert num_created_units == 0 + assert num_del_units == 1 + assert MaintenanceUnit.objects.count() == 1 + get_json_data_mock.return_value = get_fluentprogress_works_mock_data(3) + num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) + assert num_created_works == 3 + assert num_del_works == 0 + assert MaintenanceWork.objects.count() == 3 + work = MaintenanceWork.objects.first() + work_id = work.id + work.events = ["auraus"] + work.original_event_names = ["au"] + get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) + num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) + assert num_created_works == 0 + assert num_del_works == 2 + assert work_id == MaintenanceWork.objects.first().id + assert MaintenanceWork.objects.count() == 1 + # Test duplicate Unit + unit_dup = MaintenanceUnit.objects.first() + unit_dup.pk = 42 + unit_dup.save() + get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) + num_created_units, num_del_units = create_maintenance_units(INFRAROAD) + assert num_created_units == 0 + assert num_del_units == 1 + # Test duplicate work + work_dup = MaintenanceWork.objects.first() + work_dup.pk = 42 + work_dup.save() + get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) + num_created_works, num_del_works = create_maintenance_works(INFRAROAD, 1, 10) + assert num_created_works == 0 + assert num_del_works == 1 + + +@pytest.mark.django_db +@patch("maintenance.management.commands.utils.get_json_data") +def test_destia( + get_json_data_mock, administrative_division, administrative_division_geometry +): + from maintenance.management.commands.utils import ( + create_maintenance_units, + create_maintenance_works, + ) + + # Test unit creation + get_json_data_mock.return_value = get_fluentprogress_units_mock_data(2) + num_created_units, num_del_units = create_maintenance_units(DESTIA) + assert MaintenanceUnit.objects.count() == 2 + assert num_created_units == 2 + assert num_del_units == 0 + unit = MaintenanceUnit.objects.first() + unit_id = unit.id + unit.unit_id = "2817625" + unit.names = ["au"] + get_json_data_mock.return_value = get_fluentprogress_units_mock_data(1) + num_created_units, num_del_units = create_maintenance_units(DESTIA) + assert unit_id == MaintenanceUnit.objects.first().id + assert num_created_units == 0 + assert num_del_units == 1 + assert MaintenanceUnit.objects.count() == 1 + get_json_data_mock.return_value = get_fluentprogress_works_mock_data(3) + num_created_works, num_del_works = create_maintenance_works(DESTIA, 1, 10) + assert num_created_works == 3 + assert num_del_works == 0 + assert MaintenanceWork.objects.count() == 3 + work = MaintenanceWork.objects.first() + work_id = work.id + work.events = ["auraus"] + work.original_event_names = ["au"] + work = MaintenanceWork.objects.get( + original_event_names=["au", "sivuaura", "sirotin"] + ) + # Test that duplicate events are not included, as "sivuaura" and "au" are mapped to "auraus" + assert work.events == ["auraus", "liukkaudentorjunta"] + get_json_data_mock.return_value = get_fluentprogress_works_mock_data(1) + num_created_works, num_del_works = create_maintenance_works(DESTIA, 1, 10) + assert num_created_works == 0 + assert num_del_works == 2 + assert work_id == MaintenanceWork.objects.first().id + assert MaintenanceWork.objects.count() == 1 diff --git a/maintenance/tests/utils.py b/maintenance/tests/utils.py new file mode 100644 index 000000000..6d402f676 --- /dev/null +++ b/maintenance/tests/utils.py @@ -0,0 +1,360 @@ +from datetime import datetime + +from maintenance.management.commands.constants import ( + DATE_FORMATS, + INFRAROAD, + YIT, +) + + +def get_yit_vehicles_mock_data(num_elements): + data = [ + {"id": "82260ff7-589e-4cee-a8e0-124b615381f1", "vehicleTypeName": "Huoltoauto"}, + { + "id": "77396829-6275-4dbe-976f-f9d4f5254653", + "vehicleTypeName": "Kuorma-auto", + }, + ] + assert num_elements <= len(data) + return data[:num_elements] + + +def get_yit_event_types_mock_data(): + data = [ + { + "id": "a4f6188f-8c25-4f42-89be-f5a1bd7d833a", + "operationName": "Auraus ja sohjonpoisto", + }, + {"id": "a51e8e4c-8b16-4132-a882-70f6624c1f2b", "operationName": "Suolaus"}, + ] + return data + + +def get_yit_contract_mock_data(): + return "d73447e6-df70-4f4a-817d-3387b58aca6c" + + +def get_yit_routes_mock_data(num_elements): + current_date = datetime.now().date().strftime(DATE_FORMATS[YIT]) + data = [ + { + "vehicleType": "82260ff7-589e-4cee-a8e0-124b615381f1", + "length": 0.0, + "geography": { + "crs": None, + "features": [ + { + "geometry": { + "coordinates": [ + [22.315554108363656, 60.47901418729062], + [22.31555399713308, 60.47901429688299], + ], + "type": "LineString", + }, + "properties": { + "streetAddress": "Polttolaitoksenkatu 13, Turku", + "featureType": "StreetAddress", + }, + "type": "Feature", + } + ], + "type": "FeatureCollection", + }, + "created": f"{current_date}T11:52:00.5136066Z", + "updated": f"{current_date}T11:52:00.5136066Z", + "deleted": False, + "id": "9c566b34-2bb5-46b0-9c0a-99f53eada2d2", + "user": "442a5ab2-d58c-4c22-bae2-bcf55327cde7", + "contract": "d73447e6-df70-4f4a-817d-3387b58aca6c", + "startTime": f"{current_date}T11:50:21.708Z", + "endTime": f"{current_date}T11:50:21.709Z", + "operations": ["a51e8e4c-8b16-4132-a882-70f6624c1f2b"], + }, + { + # Note, the MaintenanceUnit is retrieved by the vehicleType + "vehicleType": "82260ff7-589e-4cee-a8e0-124b615381f1", + "length": 21.0, + "geography": { + "crs": None, + "features": [ + { + "geometry": { + "coordinates": [ + [22.308685, 60.471465], + [22.308758333333337, 60.47151166666666], + [22.308844999999998, 60.47154999999999], + [22.30887476486449, 60.471557320473416], + ], + "type": "LineString", + }, + "properties": { + "streetAddress": "Koroistenkaari, Turku", + "featureType": "StreetAddress", + }, + "type": "Feature", + } + ], + "type": "FeatureCollection", + }, + "created": f"{current_date}T11:50:58.6173037Z", + "updated": f"{current_date}T11:50:58.6173037Z", + "deleted": False, + "id": "aaee2c3b-4296-44b3-aba0-82b859d4eea8", + "user": "442a5ab2-d58c-4c22-bae2-bcf55327cde7", + "contract": "d73447e6-df70-4f4a-817d-3387b58aca6c", + "startTime": f"{current_date}T11:48:44.694Z", + "endTime": f"{current_date}T11:48:46.984Z", + "operations": ["a51e8e4c-8b16-4132-a882-70f6624c1f2b"], + }, + ] + assert num_elements <= len(data) + return data[:num_elements] + + +# Both Destia and Infraroad uses the fluentprogress API +def get_fluentprogress_works_mock_data(num_elements): + current_date = datetime.now().date().strftime(DATE_FORMATS[INFRAROAD]) + location_history = [ + { + "timestamp": f"{current_date} 08:29:49", + "coords": "(22.24957474 60.49515401)", + "events": ["au"], + }, + { + "timestamp": f"{current_date} 08:29:28", + "coords": "(22.24946401 60.49515848)", + "events": ["au", "sivuaura", "sirotin"], + }, + { + "timestamp": f"{current_date} 08:28:32", + "coords": "(22.24944127 60.49519463)", + "events": ["hiekoitus"], + }, + ] + assert num_elements <= len(location_history) + data = {"location_history": location_history[:num_elements]} + return data + + +def get_fluentprogress_units_mock_data(num_elements): + current_date = datetime.now().date().strftime(DATE_FORMATS[INFRAROAD]) + data = [ + { + "id": 2817625, + "last_location": { + "timestamp": f"{current_date} 06:31:34", + "coords": "(22.249642023816705 60.49569119699299)", + "events": ["au"], + }, + }, + { + "id": 12891825, + "last_location": { + "timestamp": f"{current_date} 08:29:49", + "coords": "(22.24957474 60.49515401)", + "events": ["Kenttien hoito"], + }, + }, + ] + assert num_elements <= len(data) + return data[:num_elements] + + +def get_kuntec_works_mock_data(num_elements): + current_date = datetime.now().date().strftime(DATE_FORMATS[INFRAROAD]) + routes = [ + { + "route_id": 3980827390, + "type": "route", + "start": { + "time": f"{current_date}T14:54:31Z", + "address": "M\u00e4likk\u00e4l\u00e4, 21280 Turku, Suomi", + "lat": 60.47185, + "lng": 22.21618, + }, + "avg_speed": 18, + "max_speed": 43, + "end": { + "time": f"{current_date}T15:05:56Z", + "address": "Ihalantie 7, 21200 Raisio, Suomi", + "lat": 60.47945, + "lng": 22.18221, + }, + "distance": 3565, + "polyline": "a|apJcbrfC@HEpQYdEi@rDeAnE{Sji@eOjp@qEpb@uUlkA?T@TFRHLNDb@@b@Cd@Sb@i@XeArCeJv@{@z@c@h@E~@JdC" + + "n@FCBE@C@G@G@K?KEsA@cBAI?K?}BBWF[DOBMJQBCB?D?JHFHDN@FBNBLBD@BDBES?EAEAEI[CM@EXt@Uw@Tz@BHA?EQEMO_@@HIII" + + "IAE?QAc@EXCECBEDAHWdBENAF?D?D?v@DhE@JCQ?M@M?_@@oFBKF]", + }, + { + "route_id": 3984019243, + "type": "route", + "start": { + "time": f"{current_date}T11:16:00Z", + "address": "Tuontikatu 180, 20200 Turku, Suomi", + "lat": 60.44447, + "lng": 22.21462, + }, + "avg_speed": 18, + "max_speed": 43, + "end": { + "time": f"{current_date}T11:21:55Z", + "address": "Tuontikatu, 20200 Turku, Suomi", + "lat": 60.44754, + "lng": 22.21391, + }, + "distance": 1799, + "polyline": "}p|oJkxqfCt@|AtAtELT@JFd@d@nBX`AfAlFLl@Rv@xHv[T^p@~@DJ@B?DDPD@KUACCIQUg@i@]q@m@_C{Loi@AUIKCEU" + + "_@Oc@{@yCw@aB_BaB_A_@_CI{F[IFGLI^A`@?d@IfCGxA", + }, + ] + + assert num_elements <= len(routes) + data = {"data": {"units": [{"routes": routes[:num_elements]}]}} + return data + + +def get_kuntec_units_mock_data(num_elements): + null = None + units = [ + { + "unit_id": 150635, + "box_id": 27953713, + "company_id": 5495, + "country_code": "FI", + "label": "1100781186", + "number": "HAKA 2", + "shortcut": "", + "vehicle_title": null, + "car_reg_certificate": "", + "vin": null, + "type": "car", + "icon": "tractor", + "lat": 60.46423, + "lng": 22.43703, + "direction": 161, + "speed": null, + "mileage": 11779869, + "last_update": "2023-02-25T13:20:56Z", + "ignition_total_time": 7683589, + "state": { + "name": "nodata", + "start": "2023-02-25T12:40:53Z", + "duration": 148756, + "debug_info": { + "boxId": 27953713, + "carId": 150635, + "msg": "POWEROFF", + "lastUpdate": 1677331256, + "lastValues": null, + }, + }, + "movement_state": { + "name": "nodata", + "start": "2023-02-25T12:40:53Z", + "duration": 151159, + }, + "fuel_type": "", + "avg_fuel_consumption": {"norm": 0, "measurement": "l/100km"}, + "created_at": "2019-11-05T10:10:38Z", + "io_din": [ + {"no": 1, "label": "Auraus", "state": 1}, + {"no": 2, "label": "Hiekoitus", "state": 1}, + {"no": 3, "label": "Muu ty\u00f6", "state": 0}, + ], + }, + { + "unit_id": 150662, + "box_id": 27953746, + "company_id": 5495, + "country_code": "FI", + "label": "1101049692", + "number": "TAKKU 1", + "shortcut": "", + "vehicle_title": null, + "car_reg_certificate": "", + "vin": null, + "type": "car", + "icon": "tractor", + "lat": 60.55185, + "lng": 22.20567, + "direction": 213, + "speed": null, + "mileage": 10537795, + "last_update": "2023-02-26T10:02:03Z", + "ignition_total_time": 0, + "state": { + "name": "nodata", + "start": "2023-02-26T09:33:37Z", + "duration": 74289, + "debug_info": { + "boxId": 27953746, + "carId": 150662, + "msg": "OTHER", + "lastUpdate": 1677405723, + "lastValues": {"VOLTAGE": "14289"}, + }, + }, + "movement_state": { + "name": "nodata", + "start": "2023-02-26T09:33:37Z", + "duration": 75995, + }, + "fuel_type": "", + "avg_fuel_consumption": {"norm": 0, "measurement": "l/100km"}, + "created_at": "2019-11-05T10:39:46Z", + "io_din": [ + {"no": 1, "label": "Auraus", "state": 1}, + {"no": 2, "label": "Hiekoitus", "state": 0}, + {"no": 3, "label": "Muu ty\u00f6", "state": 0}, + ], + }, + { + "unit_id": 150662, + "box_id": 27953746, + "company_id": 5495, + "country_code": "FI", + "label": "1101049692", + "number": "TAKKU 1", + "shortcut": "", + "vehicle_title": null, + "car_reg_certificate": "", + "vin": null, + "type": "car", + "icon": "tractor", + "lat": 60.55185, + "lng": 22.20567, + "direction": 213, + "speed": null, + "mileage": 10537795, + "last_update": "2023-02-26T10:02:03Z", + "ignition_total_time": 0, + "state": { + "name": "nodata", + "start": "2023-02-26T09:33:37Z", + "duration": 74289, + "debug_info": { + "boxId": 27953746, + "carId": 150662, + "msg": "OTHER", + "lastUpdate": 1677405723, + "lastValues": {"VOLTAGE": "14289"}, + }, + }, + "movement_state": { + "name": "nodata", + "start": "2023-02-26T09:33:37Z", + "duration": 75995, + }, + "fuel_type": "", + "avg_fuel_consumption": {"norm": 0, "measurement": "l/100km"}, + "created_at": "2019-11-05T10:39:46Z", + "io_din": [ + {"no": 1, "label": "Auraus", "state": 1}, + {"no": 2, "label": "Hiekoitus", "state": 0}, + {"no": 3, "label": "Muu ty\u00f6", "state": 0}, + ], + }, + ] + assert num_elements <= len(units) + data = {"data": {"units": units[:num_elements]}} + return data From f61b5263f8de7426c32a7ef720716acc84b69aa5 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 08:08:27 +0300 Subject: [PATCH 07/66] Delete, moved to maintenance applicaiton --- street_maintenance/api/serializers.py | 64 --------- street_maintenance/api/urls.py | 25 ---- street_maintenance/api/views.py | 196 -------------------------- 3 files changed, 285 deletions(-) delete mode 100644 street_maintenance/api/serializers.py delete mode 100644 street_maintenance/api/urls.py delete mode 100644 street_maintenance/api/views.py diff --git a/street_maintenance/api/serializers.py b/street_maintenance/api/serializers.py deleted file mode 100644 index 6ae513acd..000000000 --- a/street_maintenance/api/serializers.py +++ /dev/null @@ -1,64 +0,0 @@ -from django.contrib.gis.geos import LineString, Point -from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema_field -from rest_framework import serializers - -from street_maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork - - -class GeometryHistorySerializer(serializers.ModelSerializer): - geometry_type = serializers.SerializerMethodField() - - class Meta: - model = GeometryHistory - fields = [ - "id", - "geometry_type", - "events", - "timestamp", - "provider", - # Removed for permormance issues as it is not currently used - # "geometry", - "coordinates", - ] - - @extend_schema_field(OpenApiTypes.STR) - def get_geometry_type(self, obj): - return obj.geometry.geom_type - - -class ActiveEventSerializer(serializers.Serializer): - events = serializers.CharField(max_length=64) - - -class MaintenanceUnitSerializer(serializers.ModelSerializer): - class Meta: - model = MaintenanceUnit - fields = "__all__" - - -class MaintenanceWorkSerializer(serializers.ModelSerializer): - provider = serializers.PrimaryKeyRelatedField( - many=False, source="maintenance_unit.provider", read_only=True - ) - - class Meta: - model = MaintenanceWork - fields = [ - "id", - "maintenance_unit", - "provider", - "geometry", - "timestamp", - "events", - "original_event_names", - ] - - def to_representation(self, obj): - representation = super().to_representation(obj) - if isinstance(obj.geometry, Point): - representation["lat"] = obj.geometry.y - representation["lon"] = obj.geometry.x - elif isinstance(obj.geometry, LineString): - representation["coords"] = obj.geometry.coords - return representation diff --git a/street_maintenance/api/urls.py b/street_maintenance/api/urls.py deleted file mode 100644 index 473e6e3c9..000000000 --- a/street_maintenance/api/urls.py +++ /dev/null @@ -1,25 +0,0 @@ -from django.urls import include, path -from rest_framework import routers - -from . import views - -app_name = "street_maintenance" - -router = routers.DefaultRouter() -router.register("active_events", views.ActiveEventsViewSet, basename="active_events") - -router.register( - "maintenance_works", views.MaintenanceWorkViewSet, basename="maintenance_works" -) -router.register( - "maintenance_units", views.MaintenanceUnitViewSet, basename="maintenance_units" -) - -router.register( - "geometry_history", views.GeometryHitoryViewSet, basename="geometry_history" -) - -urlpatterns = [ - # re_path("^street_maintenance/active_events", ) - path("", include(router.urls), name="street_maintenance"), -] diff --git a/street_maintenance/api/views.py b/street_maintenance/api/views.py deleted file mode 100644 index 87f6f9cef..000000000 --- a/street_maintenance/api/views.py +++ /dev/null @@ -1,196 +0,0 @@ -from datetime import datetime - -from django.utils.decorators import method_decorator -from django.utils.timezone import make_aware -from django.views.decorators.cache import cache_page -from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter -from rest_framework import mixins, viewsets -from rest_framework.exceptions import ParseError -from rest_framework.pagination import PageNumberPagination - -from street_maintenance.api.serializers import ( - ActiveEventSerializer, - GeometryHistorySerializer, - MaintenanceUnitSerializer, - MaintenanceWorkSerializer, -) -from street_maintenance.management.commands.constants import ( - EVENT_CHOICES, - PROVIDERS, - START_DATE_TIME_FORMAT, -) -from street_maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork - -EXAMPLE_TIME_FORMAT = "YYYY-MM-DD HH:MM:SS" -EXAMPLE_TIME = "2022-09-18 10:00:00" -EVENT_PARAM = OpenApiParameter( - name="event", - location=OpenApiParameter.QUERY, - description=( - "Return objects of given event. " - f'Event choices are: {", ".join(EVENT_CHOICES).lower()}, ' - 'E.g. "auraus".' - ), - required=False, - type=str, -) -PROVIDER_PARAM = OpenApiParameter( - name="provider", - location=OpenApiParameter.QUERY, - description=("Return objects of given provider. " 'E.g. "INFRAROAD".'), - required=False, - type=str, -) -START_DATE_TIME_PARAM = OpenApiParameter( - name="start_date_time", - location=OpenApiParameter.QUERY, - description=( - "Get objects with timestamp newer than the start_date_time. " - f'The format for the timestamp is: {EXAMPLE_TIME_FORMAT}, e.g. "{EXAMPLE_TIME}".' - ), - required=False, - type=str, -) - - -class LargeResultsSetPagination(PageNumberPagination): - """ - Custom pagination class to allow all results in one page. - """ - - page_size_query_param = "page_size" - # Works are fetched to the remote data storage on a single page, to prevent - # duplicates. - max_page_size = 200_000 - - -class ActiveEventsViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): - queryset = MaintenanceWork.objects.order_by().distinct("events") - serializer_class = ActiveEventSerializer - - -_maintenance_works_list_parameters = [ - EVENT_PARAM, - PROVIDER_PARAM, - START_DATE_TIME_PARAM, -] - - -@extend_schema_view( - list=extend_schema( - parameters=_maintenance_works_list_parameters, - description="A MaintenanceWork object is a single work performed by a provider. The geometry can be a point " - "or a linestring. The geometry and timestamp are assigned directly from the source data. The " - "original_event_names contains the names of the events in the source data and the events field is the mapped " - "names. Note, if the geometry is a point, the latitude and longitude will be separately serialized. If the " - "geometry is a linestring a separate list of coordinates will be serialized.", - ) -) -class MaintenanceWorkViewSet(viewsets.ReadOnlyModelViewSet): - serializer_class = MaintenanceWorkSerializer - pagination_class = LargeResultsSetPagination - - def get_queryset(self): - queryset = MaintenanceWork.objects.all() - filters = self.request.query_params - - if "provider" in filters: - provider = filters["provider"].upper() - if provider in PROVIDERS: - queryset = queryset.filter(maintenance_unit__provider=provider) - else: - raise ParseError(f"Providers are: {', '.join(PROVIDERS)}") - - if "event" in filters: - queryset = queryset.filter(events__contains=[filters["event"]]) - if "start_date_time" in filters: - start_date_time = filters["start_date_time"] - try: - start_date_time = datetime.strptime( - start_date_time, START_DATE_TIME_FORMAT - ) - except ValueError: - raise ParseError( - f"'start_date_time' must be in format {EXAMPLE_TIME_FORMAT} elem.g.,'{EXAMPLE_TIME}'" - ) - - queryset = queryset.filter(timestamp__gte=make_aware(start_date_time)) - return queryset - - def list(self, request): - queryset = self.get_queryset() - filters = self.request.query_params - if "unit_id" in filters: - unit_id = filters["unit_id"] - queryset = queryset.filter(maintenance_unit__unit_id=unit_id) - - page = self.paginate_queryset(queryset) - serializer = self.serializer_class(page, many=True) - return self.get_paginated_response(serializer.data) - - -@extend_schema_view( - list=extend_schema( - description="MaintananceUnit objets are the entities that creates the MaintenanceWorks. Every MaintenanceWork " - "has a relation to a MaintenanceUnit. The type of the MaintenanceUnit can vary depending on the provider. It " - "can be a machine or a event", - ), -) -class MaintenanceUnitViewSet(viewsets.ReadOnlyModelViewSet): - queryset = MaintenanceUnit.objects.all() - serializer_class = MaintenanceUnitSerializer - - -_geometry_history_list_parameters = [ - PROVIDER_PARAM, - EVENT_PARAM, - START_DATE_TIME_PARAM, -] - - -@extend_schema_view( - list=extend_schema( - parameters=_geometry_history_list_parameters, - description="GeometryHistory objects are processed from MaintenanceWork objects. MaintenanceWorks with " - "linestrings are validated and if valid a GeometryHistory object is created with the linestring geometry. " - "From MaintenanceWork objects that cointains point data linestrings are generated. The linestrings are " - "generated by comparing the timestamp, event, distance and provider. For every valid generated linestring a " - "GeometryHistory object is created. The coordinates are in SRID 4326.", - ) -) -class GeometryHitoryViewSet(viewsets.ReadOnlyModelViewSet): - serializer_class = GeometryHistorySerializer - pagination_class = LargeResultsSetPagination - - def get_queryset(self): - queryset = GeometryHistory.objects.all() - filters = self.request.query_params - - if "provider" in filters: - provider = filters["provider"].upper() - if provider in PROVIDERS: - queryset = queryset.filter(provider=provider) - else: - raise ParseError(f"Providers are: {', '.join(PROVIDERS)}") - - if "event" in filters: - queryset = queryset.filter(events__contains=[filters["event"]]) - if "start_date_time" in filters: - start_date_time = filters["start_date_time"] - try: - start_date_time = datetime.strptime( - start_date_time, "%Y-%m-%d %H:%M:%S" - ) - except ValueError: - raise ParseError( - f"'start_date_time' must be in format {EXAMPLE_TIME_FORMAT} elem.g.,'{EXAMPLE_TIME}'" - ) - queryset = queryset.filter(timestamp__gte=make_aware(start_date_time)) - return queryset - - @method_decorator(cache_page(60 * 15)) - def list(self, request): - queryset = self.get_queryset() - page = self.paginate_queryset(queryset) - serializer = self.serializer_class(page, many=True) - return self.get_paginated_response(serializer.data) From 1534e760932cbee6c439e328e226f73c39e1d716 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:06:41 +0300 Subject: [PATCH 08/66] Delete, moved to maintenance app --- street_maintenance/README.md | 43 ---- street_maintenance/admin.py | 6 - street_maintenance/specificatio.swagger.yaml | 230 ------------------- street_maintenance/tasks.py | 28 --- 4 files changed, 307 deletions(-) delete mode 100644 street_maintenance/README.md delete mode 100644 street_maintenance/admin.py delete mode 100644 street_maintenance/specificatio.swagger.yaml delete mode 100644 street_maintenance/tasks.py diff --git a/street_maintenance/README.md b/street_maintenance/README.md deleted file mode 100644 index adc27f18e..000000000 --- a/street_maintenance/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Street Maintenance history - -Django app for importing and serving street maintenance data. - -## Importer -Name: -import_street_maintenance_history - -Providers: -* YIT -* KUNTEC -* INFRAROAD -* DESTIA - -Parameters: -* --providers, list of providers to import, e.g., --provider yit kuntec -* --history-size, the number of days to import (default is 4 and max is 31) -* --fetch-size (only available for infraroad and destia), the number of works to import per unit(default is 10000). - -### Examples: -To import DESTIA street maintenance history with history size 2: -``` -./manage.py import_street_maintenance_history --providers destia --history-size 2 -``` -To import KUNTEC and INFRAROAD street maintenance history: -``` -./manage.py import_street_maintenance_history --providers kuntec infraroad -``` -Note, only the MaintenanceWorks and MaintenanceUnits for the given provider from the latest import are stored and the rest are deleted. The GeometryHistory is generated only if more than one MaintenanceWork is created. - -### Periodically imorting -To periodically import data use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#street-maintenance-history-street_maintenancetasksimport_street_maintenance_history). - - -## Deleting street maintenance history for a provider -It is possible to delete street maintenance history for a provider. -e.g., to delete all street maintenance history for provider 'destia': -``` -./manage.py delete_street_maintenance_history destia -``` - -## API -See: specificatin.swagger.yaml \ No newline at end of file diff --git a/street_maintenance/admin.py b/street_maintenance/admin.py deleted file mode 100644 index 19154fd8d..000000000 --- a/street_maintenance/admin.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.contrib import admin - -from street_maintenance.models import MaintenanceUnit, MaintenanceWork - -admin.site.register(MaintenanceWork) -admin.site.register(MaintenanceUnit) diff --git a/street_maintenance/specificatio.swagger.yaml b/street_maintenance/specificatio.swagger.yaml deleted file mode 100644 index c1064ebdc..000000000 --- a/street_maintenance/specificatio.swagger.yaml +++ /dev/null @@ -1,230 +0,0 @@ -swagger: "2.0" - -info: - description: "Street maintenance API that serves history data of maintenance works, active events and provides history as generated geometries." - version: "1.0.0" - title: "Street Maintenance History" - -schemes: -- "https" -- "http" - -definitions: - geometry_history: - type: object - title: GeometryHistory - description: "Precalculated geometry history." - properties: - id: - type: integer - geometry_type: - type: string - events: - type: array - description: "Name of the events." - items: - type: string - timestamp: - type: string - provider: - type: string - gemoetry: - type: string - coordinates: - type: array - items: - array: - type: number - - maintenance_unit: - type: object - title: MaintenanceUnit - description: "The maintenace unit." - properties: - id: - type: integer - unit_id: - type: integer - description: "The id(name) of the unit." - geometry_type: - type: string - description: "The type of the geometry" - events: - type: array - description: "Name of the events." - items: - type: string - timestamp: - type: string - provider: - type: string - description: Name of the provider. - geometry: - type: string - coordinates: - description: "Returns if geometry is of type Linestring." - type: array - items: - type: number - - maintenance_work: - type: object - title: MaintenanceWork - description: "The isolated work performed by the unit." - properties: - id: - type: integer - geometry: - type: string - timestamp: - type: string - maintenance_unit: - type: integer - events: - type: array - description: "Name of the events." - items: - type: string - coords: - description: "Returns if geometry is of type Linestring." - type: array - items: - type: number - lat: - description: "Returns if geometry is of type Point." - type: number - lon: - description: "Returns if geometry is of type Point." - type: number - -paths: - /geometry_history/: - get: - summary: "Returns list of precalculated geometry historys." - parameters: - - $ref: "#/components/parameters/provider_param" - - $ref: "#/components/parameters/event_param" - - $ref: "#/components/parameters/start_date_time_param" - responses: - 200: - description: "List of GeometryHistory object" - schema: - type: object - properties: - results: - type: array - items: - $ref: "#/definitions/geometry_history" - - /geometry_history/{id}: - get: - summary: "Return a single precalculated geometry history object." - parameters: - - name: "id" - in: path - type: integer - required: true - responses: - 200: - description: "The geometry history object." - schema: - $ref: "#/definitions/geometry_history" - - /maintenance_units/: - get: - summary: "Returns list of maintenance units." - responses: - 200: - description: "List of maintenance units." - schema: - type: object - properties: - results: - type: array - items: - $ref: "#/definitions/maintenance_unit" - /maintenance_units/{id}: - get: - summary: "Return a maintenance unit by id." - parameters: - - name: "id" - in: path - type: integer - required: true - responses: - 200: - description: "Maintenance unit object." - schema: - $ref: "#/definitions/maintenance_unit" - - /maintenance_works/: - get: - summary: "Returns list of maintenance works. A work is a single work event." - parameters: - - $ref: "#/components/parameters/event_param" - - $ref: "#/components/parameters/start_date_time_param" - - $ref: "#/components/parameters/unit_id_param" - responses: - 200: - description: "List of maintenance works" - schema: - type: object - properties: - results: - type: array - items: - $ref: "#/definitions/maintenance_work" - /maintenance_works/{id}: - get: - summary: "Return a single maintenace work." - parameters: - - name: "id" - in: path - type: integer - required: true - responses: - 200: - description: "The maintenance work object." - schema: - $ref: "#/definitions/maintenance_work" - - /active_events/: - get: - summary: "Returns a list of active events with their mapped names. i.e., event names that are shown in the front end, not the names that are in the source data." - responses: - 200: - description: "List of active events." - -components: - parameters: - provider_param: - name: provider - in: query - description: "Return entities of given provider." - event_param: - name: event - in: query - description: "Return entities of given event. The event name is the mapped name and the active event names can be seen from the active_events endpoint." - schema: - type: string - start_date_time_param: - name: start_date_time - in: query - description: "The start date and time of the entities to fetch. Must be in format YYYY-MM-DD HH:MM:SS e.g.,'2022-09-18 10:00:00' " - schema: - type: string - unit_id_param: - name: unit_id - in: query - description: "The 'unit_id' of the maintenance unit" - schema: - type: integer - max_work_length_param: - name: max_work_length - in: query - description: "The max work length in Seconds of one uniform work. I.e, if the delta time of two works timestamps are greated than the max work length it will end the linestring and create a new. Changing the value can be usefull for different events as their length of the uniform work can vary." - default: 1800 - schema: - type: integer - - \ No newline at end of file diff --git a/street_maintenance/tasks.py b/street_maintenance/tasks.py deleted file mode 100644 index 245186e93..000000000 --- a/street_maintenance/tasks.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.core import management - -from smbackend.utils import shared_task_email - - -@shared_task_email -def delete_street_maintenance_history(*args, name="delete_street_maintenance_history"): - management.call_command("delete_street_maintenance_history", *args) - - -@shared_task_email -def import_street_maintenance_history( - name="import_street_maintenance_history", *args, **kwargs -): - if "providers" not in kwargs: - raise Exception( - "No 'providers' item in kwargs. e.g., {'providers':['destia', 'infraroad']}" - ) - if "fetch-size" not in kwargs: - kwargs["fetch-size"] = None - if "history-size" not in kwargs: - kwargs["history-size"] = None - management.call_command( - "import_street_maintenance_history", - providers=kwargs["providers"], - fetch_size=kwargs["fetch-size"], - history_size=kwargs["history-size"], - ) From f10ca1cc02455e20de6752ed0c791b39bfec8140 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:45:19 +0300 Subject: [PATCH 09/66] Change street_maintenance to maintenance in DOC_ENDPOINTS --- smbackend/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index 7d5110859..7283d55f9 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -350,9 +350,9 @@ def gettext(s): # Define the endpoints for API documentation with drf-spectacular. DOC_ENDPOINTS = [ - "/street_maintenance/geometry_history/", - "/street_maintenance/maintenance_works/", - "/street_maintenance/maintenance_units/", + "/maintenance/geometry_history/", + "/maintenance/maintenance_works/", + "/maintenance/maintenance_units/", "/environment_data/api/v1/stations/", "/environment_data/api/v1/parameters/", "/environment_data/api/v1/data/", From 1ff9f2ac62fe063c0d4b31bc621a324828b8edb8 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:51:15 +0300 Subject: [PATCH 10/66] Add tests from street_maintenance application --- maintenance/tests/conftest.py | 5 +++-- maintenance/tests/test_api.py | 1 - maintenance/tests/test_importers.py | 4 +--- maintenance/tests/utils.py | 6 +----- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index e7b138226..76ae687e5 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -10,7 +10,6 @@ ) from rest_framework.test import APIClient -from mobility_data.tests.conftest import TURKU_WKT from maintenance.management.commands.constants import ( AURAUS, INFRAROAD, @@ -18,6 +17,7 @@ LIUKKAUDENTORJUNTA, ) from maintenance.models import DEFAULT_SRID, GeometryHistory +from mobility_data.tests.conftest import TURKU_WKT UTC_TIMEZONE = pytz.timezone("UTC") @@ -66,9 +66,10 @@ def geometry_historys(): coordinates=geometry.coords, provider=KUNTEC, events=[AURAUS, LIUKKAUDENTORJUNTA], - ) + ) return GeometryHistory.objects.all() + @pytest.mark.django_db @pytest.fixture def administrative_division_type(): diff --git a/maintenance/tests/test_api.py b/maintenance/tests/test_api.py index a60725b0a..02ab75b00 100644 --- a/maintenance/tests/test_api.py +++ b/maintenance/tests/test_api.py @@ -17,7 +17,6 @@ def test_geometry_history_list(api_client, geometry_historys): url = reverse("maintenance:geometry_history-list") response = api_client.get(url) - breakpoint() assert response.json()["count"] == 5 diff --git a/maintenance/tests/test_importers.py b/maintenance/tests/test_importers.py index 6cfebb169..51dd96353 100644 --- a/maintenance/tests/test_importers.py +++ b/maintenance/tests/test_importers.py @@ -24,9 +24,7 @@ def test_yit_units( administrative_division, administrative_division_geometry, ): - from maintenance.management.commands.utils import ( - create_yit_maintenance_units, - ) + from maintenance.management.commands.utils import create_yit_maintenance_units get_yit_vehicles_mock.return_value = get_yit_vehicles_mock_data(2) num_created_units, num_del_units = create_yit_maintenance_units("test_access_token") diff --git a/maintenance/tests/utils.py b/maintenance/tests/utils.py index 6d402f676..0306c8d79 100644 --- a/maintenance/tests/utils.py +++ b/maintenance/tests/utils.py @@ -1,10 +1,6 @@ from datetime import datetime -from maintenance.management.commands.constants import ( - DATE_FORMATS, - INFRAROAD, - YIT, -) +from maintenance.management.commands.constants import DATE_FORMATS, INFRAROAD, YIT def get_yit_vehicles_mock_data(num_elements): From 8a6f9e213d294dd472a00d3cd1620aa51834a638 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:52:24 +0300 Subject: [PATCH 11/66] Add __init__.py --- maintenance/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 maintenance/__init__.py diff --git a/maintenance/__init__.py b/maintenance/__init__.py new file mode 100644 index 000000000..e69de29bb From bd1d664a764e75af74bd4d50f19b5934c73a1a5c Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:14:04 +0300 Subject: [PATCH 12/66] Add __init__.py --- maintenance/migrations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 maintenance/migrations/__init__.py diff --git a/maintenance/migrations/__init__.py b/maintenance/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From 9e4f796b2a863e449ee5215d6eb644cf687ccc62 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:14:46 +0300 Subject: [PATCH 13/66] Add tasks.py from street_maintenance application --- maintenance/tasks.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 maintenance/tasks.py diff --git a/maintenance/tasks.py b/maintenance/tasks.py new file mode 100644 index 000000000..245186e93 --- /dev/null +++ b/maintenance/tasks.py @@ -0,0 +1,28 @@ +from django.core import management + +from smbackend.utils import shared_task_email + + +@shared_task_email +def delete_street_maintenance_history(*args, name="delete_street_maintenance_history"): + management.call_command("delete_street_maintenance_history", *args) + + +@shared_task_email +def import_street_maintenance_history( + name="import_street_maintenance_history", *args, **kwargs +): + if "providers" not in kwargs: + raise Exception( + "No 'providers' item in kwargs. e.g., {'providers':['destia', 'infraroad']}" + ) + if "fetch-size" not in kwargs: + kwargs["fetch-size"] = None + if "history-size" not in kwargs: + kwargs["history-size"] = None + management.call_command( + "import_street_maintenance_history", + providers=kwargs["providers"], + fetch_size=kwargs["fetch-size"], + history_size=kwargs["history-size"], + ) From d52b36e2faa9a8e54ac93eca0e8eda8c152bd68a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:21:02 +0300 Subject: [PATCH 14/66] Add maintenance AppConfig --- maintenance/apps.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 maintenance/apps.py diff --git a/maintenance/apps.py b/maintenance/apps.py new file mode 100644 index 000000000..c386a4051 --- /dev/null +++ b/maintenance/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class MaintenanceConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "maintenance" From 1f34e1d658337e1b271d350b210f808503844b99 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:27:12 +0300 Subject: [PATCH 15/66] Delete all street_maintenance application tables --- .../0014_delete_geometryhistory_and_more.py | 26 ++++++++++ street_maintenance/models.py | 49 ------------------- 2 files changed, 26 insertions(+), 49 deletions(-) create mode 100644 street_maintenance/migrations/0014_delete_geometryhistory_and_more.py delete mode 100644 street_maintenance/models.py diff --git a/street_maintenance/migrations/0014_delete_geometryhistory_and_more.py b/street_maintenance/migrations/0014_delete_geometryhistory_and_more.py new file mode 100644 index 000000000..cde893e9e --- /dev/null +++ b/street_maintenance/migrations/0014_delete_geometryhistory_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.15 on 2024-09-18 07:48 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("street_maintenance", "0013_maintenancework_change_related_name"), + ] + + operations = [ + migrations.DeleteModel( + name="GeometryHistory", + ), + migrations.RemoveField( + model_name="maintenancework", + name="maintenance_unit", + ), + migrations.DeleteModel( + name="MaintenanceUnit", + ), + migrations.DeleteModel( + name="MaintenanceWork", + ), + ] diff --git a/street_maintenance/models.py b/street_maintenance/models.py deleted file mode 100644 index 3591c3117..000000000 --- a/street_maintenance/models.py +++ /dev/null @@ -1,49 +0,0 @@ -from django.contrib.gis.db import models -from django.contrib.postgres.fields import ArrayField - -from street_maintenance.management.commands.constants import PROVIDER_CHOICES - -DEFAULT_SRID = 4326 - - -class MaintenanceUnit(models.Model): - - unit_id = models.CharField(max_length=64, null=True) - provider = models.CharField(max_length=16, choices=PROVIDER_CHOICES, null=True) - names = ArrayField(models.CharField(max_length=64), default=list) - - def __str__(self): - return "%s" % (self.unit_id) - - -class MaintenanceWork(models.Model): - geometry = models.GeometryField(srid=DEFAULT_SRID, null=True) - events = ArrayField(models.CharField(max_length=64), default=list) - original_event_names = ArrayField(models.CharField(max_length=64), default=list) - timestamp = models.DateTimeField() - maintenance_unit = models.ForeignKey( - "MaintenanceUnit", - on_delete=models.CASCADE, - related_name="maintenance_work", - null=True, - ) - - def __str__(self): - return "%s %s" % (self.timestamp, self.events) - - class Meta: - ordering = ["-timestamp"] - - -class GeometryHistory(models.Model): - timestamp = models.DateTimeField() - geometry = models.GeometryField(srid=DEFAULT_SRID, null=True) - coordinates = ArrayField(ArrayField(models.FloatField()), default=list) - events = ArrayField(models.CharField(max_length=64), default=list) - provider = models.CharField(max_length=16, choices=PROVIDER_CHOICES, null=True) - - def __str__(self): - return "%s %s" % (self.timestamp, self.events) - - class Meta: - ordering = ["-timestamp"] From 27ba44bacc2f1154376145ac4b8e228cc9290db1 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:30:46 +0300 Subject: [PATCH 16/66] Add constants.py from street_maintenance --- maintenance/management/commands/constants.py | 193 +++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 maintenance/management/commands/constants.py diff --git a/maintenance/management/commands/constants.py b/maintenance/management/commands/constants.py new file mode 100644 index 000000000..beb9adde5 --- /dev/null +++ b/maintenance/management/commands/constants.py @@ -0,0 +1,193 @@ +import types + +from django.conf import settings + +KUNTEC_KEY = settings.KUNTEC_KEY +INFRAROAD = "INFRAROAD" +YIT = "YIT" +KUNTEC = "KUNTEC" +DESTIA = "DESTIA" +PROVIDER_CHOICES = ( + (INFRAROAD, "Infraroad"), + (YIT, "YIT"), + (KUNTEC, "Kuntec"), + (DESTIA, "Destia"), +) +PROVIDERS = [INFRAROAD, YIT, KUNTEC, DESTIA] +PROVIDER_TYPES = types.SimpleNamespace() +PROVIDER_TYPES.YIT = YIT +PROVIDER_TYPES.KUNTEC = KUNTEC +PROVIDER_TYPES.DESTIA = DESTIA +PROVIDER_TYPES.INFRAROAD = INFRAROAD + +UNITS = "UNITS" +WORKS = "WORKS" +EVENTS = "EVENTS" +ROUTES = "ROUTES" +VEHICLES = "VEHICLES" +CONTRACTS = "CONTRACTS" +TOKEN = "TOKEN" + + +URLS = { + KUNTEC: { + WORKS: "https://mapon.com/api/v1/route/list.json?key={key}&from={start}" + "&till={end}&include=polyline&unit_id={unit_id}", + UNITS: f"https://mapon.com/api/v1/unit/list.json?key={KUNTEC_KEY}&include=io_din", + }, + YIT: { + EVENTS: settings.YIT_EVENTS_URL, + ROUTES: settings.YIT_ROUTES_URL, + VEHICLES: settings.YIT_VEHICLES_URL, + CONTRACTS: settings.YIT_CONTRACTS_URL, + TOKEN: settings.YIT_TOKEN_URL, + }, + INFRAROAD: { + WORKS: "https://infraroad.fluentprogress.fi/KuntoInfraroad/v1/snowplow/{id}?history={history_size}", + UNITS: "https://infraroad.fluentprogress.fi/KuntoInfraroad/v1/snowplow/query?since=72hours", + }, + DESTIA: { + WORKS: "https://destia.fluentprogress.fi/KuntoDestia/turku/v1/snowplow/{id}?history={history_size}", + UNITS: "https://destia.fluentprogress.fi/KuntoDestia/turku/v1/snowplow/query?since=72hours", + }, +} + + +# Events are categorized into main groups: +AURAUS = "auraus" +LIUKKAUDENTORJUNTA = "liukkaudentorjunta" +PUHTAANAPITO = "puhtaanapito" +HIEKANPOISTO = "hiekanpoisto" +# MUUT is set to None as the events are not currently displayed. +MUUT = None +EVENT_CHOICES = [AURAUS, LIUKKAUDENTORJUNTA, PUHTAANAPITO, HIEKANPOISTO] +# As data providers have different names for their events, they are mapped +# with this dict, so that every event that does the same has the same name. +# The value is a list, as there can be events that belong to multiple main groups. +# e.g., event "Auraus ja hiekanpoisto". +EVENT_MAPPINGS = { + "käsihiekotus tai käsilumityöt": [AURAUS, LIUKKAUDENTORJUNTA], + "laiturin ja asema-alueen auraus": [AURAUS], + "au": [AURAUS], + "auraus": [AURAUS], + "auraus ja sohjonpoisto": [AURAUS], + "lumen poisajo": [AURAUS], + "lumensiirto": [AURAUS], + "etuaura": [AURAUS], + "alusterä": [AURAUS], + "sivuaura": [AURAUS], + "höyläys": [AURAUS], + "suolaus": [LIUKKAUDENTORJUNTA], + "suolas": [LIUKKAUDENTORJUNTA], + "suolaus (sirotinlaite)": [LIUKKAUDENTORJUNTA], + "liuossuolaus": [LIUKKAUDENTORJUNTA], + # Hiekoitus + "hi": [LIUKKAUDENTORJUNTA], + "su": [LIUKKAUDENTORJUNTA], + "hiekoitus": [LIUKKAUDENTORJUNTA], + "hiekotus": [LIUKKAUDENTORJUNTA], + "hiekoitus (sirotinlaite)": [LIUKKAUDENTORJUNTA], + "linjahiekoitus": [LIUKKAUDENTORJUNTA], + "pistehiekoitus": [LIUKKAUDENTORJUNTA], + "paannejään poisto": [LIUKKAUDENTORJUNTA], + "sirotin": [LIUKKAUDENTORJUNTA], + "laiturin ja asema-alueen liukkaudentorjunta": [LIUKKAUDENTORJUNTA], + "liukkauden torjunta": [LIUKKAUDENTORJUNTA], + # Kadunpesu + "pe": [PUHTAANAPITO], + # Harjaus + "hj": [PUHTAANAPITO], + # Hiekanpoisto + "hn": [PUHTAANAPITO], + "puhtaanapito": [PUHTAANAPITO], + "harjaus": [PUHTAANAPITO], + "pesu": [PUHTAANAPITO], + "harjaus ja sohjonpoisto": [PUHTAANAPITO], + "pölynsidonta": [PUHTAANAPITO], + "Imulakaisu": [PUHTAANAPITO], + "hiekanpoisto": [HIEKANPOISTO], + "lakaisu": [HIEKANPOISTO], + "muu": [MUUT], + "muut työt": [MUUT], + "muu työ": [MUUT], + "lisätyö": [MUUT], + "viherhoito": [MUUT], + "kaivot": [MUUT], + "metsätyöt": [MUUT], + "rikkakasvien torjunta": [MUUT], + "paikkaus": [MUUT], + "lisälaite 1": [MUUT], + "lisälaite 2": [MUUT], + "lisälaite 3": [MUUT], + "pensaiden hoitoleikkaus": [MUUT], + "puiden hoitoleikkaukset": [MUUT], + "mittaus- ja tarkastustyöt": [MUUT], + "siimaleikkurointi tai niittotyö": [MUUT], + "liikennemerkkien pesu": [MUUT], + "tiestötarkastus": [MUUT], + "roskankeräys": [MUUT], + "tuntityö": [MUUT], + "pinnan tasaus": [MUUT], + "lumivallien madaltaminen": [MUUT], + "aurausviitoitus ja kinostimet": [MUUT], + "jääkenttien hoito": [MUUT], + "leikkipaikkojen tarkastus": [MUUT], + "kenttien hoito": [MUUT], + "murskeen ajo varastoihin": [MUUT], + "huoltoteiden kunnossapito": [MUUT], + "pysäkkikatosten hoito": [MUUT], + "liikennemerkkien puhdistus": [MUUT], + "siirtoajo": [MUUT], + "Kelintarkastus": [MUUT], + "Sulamisvesien hallinta / höyrytys": [MUUT], + "Sulamisveden haittojen torjunta": [MUUT], + "Sorateiden kunnossapito": [MUUT], + "Äkillinen hoitotyö": [MUUT], + "KT-valu": [MUUT], + "Pintakelirikko": [MUUT], + "Liikennemerkkityö": [MUUT], + "Puiden hoito": [MUUT], + "Puistometsien hoito": [MUUT], + "Hiekkalaatikoiden täyttö": [MUUT], +} +TIMESTAMP_FORMATS = { + INFRAROAD: "%Y-%m-%d %H:%M:%S", + DESTIA: "%Y-%m-%d %H:%M:%S", + KUNTEC: "%Y-%m-%dT%H:%M:%SZ", + YIT: "%Y-%m-%d %H:%M:%S%z", +} +DATE_FORMATS = { + INFRAROAD: "%Y-%m-%d", + DESTIA: "%Y-%m-%d", + KUNTEC: "%Y-%m-%d", + YIT: "%Y-%m-%d", +} +# GeometryHistory API list start_date_time parameter format. +START_DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" +# The number of works(point data with a timestamp and events) to be fetched for every unit. +INFRAROAD_DEFAULT_WORKS_FETCH_SIZE = 10000 +DESTIA_DEFAULT_WORKS_FETCH_SIZE = 10000 +# In days, Note if value is increased the fetch size should also be increased. +INFRAROAD_DEFAULT_WORKS_HISTORY_SIZE = 4 +DESTIA_DEFAULT_WORKS_HISTORY_SIZE = 4 + +# Length of YIT history size in days, max value is 31. +YIT_DEFAULT_WORKS_HISTORY_SIZE = 4 +YIT_MAX_WORKS_HISTORY_SIZE = 31 + +KUNTEC_DEFAULT_WORKS_HISTORY_SIZE = 4 +KUNTEC_MAX_WORKS_HISTORY_SIZE = 31 +HISTORY_SIZE = "history_size" +FETCH_SIZE = "fetch_size" +HISTORY_SIZES = { + INFRAROAD: { + HISTORY_SIZE: INFRAROAD_DEFAULT_WORKS_HISTORY_SIZE, + FETCH_SIZE: INFRAROAD_DEFAULT_WORKS_FETCH_SIZE, + }, + DESTIA: { + HISTORY_SIZE: DESTIA_DEFAULT_WORKS_HISTORY_SIZE, + FETCH_SIZE: DESTIA_DEFAULT_WORKS_FETCH_SIZE, + }, + KUNTEC: {HISTORY_SIZE: KUNTEC_DEFAULT_WORKS_HISTORY_SIZE}, + YIT: {HISTORY_SIZE: YIT_DEFAULT_WORKS_HISTORY_SIZE}, +} From 6111a72b575964e1ad2fc86deec654a416881c83 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:34:03 +0300 Subject: [PATCH 17/66] Add models from street_maintenance application --- maintenance/migrations/0001_initial.py | 156 +++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 maintenance/migrations/0001_initial.py diff --git a/maintenance/migrations/0001_initial.py b/maintenance/migrations/0001_initial.py new file mode 100644 index 000000000..ba551a92f --- /dev/null +++ b/maintenance/migrations/0001_initial.py @@ -0,0 +1,156 @@ +# Generated by Django 4.2.15 on 2024-09-17 07:26 + +import django.contrib.gis.db.models.fields +import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="GeometryHistory", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("timestamp", models.DateTimeField()), + ( + "geometry", + django.contrib.gis.db.models.fields.GeometryField( + null=True, srid=4326 + ), + ), + ( + "coordinates", + django.contrib.postgres.fields.ArrayField( + base_field=django.contrib.postgres.fields.ArrayField( + base_field=models.FloatField(), size=None + ), + default=list, + size=None, + ), + ), + ( + "events", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=64), + default=list, + size=None, + ), + ), + ( + "provider", + models.CharField( + choices=[ + ("INFRAROAD", "Infraroad"), + ("YIT", "YIT"), + ("KUNTEC", "Kuntec"), + ("DESTIA", "Destia"), + ], + max_length=16, + null=True, + ), + ), + ], + options={ + "ordering": ["-timestamp"], + }, + ), + migrations.CreateModel( + name="MaintenanceUnit", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("unit_id", models.CharField(max_length=64, null=True)), + ( + "provider", + models.CharField( + choices=[ + ("INFRAROAD", "Infraroad"), + ("YIT", "YIT"), + ("KUNTEC", "Kuntec"), + ("DESTIA", "Destia"), + ], + max_length=16, + null=True, + ), + ), + ( + "names", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=64), + default=list, + size=None, + ), + ), + ], + ), + migrations.CreateModel( + name="MaintenanceWork", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "geometry", + django.contrib.gis.db.models.fields.GeometryField( + null=True, srid=4326 + ), + ), + ( + "events", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=64), + default=list, + size=None, + ), + ), + ( + "original_event_names", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=64), + default=list, + size=None, + ), + ), + ("timestamp", models.DateTimeField()), + ( + "maintenance_unit", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="maintenance_work", + to="maintenance.maintenanceunit", + ), + ), + ], + options={ + "ordering": ["-timestamp"], + }, + ), + ] From fec6ada47a7174c0e8c6dbe0cf61399eeb7f558c Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:38:14 +0300 Subject: [PATCH 18/66] Add API from street_maintenace application --- maintenance/api/serializers.py | 64 +++++++++++ maintenance/api/urls.py | 24 ++++ maintenance/api/views.py | 196 +++++++++++++++++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 maintenance/api/serializers.py create mode 100644 maintenance/api/urls.py create mode 100644 maintenance/api/views.py diff --git a/maintenance/api/serializers.py b/maintenance/api/serializers.py new file mode 100644 index 000000000..df17c89a9 --- /dev/null +++ b/maintenance/api/serializers.py @@ -0,0 +1,64 @@ +from django.contrib.gis.geos import LineString, Point +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork + + +class GeometryHistorySerializer(serializers.ModelSerializer): + geometry_type = serializers.SerializerMethodField() + + class Meta: + model = GeometryHistory + fields = [ + "id", + "geometry_type", + "events", + "timestamp", + "provider", + # Removed for permormance issues as it is not currently used + # "geometry", + "coordinates", + ] + + @extend_schema_field(OpenApiTypes.STR) + def get_geometry_type(self, obj): + return obj.geometry.geom_type + + +class ActiveEventSerializer(serializers.Serializer): + events = serializers.CharField(max_length=64) + + +class MaintenanceUnitSerializer(serializers.ModelSerializer): + class Meta: + model = MaintenanceUnit + fields = "__all__" + + +class MaintenanceWorkSerializer(serializers.ModelSerializer): + provider = serializers.PrimaryKeyRelatedField( + many=False, source="maintenance_unit.provider", read_only=True + ) + + class Meta: + model = MaintenanceWork + fields = [ + "id", + "maintenance_unit", + "provider", + "geometry", + "timestamp", + "events", + "original_event_names", + ] + + def to_representation(self, obj): + representation = super().to_representation(obj) + if isinstance(obj.geometry, Point): + representation["lat"] = obj.geometry.y + representation["lon"] = obj.geometry.x + elif isinstance(obj.geometry, LineString): + representation["coords"] = obj.geometry.coords + return representation diff --git a/maintenance/api/urls.py b/maintenance/api/urls.py new file mode 100644 index 000000000..9edfae900 --- /dev/null +++ b/maintenance/api/urls.py @@ -0,0 +1,24 @@ +from django.urls import include, path +from rest_framework import routers + +from . import views + +app_name = "maintenance" + +router = routers.DefaultRouter() +router.register("active_events", views.ActiveEventsViewSet, basename="active_events") + +router.register( + "maintenance_works", views.MaintenanceWorkViewSet, basename="maintenance_works" +) +router.register( + "maintenance_units", views.MaintenanceUnitViewSet, basename="maintenance_units" +) + +router.register( + "geometry_history", views.GeometryHitoryViewSet, basename="geometry_history" +) + +urlpatterns = [ + path("", include(router.urls), name="maintenance"), +] diff --git a/maintenance/api/views.py b/maintenance/api/views.py new file mode 100644 index 000000000..ef5fc6437 --- /dev/null +++ b/maintenance/api/views.py @@ -0,0 +1,196 @@ +from datetime import datetime + +from django.utils.decorators import method_decorator +from django.utils.timezone import make_aware +from django.views.decorators.cache import cache_page +from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter +from rest_framework import mixins, viewsets +from rest_framework.exceptions import ParseError +from rest_framework.pagination import PageNumberPagination + +from maintenance.api.serializers import ( + ActiveEventSerializer, + GeometryHistorySerializer, + MaintenanceUnitSerializer, + MaintenanceWorkSerializer, +) +from maintenance.management.commands.constants import ( + EVENT_CHOICES, + PROVIDERS, + START_DATE_TIME_FORMAT, +) +from maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork + +EXAMPLE_TIME_FORMAT = "YYYY-MM-DD HH:MM:SS" +EXAMPLE_TIME = "2022-09-18 10:00:00" +EVENT_PARAM = OpenApiParameter( + name="event", + location=OpenApiParameter.QUERY, + description=( + "Return objects of given event. " + f'Event choices are: {", ".join(EVENT_CHOICES).lower()}, ' + 'E.g. "auraus".' + ), + required=False, + type=str, +) +PROVIDER_PARAM = OpenApiParameter( + name="provider", + location=OpenApiParameter.QUERY, + description=("Return objects of given provider. " 'E.g. "INFRAROAD".'), + required=False, + type=str, +) +START_DATE_TIME_PARAM = OpenApiParameter( + name="start_date_time", + location=OpenApiParameter.QUERY, + description=( + "Get objects with timestamp newer than the start_date_time. " + f'The format for the timestamp is: {EXAMPLE_TIME_FORMAT}, e.g. "{EXAMPLE_TIME}".' + ), + required=False, + type=str, +) + + +class LargeResultsSetPagination(PageNumberPagination): + """ + Custom pagination class to allow all results in one page. + """ + + page_size_query_param = "page_size" + # Works are fetched to the remote data storage on a single page, to prevent + # duplicates. + max_page_size = 200_000 + + +class ActiveEventsViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): + queryset = MaintenanceWork.objects.order_by().distinct("events") + serializer_class = ActiveEventSerializer + + +_maintenance_works_list_parameters = [ + EVENT_PARAM, + PROVIDER_PARAM, + START_DATE_TIME_PARAM, +] + + +@extend_schema_view( + list=extend_schema( + parameters=_maintenance_works_list_parameters, + description="A MaintenanceWork object is a single work performed by a provider. The geometry can be a point " + "or a linestring. The geometry and timestamp are assigned directly from the source data. The " + "original_event_names contains the names of the events in the source data and the events field is the mapped " + "names. Note, if the geometry is a point, the latitude and longitude will be separately serialized. If the " + "geometry is a linestring a separate list of coordinates will be serialized.", + ) +) +class MaintenanceWorkViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = MaintenanceWorkSerializer + pagination_class = LargeResultsSetPagination + + def get_queryset(self): + queryset = MaintenanceWork.objects.all() + filters = self.request.query_params + + if "provider" in filters: + provider = filters["provider"].upper() + if provider in PROVIDERS: + queryset = queryset.filter(maintenance_unit__provider=provider) + else: + raise ParseError(f"Providers are: {', '.join(PROVIDERS)}") + + if "event" in filters: + queryset = queryset.filter(events__contains=[filters["event"]]) + if "start_date_time" in filters: + start_date_time = filters["start_date_time"] + try: + start_date_time = datetime.strptime( + start_date_time, START_DATE_TIME_FORMAT + ) + except ValueError: + raise ParseError( + f"'start_date_time' must be in format {EXAMPLE_TIME_FORMAT} elem.g.,'{EXAMPLE_TIME}'" + ) + + queryset = queryset.filter(timestamp__gte=make_aware(start_date_time)) + return queryset + + def list(self, request): + queryset = self.get_queryset() + filters = self.request.query_params + if "unit_id" in filters: + unit_id = filters["unit_id"] + queryset = queryset.filter(maintenance_unit__unit_id=unit_id) + + page = self.paginate_queryset(queryset) + serializer = self.serializer_class(page, many=True) + return self.get_paginated_response(serializer.data) + + +@extend_schema_view( + list=extend_schema( + description="MaintananceUnit objets are the entities that creates the MaintenanceWorks. Every MaintenanceWork " + "has a relation to a MaintenanceUnit. The type of the MaintenanceUnit can vary depending on the provider. It " + "can be a machine or an event.", + ), +) +class MaintenanceUnitViewSet(viewsets.ReadOnlyModelViewSet): + queryset = MaintenanceUnit.objects.all() + serializer_class = MaintenanceUnitSerializer + + +_geometry_history_list_parameters = [ + PROVIDER_PARAM, + EVENT_PARAM, + START_DATE_TIME_PARAM, +] + + +@extend_schema_view( + list=extend_schema( + parameters=_geometry_history_list_parameters, + description="GeometryHistory objects are processed from MaintenanceWork objects. MaintenanceWorks with " + "linestrings are validated and if valid a GeometryHistory object is created with the linestring geometry. " + "From MaintenanceWork objects that cointains point data linestrings are generated. The linestrings are " + "generated by comparing the timestamp, event, distance and provider. For every valid generated linestring a " + "GeometryHistory object is created. The coordinates are in SRID 4326.", + ) +) +class GeometryHitoryViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = GeometryHistorySerializer + pagination_class = LargeResultsSetPagination + + def get_queryset(self): + queryset = GeometryHistory.objects.all() + filters = self.request.query_params + + if "provider" in filters: + provider = filters["provider"].upper() + if provider in PROVIDERS: + queryset = queryset.filter(provider=provider) + else: + raise ParseError(f"Providers are: {', '.join(PROVIDERS)}") + + if "event" in filters: + queryset = queryset.filter(events__contains=[filters["event"]]) + if "start_date_time" in filters: + start_date_time = filters["start_date_time"] + try: + start_date_time = datetime.strptime( + start_date_time, "%Y-%m-%d %H:%M:%S" + ) + except ValueError: + raise ParseError( + f"'start_date_time' must be in format {EXAMPLE_TIME_FORMAT} elem.g.,'{EXAMPLE_TIME}'" + ) + queryset = queryset.filter(timestamp__gte=make_aware(start_date_time)) + return queryset + + @method_decorator(cache_page(60 * 15)) + def list(self, request): + queryset = self.get_queryset() + page = self.paginate_queryset(queryset) + serializer = self.serializer_class(page, many=True) + return self.get_paginated_response(serializer.data) From 0e261ba9602e1884b86a79fbd9fd68e885695772 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:59:30 +0300 Subject: [PATCH 19/66] Add admin from street_maintenance --- maintenance/admin.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 maintenance/admin.py diff --git a/maintenance/admin.py b/maintenance/admin.py new file mode 100644 index 000000000..80ee9fce5 --- /dev/null +++ b/maintenance/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +from maintenance.models import MaintenanceUnit, MaintenanceWork + +admin.site.register(MaintenanceWork) +admin.site.register(MaintenanceUnit) From e59ba1b99cc8ba053e451aefff0cf6cca7482790 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:00:16 +0300 Subject: [PATCH 20/66] Add street_maintenance urls for backward compatibility --- maintenance/api/urls_street_maintenance.py | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 maintenance/api/urls_street_maintenance.py diff --git a/maintenance/api/urls_street_maintenance.py b/maintenance/api/urls_street_maintenance.py new file mode 100644 index 000000000..f84e10801 --- /dev/null +++ b/maintenance/api/urls_street_maintenance.py @@ -0,0 +1,28 @@ +""" +Keep backward compatibility to street_maintenance API. +""" + +from django.urls import include, path +from rest_framework import routers + +from . import views + +app_name = "street_maintenance" + +router = routers.DefaultRouter() +router.register("active_events", views.ActiveEventsViewSet, basename="active_events") + +router.register( + "maintenance_works", views.MaintenanceWorkViewSet, basename="maintenance_works" +) +router.register( + "maintenance_units", views.MaintenanceUnitViewSet, basename="maintenance_units" +) + +router.register( + "geometry_history", views.GeometryHitoryViewSet, basename="geometry_history" +) + +urlpatterns = [ + path("", include(router.urls), name="street_maintenance"), +] From 136d5fdd956a90c9edf051e5ac53c87e1d272c0d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:00:41 +0300 Subject: [PATCH 21/66] Add maintenance urls --- smbackend/urls.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/smbackend/urls.py b/smbackend/urls.py index 94555aa30..77067919d 100644 --- a/smbackend/urls.py +++ b/smbackend/urls.py @@ -11,8 +11,9 @@ import eco_counter.api.urls import environment_data.api.urls import exceptional_situations.api.urls +import maintenance.api.urls +import maintenance.api.urls_street_maintenance import mobility_data.api.urls -import street_maintenance.api.urls from iot.api import IoTViewSet from observations.api import views as observations_views from observations.views import obtain_auth_token @@ -77,9 +78,15 @@ include(exceptional_situations.api.urls), name="exceptional_situations", ), + re_path( + "", + include(maintenance.api.urls), + name="maintenance", + ), + # Keep backward compatibility to street_maintenance API re_path( r"^street_maintenance/", - include(street_maintenance.api.urls), + include(maintenance.api.urls_street_maintenance), name="street_maintenance", ), re_path(r"", include(shortcutter_urls)), From 658c34af354e97de828912563558824c9a6e0c1d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:01:17 +0300 Subject: [PATCH 22/66] Replace street_maintenance logger settings with maintenance --- smbackend/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/smbackend/settings.py b/smbackend/settings.py index 7283d55f9..64b3cd5fd 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -77,7 +77,7 @@ ECO_COUNTER_LOG_LEVEL=(str, "INFO"), MOBILITY_DATA_LOG_LEVEL=(str, "INFO"), BICYCLE_NETWORK_LOG_LEVEL=(str, "INFO"), - STREET_MAINTENANCE_LOG_LEVEL=(str, "INFO"), + MAINTENANCE_LOG_LEVEL=(str, "INFO"), ENVIRONMENT_DATA_LOG_LEVEL=(str, "INFO"), EXCEPTIONAL_SITUATIONS_LOG_LEVEL=(str, "INFO"), ) @@ -102,7 +102,7 @@ ECO_COUNTER_LOG_LEVEL = env("ECO_COUNTER_LOG_LEVEL") MOBILITY_DATA_LOG_LEVEL = env("MOBILITY_DATA_LOG_LEVEL") BICYCLE_NETWORK_LOG_LEVEL = env("BICYCLE_NETWORK_LOG_LEVEL") -STREET_MAINTENANCE_LOG_LEVEL = env("STREET_MAINTENANCE_LOG_LEVEL") +MAINTENANCE_LOG_LEVEL = env("MAINTENANCE_LOG_LEVEL") ENVIRONMENT_DATA_LOG_LEVEL = env("ENVIRONMENT_DATA_LOG_LEVEL") EXCEPTIONAL_SITUATIONS_LOG_LEVEL = env("EXCEPTIONAL_SITUATIONS_LOG_LEVEL") @@ -332,9 +332,9 @@ def gettext(s): "handlers": ["console"], "level": BICYCLE_NETWORK_LOG_LEVEL, }, - "street_maintenance": { + "maintenance": { "handlers": ["console"], - "level": STREET_MAINTENANCE_LOG_LEVEL, + "level": MAINTENANCE_LOG_LEVEL, }, "environment_data": { "handlers": ["console"], From 875ba9e35b1a9f98c6c0732723065df13827136e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:02:04 +0300 Subject: [PATCH 23/66] Add README from street_maintenance --- maintenance/README.md | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 maintenance/README.md diff --git a/maintenance/README.md b/maintenance/README.md new file mode 100644 index 000000000..08478af70 --- /dev/null +++ b/maintenance/README.md @@ -0,0 +1,44 @@ +# Maintenance history + +Django app for importing, processing and serving maintenance data. + +## Street maintenance history +### Importer +Name: +import_street_maintenance_history + +Providers: +* YIT +* KUNTEC +* INFRAROAD +* DESTIA + +Parameters: +* --providers, list of providers to import, e.g., --provider yit kuntec +* --history-size, the number of days to import (default is 4 and max is 31) +* --fetch-size (only available for infraroad and destia), the number of works to import per unit(default is 10000). + +### Examples: +To import DESTIA street maintenance history with history size 2: +``` +./manage.py import_street_maintenance_history --providers destia --history-size 2 +``` +To import KUNTEC and INFRAROAD street maintenance history: +``` +./manage.py import_street_maintenance_history --providers kuntec infraroad +``` +Note, only the MaintenanceWorks and MaintenanceUnits for the given provider from the latest import are stored and the rest are deleted. The GeometryHistory is generated only if more than one MaintenanceWork is created. + +### Periodically imorting +To periodically import data use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#street-maintenance-history-street_maintenancetasksimport_street_maintenance_history). + + +## Deleting street maintenance history for a provider +It is possible to delete street maintenance history for a provider. +e.g., to delete all street maintenance history for provider 'destia': +``` +./manage.py delete_street_maintenance_history destia +``` + +## API +See: specificatin.swagger.yaml \ No newline at end of file From 76eb781a0abd926bb6b0d4ab856626352b6a59e2 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:10:50 +0300 Subject: [PATCH 24/66] Add utils.py from street_maintenance applicaiton --- maintenance/management/commands/utils.py | 643 +++++++++++++++++++++++ 1 file changed, 643 insertions(+) create mode 100644 maintenance/management/commands/utils.py diff --git a/maintenance/management/commands/utils.py b/maintenance/management/commands/utils.py new file mode 100644 index 000000000..b07189b9d --- /dev/null +++ b/maintenance/management/commands/utils.py @@ -0,0 +1,643 @@ +import logging +import re +import zoneinfo +from datetime import datetime, timedelta + +import numpy as np +import polyline +import requests +from django import db +from django.conf import settings +from django.contrib.gis.geos import LineString, Point +from munigeo.models import AdministrativeDivision, AdministrativeDivisionGeometry + +from maintenance.models import ( + DEFAULT_SRID, + GeometryHistory, + MaintenanceUnit, + MaintenanceWork, +) + +from .constants import ( + CONTRACTS, + EVENT_MAPPINGS, + EVENTS, + KUNTEC, + KUNTEC_KEY, + ROUTES, + TIMESTAMP_FORMATS, + TOKEN, + UNITS, + URLS, + VEHICLES, + WORKS, + YIT, +) + +logger = logging.getLogger("maintenance") +# In seconds +MAX_WORK_LENGTH = 60 +VALID_LINESTRING_MAX_POINT_DISTANCE = 0.01 + + +def get_turku_boundary(): + division_turku = AdministrativeDivision.objects.get(name="Turku") + turku_boundary = AdministrativeDivisionGeometry.objects.get( + division=division_turku + ).boundary + turku_boundary.transform(DEFAULT_SRID) + return turku_boundary + + +TURKU_BOUNDARY = get_turku_boundary() + + +def get_json_data(url): + response = requests.get(url) + if response.status_code != 200: + logger.warning( + f"Fetching Maintenance Unit {url} status code: {response.status_code} response: {response.content}" + ) + return {} + return response.json() + + +def check_linestring_validity( + linestring, threshold=VALID_LINESTRING_MAX_POINT_DISTANCE +): + """ + The LineString is considered invalid if distance between two points is + greater than VALID_LINESTRING_MAX_POINT_DISTANCE. + The lower the threshold value the more serve the validating will be. + """ + prev_coord = None + for coord in linestring.coords: + if prev_coord: + p1 = Point(coord, srid=DEFAULT_SRID) + p2 = Point(prev_coord, srid=DEFAULT_SRID) + if p1.distance(p2) > threshold: + return False + prev_coord = coord + return True + + +def add_geometry_history_objects(objects, points, elem, provider): + """ + A GeometryHistory instance is added to objects that is passed by reference. + Returns number of discarded linestring. + """ + geometry = LineString(points, srid=DEFAULT_SRID) + if check_linestring_validity(geometry, 0.005): + objects.append( + GeometryHistory( + provider=provider, + coordinates=geometry.coords, + timestamp=elem.timestamp, + events=elem.events, + geometry=geometry, + ) + ) + return 0 + else: + return 1 + + +def get_valid_linestrings(linestring, threshold=VALID_LINESTRING_MAX_POINT_DISTANCE): + prev_coord = None + coords = [] + geometries = [] + for coord in linestring.coords: + if prev_coord: + p1 = Point(coord, srid=DEFAULT_SRID) + p2 = Point(prev_coord, srid=DEFAULT_SRID) + if p1.distance(p2) > threshold: + if len(coords) > 1: + geometries.append(LineString(coords, srid=DEFAULT_SRID)) + coords = [prev_coord] + else: + coords = [] + + coords.append(coord) + prev_coord = coord + + if len(coords) > 1: + geometry = LineString(coords, srid=DEFAULT_SRID) + if check_linestring_validity(geometry, threshold): + geometries.append(geometry) + return geometries + + +def get_linestrings_from_points(objects, queryset, provider): + """ + Point data is generated into LineStrings. This is done by iterating the + point data for every MaintenanceUnit for the given provider. + """ + unit_ids = ( + queryset.order_by("maintenance_unit_id") + .values_list("maintenance_unit_id", flat=True) + .distinct("maintenance_unit_id") + ) + discarded_linestrings = 0 + discarded_points = 0 + for unit_id in unit_ids: + # Temporary store points to list for LineString creation + points = [] + qs = queryset.filter(maintenance_unit_id=unit_id).order_by( + "events", "timestamp" + ) + prev_timestamp = None + current_events = None + prev_geometry = None + + for elem in qs: + if not current_events: + current_events = elem.events + if prev_timestamp and prev_geometry: + delta_time = abs(elem.timestamp - prev_timestamp) + # If delta_time is bigger than the MAX_WORK_LENGTH, then we can assume + # that the work should not be in the same linestring/point or the events + # has changed. + if ( + delta_time.seconds > MAX_WORK_LENGTH + or current_events != elem.events + ): + if len(points) > 1: + discarded_linestrings += add_geometry_history_objects( + objects, points, elem, provider + ) + else: + discarded_points += 1 + current_events = elem.events + points = [] + prev_geometry = elem.geometry + points.append(elem.geometry) + prev_timestamp = elem.timestamp + if len(points) > 1: + discarded_linestrings += add_geometry_history_objects( + objects, points, elem, provider + ) + return discarded_linestrings, discarded_points + + +@db.transaction.atomic +def precalculate_geometry_history(provider): + """ + Function that populates the GeometryHistory model for a provider. + LineString geometrys in MaintenanceWorks will be added as they are. + """ + GeometryHistory.objects.filter(provider=provider).delete() + objects = [] + queryset = MaintenanceWork.objects.filter( + maintenance_unit__provider=provider + ).order_by("timestamp") + elements_to_remove = [] + # Add works that are linestrings, + discarded_linestrings = 0 + discarded_points = 0 + for elem in queryset: + if isinstance(elem.geometry, LineString): + if check_linestring_validity(elem.geometry): + objects.append( + GeometryHistory( + provider=provider, + coordinates=elem.geometry.coords, + timestamp=elem.timestamp, + events=elem.events, + geometry=elem.geometry, + ) + ) + else: + discarded_linestrings += 1 + elements_to_remove.append(elem.id) + + # Remove the linestring elements, as they are not needed when generating + # linestrings from point data + queryset = queryset.exclude(id__in=elements_to_remove) + results = get_linestrings_from_points(objects, queryset, provider) + discarded_linestrings += results[0] + discarded_points += results[1] + GeometryHistory.objects.bulk_create(objects) + logger.info(f"Discarded {discarded_points} points in linestring generation") + logger.info(f"Discarded {discarded_linestrings} invalid LineStrings") + logger.info(f"Created {len(objects)} HistoryGeometry rows for provider: {provider}") + + +def get_linestring_in_boundary(linestring, boundary): + """ + Returns a linestring from the input linestring where all the coordinates + are inside the boundary. If linestring creation is not possible return False. + """ + coords = [ + coord + for coord in linestring.coords + if boundary.covers(Point(coord, srid=DEFAULT_SRID)) + ] + if len(coords) > 1: + linestring = LineString(coords, srid=DEFAULT_SRID) + return linestring + else: + return False + + +def handle_unit(filter, objs_to_delete): + num_created = 0 + queryset = MaintenanceUnit.objects.filter(**filter) + queryset_count = queryset.count() + if queryset_count == 0: + MaintenanceUnit.objects.create(**filter) + num_created += 1 + else: + # Keep the first element and if duplicates leave them for deletion. + id = queryset.first().id + if id in objs_to_delete: + objs_to_delete.remove(id) + return num_created + + +def handle_work(filter, objs_to_delete): + num_created = 0 + queryset = MaintenanceWork.objects.filter(**filter) + queryset_count = queryset.count() + + if queryset_count == 0: + MaintenanceWork.objects.create(**filter) + num_created += 1 + else: + # Keep the first element and if duplicates leave them for deletion. + id = queryset.first().id + if id in objs_to_delete: + objs_to_delete.remove(queryset.first().id) + return num_created + + +@db.transaction.atomic +def create_yit_maintenance_works(access_token, history_size): + contract = get_yit_contract(access_token) + list_of_events = get_yit_event_types(access_token) + event_name_mappings = create_dict_from_yit_events(list_of_events) + routes = get_yit_routes(access_token, contract, history_size) + objs_to_delete = list( + MaintenanceWork.objects.filter(maintenance_unit__provider=YIT).values_list( + "id", flat=True + ) + ) + num_created = 0 + for route in routes: + if len(route["geography"]["features"]) > 1: + logger.warning( + f"Route contains multiple features. {route['geography']['features']}" + ) + coordinates = route["geography"]["features"][0]["geometry"]["coordinates"] + + if is_nested_coordinates(coordinates) and len(coordinates) > 1: + geometry = LineString(coordinates, srid=DEFAULT_SRID) + else: + # Remove other data, contains faulty linestrings. + continue + # Create linestring that is inside the boundary of Turku + # and discard parts of the geometry if they are outside the boundary. + geometry = get_linestring_in_boundary(geometry, TURKU_BOUNDARY) + if not geometry: + continue + events = [] + original_event_names = [] + operations = route["operations"] + for operation in operations: + event_name = event_name_mappings[operation].lower() + if event_name in EVENT_MAPPINGS: + for e in EVENT_MAPPINGS[event_name]: + # If mapping value is None, the event is not used. + if e: + if e not in events: + events.append(e) + original_event_names.append(event_name_mappings[operation]) + else: + logger.warning( + f"Found unmapped event: {event_name_mappings[operation]}" + ) + + # If no events found discard the work + if len(events) == 0: + continue + if len(route["geography"]["features"]) > 1: + logger.warning( + f"Route contains multiple features. {route['geography']['features']}" + ) + unit_id = route["vehicleType"] + try: + unit = MaintenanceUnit.objects.get(unit_id=unit_id) + except MaintenanceUnit.DoesNotExist: + logger.warning(f"Maintenance unit: {unit_id}, not found.") + continue + filter = { + "timestamp": route["startTime"], + "maintenance_unit": unit, + "geometry": geometry, + "events": events, + "original_event_names": original_event_names, + } + num_created += handle_work(filter, objs_to_delete) + + MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() + return num_created, len(objs_to_delete) + + +@db.transaction.atomic +def create_kuntec_maintenance_works(history_size): + num_created = 0 + now = datetime.now() + start = (now - timedelta(days=history_size)).strftime(TIMESTAMP_FORMATS[KUNTEC]) + end = now.strftime(TIMESTAMP_FORMATS[KUNTEC]) + objs_to_delete = list( + MaintenanceWork.objects.filter(maintenance_unit__provider=KUNTEC).values_list( + "id", flat=True + ) + ) + for unit in MaintenanceUnit.objects.filter(provider=KUNTEC): + url = URLS[KUNTEC][WORKS].format( + key=KUNTEC_KEY, start=start, end=end, unit_id=unit.unit_id + ) + json_data = get_json_data(url) + if "data" in json_data: + for unit_data in json_data["data"]["units"]: + for route in unit_data["routes"]: + events = [] + original_event_names = [] + # Routes of type 'stop' are discarded. + if route["type"] == "route": + # Check for mapped events to include as works. + for name in unit.names: + event_name = name.lower() + if event_name in EVENT_MAPPINGS: + for e in EVENT_MAPPINGS[event_name]: + # If mapping value is None, the event is not used. + if e: + if e not in events: + events.append(e) + original_event_names.append(name) + else: + logger.warning(f"Found unmapped event: {event_name}") + # If route has mapped event(s) and contains a polyline add work. + if len(events) > 0 and "polyline" in route: + coords = polyline.decode(route["polyline"], geojson=True) + if len(coords) > 1: + geometry = LineString(coords, srid=DEFAULT_SRID) + else: + continue + # Create linestring that is inside the boundary of Turku + # and discard parts of the geometry if they are outside the boundary. + geometry = get_linestring_in_boundary(geometry, TURKU_BOUNDARY) + if not geometry: + continue + timestamp = route["start"]["time"] + filter = { + "timestamp": timestamp, + "maintenance_unit": unit, + "geometry": geometry, + "events": events, + "original_event_names": original_event_names, + } + num_created += handle_work(filter, objs_to_delete) + + MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() + return num_created, len(objs_to_delete) + + +@db.transaction.atomic +def create_maintenance_works(provider, history_size, fetch_size): + turku_boundary = get_turku_boundary() + num_created = 0 + + import_from_date_time = datetime.now() - timedelta(days=history_size) + import_from_date_time = import_from_date_time.replace( + tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki") + ) + objs_to_delete = list( + MaintenanceWork.objects.filter(maintenance_unit__provider=provider).values_list( + "id", flat=True + ) + ) + for unit in MaintenanceUnit.objects.filter(provider=provider): + json_data = get_json_data( + URLS[provider][WORKS].format(id=unit.unit_id, history_size=fetch_size) + ) + if "location_history" in json_data: + json_data = json_data["location_history"] + else: + logger.warning(f"Location history not found for unit: {unit.unit_id}") + continue + for work in json_data: + timestamp = datetime.strptime( + work["timestamp"], TIMESTAMP_FORMATS[provider] + ).replace(tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")) + # Discard events older then import_from_date_time as they will + # never be displayed + if timestamp < import_from_date_time: + continue + coords = work["coords"] + coords = [float(c) for c in re.sub(r"[()]", "", coords).split(" ")] + point = Point(coords[0], coords[1], srid=DEFAULT_SRID) + # discard events outside Turku. + if not turku_boundary.covers(point): + continue + + events = [] + original_event_names = [] + for event in work["events"]: + event_name = event.lower() + if event_name in EVENT_MAPPINGS: + for e in EVENT_MAPPINGS[event_name]: + # If mapping value is None, the event is not used. + if e: + if e not in events: + events.append(e) + original_event_names.append(event) + else: + logger.warning(f"Found unmapped event: {event}") + # If no events found discard the work + if len(events) == 0: + continue + filter = { + "timestamp": timestamp, + "maintenance_unit": unit, + "geometry": point, + "events": events, + "original_event_names": original_event_names, + } + num_created += handle_work(filter, objs_to_delete) + + MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() + return num_created, len(objs_to_delete) + + +@db.transaction.atomic +def create_maintenance_units(provider): + num_created = 0 + objs_to_delete = list( + MaintenanceUnit.objects.filter(provider=provider).values_list("id", flat=True) + ) + for unit in get_json_data(URLS[provider][UNITS]): + # The names of the unit is derived from the events. + names = [n for n in unit["last_location"]["events"]] + filter = { + "unit_id": unit["id"], + "names": names, + "provider": provider, + } + num_created += handle_unit(filter, objs_to_delete) + + MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() + return num_created, len(objs_to_delete) + + +def get_yit_contract(access_token): + response = requests.get( + URLS[YIT][CONTRACTS], headers={"Authorization": f"Bearer {access_token}"} + ) + assert ( + response.status_code == 200 + ), "Fetcing YIT Contract {} failed, status code: {}".format( + URLS[YIT][CONTRACTS], response.status_code + ) + return response.json()[0].get("id", None) + + +def get_yit_event_types(access_token): + response = requests.get( + URLS[YIT][EVENTS], headers={"Authorization": f"Bearer {access_token}"} + ) + assert ( + response.status_code == 200 + ), " Fetching YIT event types {} failed, status code: {}".format( + URLS[YIT][EVENTS], response.status_code + ) + return response.json() + + +def create_dict_from_yit_events(list_of_events): + events = {} + for event in list_of_events: + events[event["id"]] = event["operationName"] + return events + + +@db.transaction.atomic +def create_kuntec_maintenance_units(): + json_data = get_json_data(URLS[KUNTEC][UNITS]) + no_io_din = 0 + num_created = 0 + objs_to_delete = list( + MaintenanceUnit.objects.filter(provider=KUNTEC).values_list("id", flat=True) + ) + for unit in json_data["data"]["units"]: + names = [] + if "io_din" in unit: + on_states = 0 + # example io_din field: {'no': 3, 'label': 'Muu työ', 'state': 0} + for io in unit["io_din"]: + if io["state"] == 1: + on_states += 1 + names.append(io["label"]) + # If names, we have a unit with at least one io_din with State On. + if len(names) > 0: + filter = { + "unit_id": unit["unit_id"], + "names": names, + "provider": KUNTEC, + } + num_created += handle_unit(filter, objs_to_delete) + else: + no_io_din += 1 + MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() + logger.info( + f"Discarding {no_io_din} Kuntec units that do not have a io_din with Status 'On'(1)." + ) + return num_created, len(objs_to_delete) + + +def get_yit_vehicles(access_token): + response = requests.get( + URLS[YIT][VEHICLES], headers={"Authorization": f"Bearer {access_token}"} + ) + assert ( + response.status_code == 200 + ), " Fetching YIT vehicles {} failed, status code: {}".format( + URLS[YIT][VEHICLES], response.status_code + ) + return response.json() + + +@db.transaction.atomic +def create_yit_maintenance_units(access_token): + vehicles = get_yit_vehicles(access_token) + num_created = 0 + objs_to_delete = list( + MaintenanceUnit.objects.filter(provider=YIT).values_list("id", flat=True) + ) + for unit in vehicles: + names = [unit["vehicleTypeName"]] + filter = { + "unit_id": unit["id"], + "names": names, + "provider": YIT, + } + num_created += handle_unit(filter, objs_to_delete) + + MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() + return num_created, len(objs_to_delete) + + +def get_yit_routes(access_token, contract, history_size): + now = datetime.now() + end = now.replace(tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")).strftime( + TIMESTAMP_FORMATS[YIT] + ) + start = ( + (now - timedelta(days=history_size)) + .replace(tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")) + .strftime(TIMESTAMP_FORMATS[YIT]) + ) + params = { + "contract": contract, + "start": start, + "end": end, + } + response = requests.get( + URLS[YIT][ROUTES], + headers={"Authorization": f"Bearer {access_token}"}, + params=params, + ) + assert ( + response.status_code == 200 + ), "Fetching YIT routes {}, failed, status code: {}".format( + URLS[YIT][ROUTES], response.status_code + ) + return response.json() + + +def get_yit_access_token(): + """ + Note the IP address of the host calling Autori API (hosts YIT data) must be + given for whitelistning. + """ + assert settings.YIT_SCOPE, "YIT_SCOPE not defined in environment." + assert settings.YIT_CLIENT_ID, "YIT_CLIENT_ID not defined in environment." + assert settings.YIT_CLIENT_SECRET, "YIT_CLIENT_SECRET not defined in environment." + data = { + "grant_type": "client_credentials", + "scope": settings.YIT_SCOPE, + "client_id": settings.YIT_CLIENT_ID, + "client_secret": settings.YIT_CLIENT_SECRET, + } + response = requests.post(URLS[YIT][TOKEN], data=data) + assert ( + response.status_code == 200 + ), "Fetchin oauth2 token from YIT {} failed, status code: {}".format( + URLS[YIT][TOKEN], response.status_code + ) + access_token = response.json().get("access_token", None) + return access_token + + +def is_nested_coordinates(coordinates): + return bool(np.array(coordinates).ndim > 1) From 177ad0e25bf9e0c35e1a458014102e4561d179be Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:11:22 +0300 Subject: [PATCH 25/66] Add management commands from street_maintenance application --- .../delete_street_maintenance_history.py | 61 ++++++++ .../import_street_maintenance_history.py | 132 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 maintenance/management/commands/delete_street_maintenance_history.py create mode 100644 maintenance/management/commands/import_street_maintenance_history.py diff --git a/maintenance/management/commands/delete_street_maintenance_history.py b/maintenance/management/commands/delete_street_maintenance_history.py new file mode 100644 index 000000000..c946b6db5 --- /dev/null +++ b/maintenance/management/commands/delete_street_maintenance_history.py @@ -0,0 +1,61 @@ +import logging + +from django.core.management import BaseCommand + +from maintenance.models import GeometryHistory, MaintenanceUnit + +from .constants import PROVIDERS + +logger = logging.getLogger("maintenance") + +# Add deprecated provider name 'AUTORI' +PROVIDERS.append("AUTORI") + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "providers", + type=str, + nargs="*", + help=", ".join(PROVIDERS), + ) + + def handle(self, *args, **options): + providers = [p.upper() for p in options.get("providers", None)] + + for provider in providers: + if provider not in PROVIDERS: + logger.error( + f"Invalid providers argument {provider}, choices are: {', '.join(PROVIDERS)}" + ) + continue + + logger.info(f"Deleting street maintenance history for {provider}.") + provider = provider.upper() + deleted_units = MaintenanceUnit.objects.filter(provider=provider).delete() + deleted_histories = GeometryHistory.objects.filter( + provider=provider + ).delete() + if "street_maintenance.MaintenanceUnit" in deleted_units[1]: + num_deleted_units = deleted_units[1][ + "street_maintenance.MaintenanceUnit" + ] + else: + num_deleted_units = 0 + if "street_maintenance.MaintenanceWork" in deleted_units[1]: + num_deleted_works = deleted_units[1][ + "street_maintenance.MaintenanceWork" + ] + else: + num_deleted_works = 0 + if "street_maintenance.GeometryHistory" in deleted_histories[1]: + num_deleted_histories = deleted_histories[1][ + "street_maintenance.GeometryHistory" + ] + else: + num_deleted_histories = 0 + + logger.info(f"GeometryHistorys deleted {num_deleted_histories}.") + logger.info(f"MaintenanceUnits deleted {num_deleted_units}.") + logger.info(f"MaintenanceWorks deleted {num_deleted_works}.") diff --git a/maintenance/management/commands/import_street_maintenance_history.py b/maintenance/management/commands/import_street_maintenance_history.py new file mode 100644 index 000000000..db936b3b3 --- /dev/null +++ b/maintenance/management/commands/import_street_maintenance_history.py @@ -0,0 +1,132 @@ +import logging +from datetime import datetime + +from django.core.management import BaseCommand + +from maintenance.models import MaintenanceUnit, MaintenanceWork + +from .constants import ( + FETCH_SIZE, + HISTORY_SIZE, + HISTORY_SIZES, + PROVIDER_TYPES, + PROVIDERS, +) +from .utils import ( + create_kuntec_maintenance_units, + create_kuntec_maintenance_works, + create_maintenance_units, + create_maintenance_works, + create_yit_maintenance_units, + create_yit_maintenance_works, + get_yit_access_token, + precalculate_geometry_history, +) + +logger = logging.getLogger("maintenance") + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--fetch-size", + type=int, + nargs="+", + default=False, + help=("Max number of location history items to fetch per unit."), + ) + parser.add_argument( + "--history-size", + type=int, + nargs="+", + default=False, + help=("History size in days."), + ) + parser.add_argument( + "--providers", + type=str, + nargs="+", + default=False, + help=", ".join(PROVIDERS), + ) + + def handle(self, *args, **options): + history_size = None + fetch_size = None + + if options["history_size"]: + history_size = options["history_size"] + history_size = ( + history_size[0] if type(history_size) == list else history_size + ) + if options["fetch_size"]: + fetch_size = options["fetch_size"] + fetch_size = fetch_size[0] if type(fetch_size) == list else fetch_size + + providers = [p.upper() for p in options.get("providers", None)] + for provider in providers: + if provider not in PROVIDERS: + logger.warning( + f"Provider {provider} not defined, choices are {', '.join(PROVIDERS)}" + ) + continue + start_time = datetime.now() + history_size = ( + history_size if history_size else HISTORY_SIZES[provider][HISTORY_SIZE] + ) + fetch_size = ( + fetch_size + if fetch_size + else HISTORY_SIZES[provider].get(FETCH_SIZE, None) + ) + match provider.upper(): + case PROVIDER_TYPES.DESTIA | PROVIDER_TYPES.INFRAROAD: + num_created_units, num_del_units = create_maintenance_units( + provider + ) + num_created_works, num_del_works = create_maintenance_works( + provider, history_size, fetch_size + ) + case PROVIDER_TYPES.KUNTEC: + num_created_units, num_del_units = create_kuntec_maintenance_units() + num_created_works, num_del_works = create_kuntec_maintenance_works( + history_size + ) + + case PROVIDER_TYPES.YIT: + access_token = get_yit_access_token() + num_created_units, num_del_units = create_yit_maintenance_units( + access_token + ) + num_created_works, num_del_works = create_yit_maintenance_works( + access_token, history_size + ) + + tot_num_units = MaintenanceUnit.objects.filter(provider=provider).count() + tot_num_works = MaintenanceWork.objects.filter( + maintenance_unit__provider=provider + ).count() + logger.info( + f"Deleted {num_del_units} obsolete Units for provider {provider}" + ) + logger.info( + f"Created {num_created_units} units of total {tot_num_units} units for provider {provider}" + ) + logger.info( + f"Deleted {num_del_works} obsolete Works for provider {provider}" + ) + logger.info( + f"Created {num_created_works} Works of total {tot_num_works} Works for provider {provider}" + ) + + if num_created_works > 0: + precalculate_geometry_history(provider) + else: + logger.warning( + f"No works created for {provider}, skipping geometry history population." + ) + end_time = datetime.now() + duration = end_time - start_time + logger.info( + f"Imported {provider} street maintenance history in: {duration}" + ) From c728fec7a04464d2592c42adb907e613393c7616 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:31:36 +0300 Subject: [PATCH 26/66] Add specifications --- maintenance/specificatio.swagger.yaml | 230 ++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 maintenance/specificatio.swagger.yaml diff --git a/maintenance/specificatio.swagger.yaml b/maintenance/specificatio.swagger.yaml new file mode 100644 index 000000000..229078bb9 --- /dev/null +++ b/maintenance/specificatio.swagger.yaml @@ -0,0 +1,230 @@ +swagger: "2.0" + +info: + description: "Maintenance API that serves history data of maintenance works, active events and provides history as generated geometries." + version: "1.0.0" + title: Maintenance History" + +schemes: +- "https" +- "http" + +definitions: + geometry_history: + type: object + title: GeometryHistory + description: "Precalculated geometry history." + properties: + id: + type: integer + geometry_type: + type: string + events: + type: array + description: "Name of the events." + items: + type: string + timestamp: + type: string + provider: + type: string + gemoetry: + type: string + coordinates: + type: array + items: + array: + type: number + + maintenance_unit: + type: object + title: MaintenanceUnit + description: "The maintenace unit." + properties: + id: + type: integer + unit_id: + type: integer + description: "The id(name) of the unit." + geometry_type: + type: string + description: "The type of the geometry" + events: + type: array + description: "Name of the events." + items: + type: string + timestamp: + type: string + provider: + type: string + description: Name of the provider. + geometry: + type: string + coordinates: + description: "Returns if geometry is of type Linestring." + type: array + items: + type: number + + maintenance_work: + type: object + title: MaintenanceWork + description: "The isolated work performed by the unit." + properties: + id: + type: integer + geometry: + type: string + timestamp: + type: string + maintenance_unit: + type: integer + events: + type: array + description: "Name of the events." + items: + type: string + coords: + description: "Returns if geometry is of type Linestring." + type: array + items: + type: number + lat: + description: "Returns if geometry is of type Point." + type: number + lon: + description: "Returns if geometry is of type Point." + type: number + +paths: + /geometry_history/: + get: + summary: "Returns list of precalculated geometry historys." + parameters: + - $ref: "#/components/parameters/provider_param" + - $ref: "#/components/parameters/event_param" + - $ref: "#/components/parameters/start_date_time_param" + responses: + 200: + description: "List of GeometryHistory object" + schema: + type: object + properties: + results: + type: array + items: + $ref: "#/definitions/geometry_history" + + /geometry_history/{id}: + get: + summary: "Return a single precalculated geometry history object." + parameters: + - name: "id" + in: path + type: integer + required: true + responses: + 200: + description: "The geometry history object." + schema: + $ref: "#/definitions/geometry_history" + + /maintenance_units/: + get: + summary: "Returns list of maintenance units." + responses: + 200: + description: "List of maintenance units." + schema: + type: object + properties: + results: + type: array + items: + $ref: "#/definitions/maintenance_unit" + /maintenance_units/{id}: + get: + summary: "Return a maintenance unit by id." + parameters: + - name: "id" + in: path + type: integer + required: true + responses: + 200: + description: "Maintenance unit object." + schema: + $ref: "#/definitions/maintenance_unit" + + /maintenance_works/: + get: + summary: "Returns list of maintenance works. A work is a single work event." + parameters: + - $ref: "#/components/parameters/event_param" + - $ref: "#/components/parameters/start_date_time_param" + - $ref: "#/components/parameters/unit_id_param" + responses: + 200: + description: "List of maintenance works" + schema: + type: object + properties: + results: + type: array + items: + $ref: "#/definitions/maintenance_work" + /maintenance_works/{id}: + get: + summary: "Return a single maintenace work." + parameters: + - name: "id" + in: path + type: integer + required: true + responses: + 200: + description: "The maintenance work object." + schema: + $ref: "#/definitions/maintenance_work" + + /active_events/: + get: + summary: "Returns a list of active events with their mapped names. i.e., event names that are shown in the front end, not the names that are in the source data." + responses: + 200: + description: "List of active events." + +components: + parameters: + provider_param: + name: provider + in: query + description: "Return entities of given provider." + event_param: + name: event + in: query + description: "Return entities of given event. The event name is the mapped name and the active event names can be seen from the active_events endpoint." + schema: + type: string + start_date_time_param: + name: start_date_time + in: query + description: "The start date and time of the entities to fetch. Must be in format YYYY-MM-DD HH:MM:SS e.g.,'2022-09-18 10:00:00' " + schema: + type: string + unit_id_param: + name: unit_id + in: query + description: "The 'unit_id' of the maintenance unit" + schema: + type: integer + max_work_length_param: + name: max_work_length + in: query + description: "The max work length in Seconds of one uniform work. I.e, if the delta time of two works timestamps are greated than the max work length it will end the linestring and create a new. Changing the value can be usefull for different events as their length of the uniform work can vary." + default: 1800 + schema: + type: integer + + \ No newline at end of file From b6d615b5936609e5fe541ab926a2fe394e08409b Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:44:18 +0300 Subject: [PATCH 27/66] Delete, as moved to maintenance application --- .../management/commands/constants.py | 193 ------ .../management/commands/utils.py | 643 ------------------ 2 files changed, 836 deletions(-) delete mode 100644 street_maintenance/management/commands/constants.py delete mode 100644 street_maintenance/management/commands/utils.py diff --git a/street_maintenance/management/commands/constants.py b/street_maintenance/management/commands/constants.py deleted file mode 100644 index beb9adde5..000000000 --- a/street_maintenance/management/commands/constants.py +++ /dev/null @@ -1,193 +0,0 @@ -import types - -from django.conf import settings - -KUNTEC_KEY = settings.KUNTEC_KEY -INFRAROAD = "INFRAROAD" -YIT = "YIT" -KUNTEC = "KUNTEC" -DESTIA = "DESTIA" -PROVIDER_CHOICES = ( - (INFRAROAD, "Infraroad"), - (YIT, "YIT"), - (KUNTEC, "Kuntec"), - (DESTIA, "Destia"), -) -PROVIDERS = [INFRAROAD, YIT, KUNTEC, DESTIA] -PROVIDER_TYPES = types.SimpleNamespace() -PROVIDER_TYPES.YIT = YIT -PROVIDER_TYPES.KUNTEC = KUNTEC -PROVIDER_TYPES.DESTIA = DESTIA -PROVIDER_TYPES.INFRAROAD = INFRAROAD - -UNITS = "UNITS" -WORKS = "WORKS" -EVENTS = "EVENTS" -ROUTES = "ROUTES" -VEHICLES = "VEHICLES" -CONTRACTS = "CONTRACTS" -TOKEN = "TOKEN" - - -URLS = { - KUNTEC: { - WORKS: "https://mapon.com/api/v1/route/list.json?key={key}&from={start}" - "&till={end}&include=polyline&unit_id={unit_id}", - UNITS: f"https://mapon.com/api/v1/unit/list.json?key={KUNTEC_KEY}&include=io_din", - }, - YIT: { - EVENTS: settings.YIT_EVENTS_URL, - ROUTES: settings.YIT_ROUTES_URL, - VEHICLES: settings.YIT_VEHICLES_URL, - CONTRACTS: settings.YIT_CONTRACTS_URL, - TOKEN: settings.YIT_TOKEN_URL, - }, - INFRAROAD: { - WORKS: "https://infraroad.fluentprogress.fi/KuntoInfraroad/v1/snowplow/{id}?history={history_size}", - UNITS: "https://infraroad.fluentprogress.fi/KuntoInfraroad/v1/snowplow/query?since=72hours", - }, - DESTIA: { - WORKS: "https://destia.fluentprogress.fi/KuntoDestia/turku/v1/snowplow/{id}?history={history_size}", - UNITS: "https://destia.fluentprogress.fi/KuntoDestia/turku/v1/snowplow/query?since=72hours", - }, -} - - -# Events are categorized into main groups: -AURAUS = "auraus" -LIUKKAUDENTORJUNTA = "liukkaudentorjunta" -PUHTAANAPITO = "puhtaanapito" -HIEKANPOISTO = "hiekanpoisto" -# MUUT is set to None as the events are not currently displayed. -MUUT = None -EVENT_CHOICES = [AURAUS, LIUKKAUDENTORJUNTA, PUHTAANAPITO, HIEKANPOISTO] -# As data providers have different names for their events, they are mapped -# with this dict, so that every event that does the same has the same name. -# The value is a list, as there can be events that belong to multiple main groups. -# e.g., event "Auraus ja hiekanpoisto". -EVENT_MAPPINGS = { - "käsihiekotus tai käsilumityöt": [AURAUS, LIUKKAUDENTORJUNTA], - "laiturin ja asema-alueen auraus": [AURAUS], - "au": [AURAUS], - "auraus": [AURAUS], - "auraus ja sohjonpoisto": [AURAUS], - "lumen poisajo": [AURAUS], - "lumensiirto": [AURAUS], - "etuaura": [AURAUS], - "alusterä": [AURAUS], - "sivuaura": [AURAUS], - "höyläys": [AURAUS], - "suolaus": [LIUKKAUDENTORJUNTA], - "suolas": [LIUKKAUDENTORJUNTA], - "suolaus (sirotinlaite)": [LIUKKAUDENTORJUNTA], - "liuossuolaus": [LIUKKAUDENTORJUNTA], - # Hiekoitus - "hi": [LIUKKAUDENTORJUNTA], - "su": [LIUKKAUDENTORJUNTA], - "hiekoitus": [LIUKKAUDENTORJUNTA], - "hiekotus": [LIUKKAUDENTORJUNTA], - "hiekoitus (sirotinlaite)": [LIUKKAUDENTORJUNTA], - "linjahiekoitus": [LIUKKAUDENTORJUNTA], - "pistehiekoitus": [LIUKKAUDENTORJUNTA], - "paannejään poisto": [LIUKKAUDENTORJUNTA], - "sirotin": [LIUKKAUDENTORJUNTA], - "laiturin ja asema-alueen liukkaudentorjunta": [LIUKKAUDENTORJUNTA], - "liukkauden torjunta": [LIUKKAUDENTORJUNTA], - # Kadunpesu - "pe": [PUHTAANAPITO], - # Harjaus - "hj": [PUHTAANAPITO], - # Hiekanpoisto - "hn": [PUHTAANAPITO], - "puhtaanapito": [PUHTAANAPITO], - "harjaus": [PUHTAANAPITO], - "pesu": [PUHTAANAPITO], - "harjaus ja sohjonpoisto": [PUHTAANAPITO], - "pölynsidonta": [PUHTAANAPITO], - "Imulakaisu": [PUHTAANAPITO], - "hiekanpoisto": [HIEKANPOISTO], - "lakaisu": [HIEKANPOISTO], - "muu": [MUUT], - "muut työt": [MUUT], - "muu työ": [MUUT], - "lisätyö": [MUUT], - "viherhoito": [MUUT], - "kaivot": [MUUT], - "metsätyöt": [MUUT], - "rikkakasvien torjunta": [MUUT], - "paikkaus": [MUUT], - "lisälaite 1": [MUUT], - "lisälaite 2": [MUUT], - "lisälaite 3": [MUUT], - "pensaiden hoitoleikkaus": [MUUT], - "puiden hoitoleikkaukset": [MUUT], - "mittaus- ja tarkastustyöt": [MUUT], - "siimaleikkurointi tai niittotyö": [MUUT], - "liikennemerkkien pesu": [MUUT], - "tiestötarkastus": [MUUT], - "roskankeräys": [MUUT], - "tuntityö": [MUUT], - "pinnan tasaus": [MUUT], - "lumivallien madaltaminen": [MUUT], - "aurausviitoitus ja kinostimet": [MUUT], - "jääkenttien hoito": [MUUT], - "leikkipaikkojen tarkastus": [MUUT], - "kenttien hoito": [MUUT], - "murskeen ajo varastoihin": [MUUT], - "huoltoteiden kunnossapito": [MUUT], - "pysäkkikatosten hoito": [MUUT], - "liikennemerkkien puhdistus": [MUUT], - "siirtoajo": [MUUT], - "Kelintarkastus": [MUUT], - "Sulamisvesien hallinta / höyrytys": [MUUT], - "Sulamisveden haittojen torjunta": [MUUT], - "Sorateiden kunnossapito": [MUUT], - "Äkillinen hoitotyö": [MUUT], - "KT-valu": [MUUT], - "Pintakelirikko": [MUUT], - "Liikennemerkkityö": [MUUT], - "Puiden hoito": [MUUT], - "Puistometsien hoito": [MUUT], - "Hiekkalaatikoiden täyttö": [MUUT], -} -TIMESTAMP_FORMATS = { - INFRAROAD: "%Y-%m-%d %H:%M:%S", - DESTIA: "%Y-%m-%d %H:%M:%S", - KUNTEC: "%Y-%m-%dT%H:%M:%SZ", - YIT: "%Y-%m-%d %H:%M:%S%z", -} -DATE_FORMATS = { - INFRAROAD: "%Y-%m-%d", - DESTIA: "%Y-%m-%d", - KUNTEC: "%Y-%m-%d", - YIT: "%Y-%m-%d", -} -# GeometryHistory API list start_date_time parameter format. -START_DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" -# The number of works(point data with a timestamp and events) to be fetched for every unit. -INFRAROAD_DEFAULT_WORKS_FETCH_SIZE = 10000 -DESTIA_DEFAULT_WORKS_FETCH_SIZE = 10000 -# In days, Note if value is increased the fetch size should also be increased. -INFRAROAD_DEFAULT_WORKS_HISTORY_SIZE = 4 -DESTIA_DEFAULT_WORKS_HISTORY_SIZE = 4 - -# Length of YIT history size in days, max value is 31. -YIT_DEFAULT_WORKS_HISTORY_SIZE = 4 -YIT_MAX_WORKS_HISTORY_SIZE = 31 - -KUNTEC_DEFAULT_WORKS_HISTORY_SIZE = 4 -KUNTEC_MAX_WORKS_HISTORY_SIZE = 31 -HISTORY_SIZE = "history_size" -FETCH_SIZE = "fetch_size" -HISTORY_SIZES = { - INFRAROAD: { - HISTORY_SIZE: INFRAROAD_DEFAULT_WORKS_HISTORY_SIZE, - FETCH_SIZE: INFRAROAD_DEFAULT_WORKS_FETCH_SIZE, - }, - DESTIA: { - HISTORY_SIZE: DESTIA_DEFAULT_WORKS_HISTORY_SIZE, - FETCH_SIZE: DESTIA_DEFAULT_WORKS_FETCH_SIZE, - }, - KUNTEC: {HISTORY_SIZE: KUNTEC_DEFAULT_WORKS_HISTORY_SIZE}, - YIT: {HISTORY_SIZE: YIT_DEFAULT_WORKS_HISTORY_SIZE}, -} diff --git a/street_maintenance/management/commands/utils.py b/street_maintenance/management/commands/utils.py deleted file mode 100644 index 0a7adec3e..000000000 --- a/street_maintenance/management/commands/utils.py +++ /dev/null @@ -1,643 +0,0 @@ -import logging -import re -import zoneinfo -from datetime import datetime, timedelta - -import numpy as np -import polyline -import requests -from django import db -from django.conf import settings -from django.contrib.gis.geos import LineString, Point -from munigeo.models import AdministrativeDivision, AdministrativeDivisionGeometry - -from street_maintenance.models import ( - DEFAULT_SRID, - GeometryHistory, - MaintenanceUnit, - MaintenanceWork, -) - -from .constants import ( - CONTRACTS, - EVENT_MAPPINGS, - EVENTS, - KUNTEC, - KUNTEC_KEY, - ROUTES, - TIMESTAMP_FORMATS, - TOKEN, - UNITS, - URLS, - VEHICLES, - WORKS, - YIT, -) - -logger = logging.getLogger("street_maintenance") -# In seconds -MAX_WORK_LENGTH = 60 -VALID_LINESTRING_MAX_POINT_DISTANCE = 0.01 - - -def get_turku_boundary(): - division_turku = AdministrativeDivision.objects.get(name="Turku") - turku_boundary = AdministrativeDivisionGeometry.objects.get( - division=division_turku - ).boundary - turku_boundary.transform(DEFAULT_SRID) - return turku_boundary - - -TURKU_BOUNDARY = get_turku_boundary() - - -def get_json_data(url): - response = requests.get(url) - if response.status_code != 200: - logger.warning( - f"Fetching Maintenance Unit {url} status code: {response.status_code} response: {response.content}" - ) - return {} - return response.json() - - -def check_linestring_validity( - linestring, threshold=VALID_LINESTRING_MAX_POINT_DISTANCE -): - """ - The LineString is considered invalid if distance between two points is - greater than VALID_LINESTRING_MAX_POINT_DISTANCE. - The lower the threshold value the more serve the validating will be. - """ - prev_coord = None - for coord in linestring.coords: - if prev_coord: - p1 = Point(coord, srid=DEFAULT_SRID) - p2 = Point(prev_coord, srid=DEFAULT_SRID) - if p1.distance(p2) > threshold: - return False - prev_coord = coord - return True - - -def add_geometry_history_objects(objects, points, elem, provider): - """ - A GeometryHistory instance is added to objects that is passed by reference. - Returns number of discarded linestring. - """ - geometry = LineString(points, srid=DEFAULT_SRID) - if check_linestring_validity(geometry, 0.005): - objects.append( - GeometryHistory( - provider=provider, - coordinates=geometry.coords, - timestamp=elem.timestamp, - events=elem.events, - geometry=geometry, - ) - ) - return 0 - else: - return 1 - - -def get_valid_linestrings(linestring, threshold=VALID_LINESTRING_MAX_POINT_DISTANCE): - prev_coord = None - coords = [] - geometries = [] - for coord in linestring.coords: - if prev_coord: - p1 = Point(coord, srid=DEFAULT_SRID) - p2 = Point(prev_coord, srid=DEFAULT_SRID) - if p1.distance(p2) > threshold: - if len(coords) > 1: - geometries.append(LineString(coords, srid=DEFAULT_SRID)) - coords = [prev_coord] - else: - coords = [] - - coords.append(coord) - prev_coord = coord - - if len(coords) > 1: - geometry = LineString(coords, srid=DEFAULT_SRID) - if check_linestring_validity(geometry, threshold): - geometries.append(geometry) - return geometries - - -def get_linestrings_from_points(objects, queryset, provider): - """ - Point data is generated into LineStrings. This is done by iterating the - point data for every MaintenanceUnit for the given provider. - """ - unit_ids = ( - queryset.order_by("maintenance_unit_id") - .values_list("maintenance_unit_id", flat=True) - .distinct("maintenance_unit_id") - ) - discarded_linestrings = 0 - discarded_points = 0 - for unit_id in unit_ids: - # Temporary store points to list for LineString creation - points = [] - qs = queryset.filter(maintenance_unit_id=unit_id).order_by( - "events", "timestamp" - ) - prev_timestamp = None - current_events = None - prev_geometry = None - - for elem in qs: - if not current_events: - current_events = elem.events - if prev_timestamp and prev_geometry: - delta_time = abs(elem.timestamp - prev_timestamp) - # If delta_time is bigger than the MAX_WORK_LENGTH, then we can assume - # that the work should not be in the same linestring/point or the events - # has changed. - if ( - delta_time.seconds > MAX_WORK_LENGTH - or current_events != elem.events - ): - if len(points) > 1: - discarded_linestrings += add_geometry_history_objects( - objects, points, elem, provider - ) - else: - discarded_points += 1 - current_events = elem.events - points = [] - prev_geometry = elem.geometry - points.append(elem.geometry) - prev_timestamp = elem.timestamp - if len(points) > 1: - discarded_linestrings += add_geometry_history_objects( - objects, points, elem, provider - ) - return discarded_linestrings, discarded_points - - -@db.transaction.atomic -def precalculate_geometry_history(provider): - """ - Function that populates the GeometryHistory model for a provider. - LineString geometrys in MaintenanceWorks will be added as they are. - """ - GeometryHistory.objects.filter(provider=provider).delete() - objects = [] - queryset = MaintenanceWork.objects.filter( - maintenance_unit__provider=provider - ).order_by("timestamp") - elements_to_remove = [] - # Add works that are linestrings, - discarded_linestrings = 0 - discarded_points = 0 - for elem in queryset: - if isinstance(elem.geometry, LineString): - if check_linestring_validity(elem.geometry): - objects.append( - GeometryHistory( - provider=provider, - coordinates=elem.geometry.coords, - timestamp=elem.timestamp, - events=elem.events, - geometry=elem.geometry, - ) - ) - else: - discarded_linestrings += 1 - elements_to_remove.append(elem.id) - - # Remove the linestring elements, as they are not needed when generating - # linestrings from point data - queryset = queryset.exclude(id__in=elements_to_remove) - results = get_linestrings_from_points(objects, queryset, provider) - discarded_linestrings += results[0] - discarded_points += results[1] - GeometryHistory.objects.bulk_create(objects) - logger.info(f"Discarded {discarded_points} points in linestring generation") - logger.info(f"Discarded {discarded_linestrings} invalid LineStrings") - logger.info(f"Created {len(objects)} HistoryGeometry rows for provider: {provider}") - - -def get_linestring_in_boundary(linestring, boundary): - """ - Returns a linestring from the input linestring where all the coordinates - are inside the boundary. If linestring creation is not possible return False. - """ - coords = [ - coord - for coord in linestring.coords - if boundary.covers(Point(coord, srid=DEFAULT_SRID)) - ] - if len(coords) > 1: - linestring = LineString(coords, srid=DEFAULT_SRID) - return linestring - else: - return False - - -def handle_unit(filter, objs_to_delete): - num_created = 0 - queryset = MaintenanceUnit.objects.filter(**filter) - queryset_count = queryset.count() - if queryset_count == 0: - MaintenanceUnit.objects.create(**filter) - num_created += 1 - else: - # Keep the first element and if duplicates leave them for deletion. - id = queryset.first().id - if id in objs_to_delete: - objs_to_delete.remove(id) - return num_created - - -def handle_work(filter, objs_to_delete): - num_created = 0 - queryset = MaintenanceWork.objects.filter(**filter) - queryset_count = queryset.count() - - if queryset_count == 0: - MaintenanceWork.objects.create(**filter) - num_created += 1 - else: - # Keep the first element and if duplicates leave them for deletion. - id = queryset.first().id - if id in objs_to_delete: - objs_to_delete.remove(queryset.first().id) - return num_created - - -@db.transaction.atomic -def create_yit_maintenance_works(access_token, history_size): - contract = get_yit_contract(access_token) - list_of_events = get_yit_event_types(access_token) - event_name_mappings = create_dict_from_yit_events(list_of_events) - routes = get_yit_routes(access_token, contract, history_size) - objs_to_delete = list( - MaintenanceWork.objects.filter(maintenance_unit__provider=YIT).values_list( - "id", flat=True - ) - ) - num_created = 0 - for route in routes: - if len(route["geography"]["features"]) > 1: - logger.warning( - f"Route contains multiple features. {route['geography']['features']}" - ) - coordinates = route["geography"]["features"][0]["geometry"]["coordinates"] - - if is_nested_coordinates(coordinates) and len(coordinates) > 1: - geometry = LineString(coordinates, srid=DEFAULT_SRID) - else: - # Remove other data, contains faulty linestrings. - continue - # Create linestring that is inside the boundary of Turku - # and discard parts of the geometry if they are outside the boundary. - geometry = get_linestring_in_boundary(geometry, TURKU_BOUNDARY) - if not geometry: - continue - events = [] - original_event_names = [] - operations = route["operations"] - for operation in operations: - event_name = event_name_mappings[operation].lower() - if event_name in EVENT_MAPPINGS: - for e in EVENT_MAPPINGS[event_name]: - # If mapping value is None, the event is not used. - if e: - if e not in events: - events.append(e) - original_event_names.append(event_name_mappings[operation]) - else: - logger.warning( - f"Found unmapped event: {event_name_mappings[operation]}" - ) - - # If no events found discard the work - if len(events) == 0: - continue - if len(route["geography"]["features"]) > 1: - logger.warning( - f"Route contains multiple features. {route['geography']['features']}" - ) - unit_id = route["vehicleType"] - try: - unit = MaintenanceUnit.objects.get(unit_id=unit_id) - except MaintenanceUnit.DoesNotExist: - logger.warning(f"Maintenance unit: {unit_id}, not found.") - continue - filter = { - "timestamp": route["startTime"], - "maintenance_unit": unit, - "geometry": geometry, - "events": events, - "original_event_names": original_event_names, - } - num_created += handle_work(filter, objs_to_delete) - - MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() - return num_created, len(objs_to_delete) - - -@db.transaction.atomic -def create_kuntec_maintenance_works(history_size): - num_created = 0 - now = datetime.now() - start = (now - timedelta(days=history_size)).strftime(TIMESTAMP_FORMATS[KUNTEC]) - end = now.strftime(TIMESTAMP_FORMATS[KUNTEC]) - objs_to_delete = list( - MaintenanceWork.objects.filter(maintenance_unit__provider=KUNTEC).values_list( - "id", flat=True - ) - ) - for unit in MaintenanceUnit.objects.filter(provider=KUNTEC): - url = URLS[KUNTEC][WORKS].format( - key=KUNTEC_KEY, start=start, end=end, unit_id=unit.unit_id - ) - json_data = get_json_data(url) - if "data" in json_data: - for unit_data in json_data["data"]["units"]: - for route in unit_data["routes"]: - events = [] - original_event_names = [] - # Routes of type 'stop' are discarded. - if route["type"] == "route": - # Check for mapped events to include as works. - for name in unit.names: - event_name = name.lower() - if event_name in EVENT_MAPPINGS: - for e in EVENT_MAPPINGS[event_name]: - # If mapping value is None, the event is not used. - if e: - if e not in events: - events.append(e) - original_event_names.append(name) - else: - logger.warning(f"Found unmapped event: {event_name}") - # If route has mapped event(s) and contains a polyline add work. - if len(events) > 0 and "polyline" in route: - coords = polyline.decode(route["polyline"], geojson=True) - if len(coords) > 1: - geometry = LineString(coords, srid=DEFAULT_SRID) - else: - continue - # Create linestring that is inside the boundary of Turku - # and discard parts of the geometry if they are outside the boundary. - geometry = get_linestring_in_boundary(geometry, TURKU_BOUNDARY) - if not geometry: - continue - timestamp = route["start"]["time"] - filter = { - "timestamp": timestamp, - "maintenance_unit": unit, - "geometry": geometry, - "events": events, - "original_event_names": original_event_names, - } - num_created += handle_work(filter, objs_to_delete) - - MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() - return num_created, len(objs_to_delete) - - -@db.transaction.atomic -def create_maintenance_works(provider, history_size, fetch_size): - turku_boundary = get_turku_boundary() - num_created = 0 - - import_from_date_time = datetime.now() - timedelta(days=history_size) - import_from_date_time = import_from_date_time.replace( - tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki") - ) - objs_to_delete = list( - MaintenanceWork.objects.filter(maintenance_unit__provider=provider).values_list( - "id", flat=True - ) - ) - for unit in MaintenanceUnit.objects.filter(provider=provider): - json_data = get_json_data( - URLS[provider][WORKS].format(id=unit.unit_id, history_size=fetch_size) - ) - if "location_history" in json_data: - json_data = json_data["location_history"] - else: - logger.warning(f"Location history not found for unit: {unit.unit_id}") - continue - for work in json_data: - timestamp = datetime.strptime( - work["timestamp"], TIMESTAMP_FORMATS[provider] - ).replace(tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")) - # Discard events older then import_from_date_time as they will - # never be displayed - if timestamp < import_from_date_time: - continue - coords = work["coords"] - coords = [float(c) for c in re.sub(r"[()]", "", coords).split(" ")] - point = Point(coords[0], coords[1], srid=DEFAULT_SRID) - # discard events outside Turku. - if not turku_boundary.covers(point): - continue - - events = [] - original_event_names = [] - for event in work["events"]: - event_name = event.lower() - if event_name in EVENT_MAPPINGS: - for e in EVENT_MAPPINGS[event_name]: - # If mapping value is None, the event is not used. - if e: - if e not in events: - events.append(e) - original_event_names.append(event) - else: - logger.warning(f"Found unmapped event: {event}") - # If no events found discard the work - if len(events) == 0: - continue - filter = { - "timestamp": timestamp, - "maintenance_unit": unit, - "geometry": point, - "events": events, - "original_event_names": original_event_names, - } - num_created += handle_work(filter, objs_to_delete) - - MaintenanceWork.objects.filter(id__in=objs_to_delete).delete() - return num_created, len(objs_to_delete) - - -@db.transaction.atomic -def create_maintenance_units(provider): - num_created = 0 - objs_to_delete = list( - MaintenanceUnit.objects.filter(provider=provider).values_list("id", flat=True) - ) - for unit in get_json_data(URLS[provider][UNITS]): - # The names of the unit is derived from the events. - names = [n for n in unit["last_location"]["events"]] - filter = { - "unit_id": unit["id"], - "names": names, - "provider": provider, - } - num_created += handle_unit(filter, objs_to_delete) - - MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() - return num_created, len(objs_to_delete) - - -def get_yit_contract(access_token): - response = requests.get( - URLS[YIT][CONTRACTS], headers={"Authorization": f"Bearer {access_token}"} - ) - assert ( - response.status_code == 200 - ), "Fetcing YIT Contract {} failed, status code: {}".format( - URLS[YIT][CONTRACTS], response.status_code - ) - return response.json()[0].get("id", None) - - -def get_yit_event_types(access_token): - response = requests.get( - URLS[YIT][EVENTS], headers={"Authorization": f"Bearer {access_token}"} - ) - assert ( - response.status_code == 200 - ), " Fetching YIT event types {} failed, status code: {}".format( - URLS[YIT][EVENTS], response.status_code - ) - return response.json() - - -def create_dict_from_yit_events(list_of_events): - events = {} - for event in list_of_events: - events[event["id"]] = event["operationName"] - return events - - -@db.transaction.atomic -def create_kuntec_maintenance_units(): - json_data = get_json_data(URLS[KUNTEC][UNITS]) - no_io_din = 0 - num_created = 0 - objs_to_delete = list( - MaintenanceUnit.objects.filter(provider=KUNTEC).values_list("id", flat=True) - ) - for unit in json_data["data"]["units"]: - names = [] - if "io_din" in unit: - on_states = 0 - # example io_din field: {'no': 3, 'label': 'Muu työ', 'state': 0} - for io in unit["io_din"]: - if io["state"] == 1: - on_states += 1 - names.append(io["label"]) - # If names, we have a unit with at least one io_din with State On. - if len(names) > 0: - filter = { - "unit_id": unit["unit_id"], - "names": names, - "provider": KUNTEC, - } - num_created += handle_unit(filter, objs_to_delete) - else: - no_io_din += 1 - MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() - logger.info( - f"Discarding {no_io_din} Kuntec units that do not have a io_din with Status 'On'(1)." - ) - return num_created, len(objs_to_delete) - - -def get_yit_vehicles(access_token): - response = requests.get( - URLS[YIT][VEHICLES], headers={"Authorization": f"Bearer {access_token}"} - ) - assert ( - response.status_code == 200 - ), " Fetching YIT vehicles {} failed, status code: {}".format( - URLS[YIT][VEHICLES], response.status_code - ) - return response.json() - - -@db.transaction.atomic -def create_yit_maintenance_units(access_token): - vehicles = get_yit_vehicles(access_token) - num_created = 0 - objs_to_delete = list( - MaintenanceUnit.objects.filter(provider=YIT).values_list("id", flat=True) - ) - for unit in vehicles: - names = [unit["vehicleTypeName"]] - filter = { - "unit_id": unit["id"], - "names": names, - "provider": YIT, - } - num_created += handle_unit(filter, objs_to_delete) - - MaintenanceUnit.objects.filter(id__in=objs_to_delete).delete() - return num_created, len(objs_to_delete) - - -def get_yit_routes(access_token, contract, history_size): - now = datetime.now() - end = now.replace(tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")).strftime( - TIMESTAMP_FORMATS[YIT] - ) - start = ( - (now - timedelta(days=history_size)) - .replace(tzinfo=zoneinfo.ZoneInfo("Europe/Helsinki")) - .strftime(TIMESTAMP_FORMATS[YIT]) - ) - params = { - "contract": contract, - "start": start, - "end": end, - } - response = requests.get( - URLS[YIT][ROUTES], - headers={"Authorization": f"Bearer {access_token}"}, - params=params, - ) - assert ( - response.status_code == 200 - ), "Fetching YIT routes {}, failed, status code: {}".format( - URLS[YIT][ROUTES], response.status_code - ) - return response.json() - - -def get_yit_access_token(): - """ - Note the IP address of the host calling Autori API (hosts YIT data) must be - given for whitelistning. - """ - assert settings.YIT_SCOPE, "YIT_SCOPE not defined in environment." - assert settings.YIT_CLIENT_ID, "YIT_CLIENT_ID not defined in environment." - assert settings.YIT_CLIENT_SECRET, "YIT_CLIENT_SECRET not defined in environment." - data = { - "grant_type": "client_credentials", - "scope": settings.YIT_SCOPE, - "client_id": settings.YIT_CLIENT_ID, - "client_secret": settings.YIT_CLIENT_SECRET, - } - response = requests.post(URLS[YIT][TOKEN], data=data) - assert ( - response.status_code == 200 - ), "Fetchin oauth2 token from YIT {} failed, status code: {}".format( - URLS[YIT][TOKEN], response.status_code - ) - access_token = response.json().get("access_token", None) - return access_token - - -def is_nested_coordinates(coordinates): - return bool(np.array(coordinates).ndim > 1) From a75d5bad1e1ab58c84713a879e9c79b0ff298b28 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:39:00 +0300 Subject: [PATCH 28/66] Fix maintenance urls prefix --- smbackend/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smbackend/urls.py b/smbackend/urls.py index 77067919d..a0a3b0aaf 100644 --- a/smbackend/urls.py +++ b/smbackend/urls.py @@ -79,7 +79,7 @@ name="exceptional_situations", ), re_path( - "", + r"^maintenance/", include(maintenance.api.urls), name="maintenance", ), From 121e21d98ab13571bcf22365b338ee35cbb1b8e5 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 24 Sep 2024 08:52:18 +0300 Subject: [PATCH 29/66] Rename file --- ..._importers.py => test_street_maintenance_history_importers.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename maintenance/tests/{test_importers.py => test_street_maintenance_history_importers.py} (100%) diff --git a/maintenance/tests/test_importers.py b/maintenance/tests/test_street_maintenance_history_importers.py similarity index 100% rename from maintenance/tests/test_importers.py rename to maintenance/tests/test_street_maintenance_history_importers.py From 1993b3aa53bc1f9d75293787c9ca6e59961377e0 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:19:27 +0300 Subject: [PATCH 30/66] Add function get_data_layer --- maintenance/management/commands/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/maintenance/management/commands/utils.py b/maintenance/management/commands/utils.py index b07189b9d..0a0ecb780 100644 --- a/maintenance/management/commands/utils.py +++ b/maintenance/management/commands/utils.py @@ -8,6 +8,7 @@ import requests from django import db from django.conf import settings +from django.contrib.gis.gdal import DataSource from django.contrib.gis.geos import LineString, Point from munigeo.models import AdministrativeDivision, AdministrativeDivisionGeometry @@ -41,7 +42,10 @@ def get_turku_boundary(): - division_turku = AdministrativeDivision.objects.get(name="Turku") + try: + division_turku = AdministrativeDivision.objects.get(name="Turku") + except AdministrativeDivision.DoesNotExist: + return None turku_boundary = AdministrativeDivisionGeometry.objects.get( division=division_turku ).boundary @@ -641,3 +645,9 @@ def get_yit_access_token(): def is_nested_coordinates(coordinates): return bool(np.array(coordinates).ndim > 1) + + +def get_data_layer(url): + ds = DataSource(url) + assert len(ds) == 1 + return ds[0] From 5d2f9763e1bfe2c9aa82a09f2b938d106a243159 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:26:45 +0300 Subject: [PATCH 31/66] Add importer for ski trails --- .../management/commands/import_ski_trails.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 maintenance/management/commands/import_ski_trails.py diff --git a/maintenance/management/commands/import_ski_trails.py b/maintenance/management/commands/import_ski_trails.py new file mode 100644 index 000000000..8853cc093 --- /dev/null +++ b/maintenance/management/commands/import_ski_trails.py @@ -0,0 +1,59 @@ +import logging + +from django.contrib.gis.geos import GEOSGeometry +from django.contrib.gis.geos.error import GEOSException +from django.core.management.base import BaseCommand +from django.db.utils import IntegrityError + +from maintenance.models import UnitMaintenance, UnitMaintenanceGeometry + +from .utils import get_data_layer + +logger = logging.getLogger(__name__) + +URL = ( + "https://api.paikannuspalvelu.fi/v1/public/track/" + "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson" +) + + +def save_trails(layer): + num_saved = 0 + for feature in layer: + try: + geometry = UnitMaintenanceGeometry() + try: + geometry.geometry = GEOSGeometry( + feature.geom.wkt, srid=feature.geom.srid + ) + except GEOSException: + logger.error(f"Invalid geometry {feature.geom.wkt}, skipping...") + continue + + geometry.geometry_id = feature["construction_point_id"].as_int() + try: + geometry.save() + num_saved += 1 + except IntegrityError: + logger.error(f"geometry id {geometry.geometry_id} exists, skipping...") + except Exception as exp: + logger.error(f"Could not save ski trail {feature}, reason {exp}") + return num_saved + + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument( + "--delete", + action="store_true", + help="Delete all ski trails that have a unit maintenance relationship, before importing.", + ) + + def handle(self, *args, **options): + if options.get("delete", False): + UnitMaintenanceGeometry.objects.filter( + unit_maintenance__target=UnitMaintenance.SKI_TRAIL + ).delete() + layer = get_data_layer(URL) + logger.info(f"Saved {save_trails(layer)} ski trails.") From 1b707dbb73dfc89b958c15cfe12a01b3f626b242 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:27:01 +0300 Subject: [PATCH 32/66] Add task to import ski trails --- maintenance/tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/maintenance/tasks.py b/maintenance/tasks.py index 245186e93..0d5e358f1 100644 --- a/maintenance/tasks.py +++ b/maintenance/tasks.py @@ -3,6 +3,11 @@ from smbackend.utils import shared_task_email +@shared_task_email +def import_ski_trails(*args, name="import_ski_trails"): + management.call_command("import_ski_trails", *args) + + @shared_task_email def delete_street_maintenance_history(*args, name="delete_street_maintenance_history"): management.call_command("delete_street_maintenance_history", *args) From 49dd07be5ea663307c2d98cb51f54af40f56b872 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:27:38 +0300 Subject: [PATCH 33/66] Add fixture data for testing importing of ski trails --- maintenance/tests/data/ski_trails.geojson | 110 ++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 maintenance/tests/data/ski_trails.geojson diff --git a/maintenance/tests/data/ski_trails.geojson b/maintenance/tests/data/ski_trails.geojson new file mode 100644 index 000000000..c3d9beb26 --- /dev/null +++ b/maintenance/tests/data/ski_trails.geojson @@ -0,0 +1,110 @@ +{ + "__comment": "Paikannuspalvelu.fi API, please contact: info [at] paikannuspalvelu [dot] fi for description.", + "__format": "geojson", + "__generated": "2024-09-23 16:11:44", + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "track_id": 1, + "name": "Broken", + "construction_broken_id": 898989 + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [0, 0] + ] + } + }, + { + "type": "Feature", + "properties": { + "track_id": 1, + "name": "Oriketo-Räntämäki", + "construction_point_id": 863 + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [22.3149967484872, 60.4802232613912], + [22.3147906754579, 60.4801233028375], + [22.3145429234268, 60.4800857695957], + [22.3143929961208, 60.4800227127172], + [22.3124718920739, 60.4792771008403], + [22.3119577687763, 60.4790840095889], + [22.3114046783066, 60.478895710334], + [22.310957751619, 60.4788360419992], + [22.3103921680913, 60.4788431739947], + [22.3102104886824, 60.4788197788662], + [22.3099897126849, 60.4787270163438], + [22.3097819870241, 60.4786675944077], + [22.3095240191994, 60.4784943624448], + [22.309016838922, 60.4783512594607], + [22.3088082533975, 60.4782784339478], + [22.3085504749605, 60.4782851386108], + [22.3083731514688, 60.4782912508959], + [22.3078829028877, 60.4786656995758], + [22.3077048566751, 60.4787873434822], + [22.3072939854885, 60.4791111791221], + [22.3065606153373, 60.4796171838217], + [22.305450374535, 60.4796359468219], + [22.3053190896709, 60.480171539005], + [22.3052212480635, 60.4805084209146], + [22.3049755490792, 60.4808935261734], + [22.3052247049333, 60.4804963760325], + [22.3053716545553, 60.4800106030839], + [22.3055734438911, 60.4790766786728], + [22.3054114281505, 60.4786822532053], + [22.304864564618, 60.4784586634528], + [22.304080364548, 60.4782444335458] + ] + } + }, + { + "type": "Feature", + "properties": { + "track_id": 1, + "name": "Duplicate construction point id", + "construction_point_id": 863 + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [22.3149967484872, 60.4802232613912], + [22.3147906754579, 60.4801233028375], + [22.3145429234268, 60.4800857695957], + [22.3143929961208, 60.4800227127172], + [22.3124718920739, 60.4792771008403], + [22.3119577687763, 60.4790840095889], + [22.3114046783066, 60.478895710334], + [22.310957751619, 60.4788360419992], + [22.3103921680913, 60.4788431739947], + [22.3102104886824, 60.4788197788662], + [22.3099897126849, 60.4787270163438], + [22.3097819870241, 60.4786675944077], + [22.3095240191994, 60.4784943624448], + [22.309016838922, 60.4783512594607], + [22.3088082533975, 60.4782784339478], + [22.3085504749605, 60.4782851386108], + [22.3083731514688, 60.4782912508959], + [22.3078829028877, 60.4786656995758], + [22.3077048566751, 60.4787873434822], + [22.3072939854885, 60.4791111791221], + [22.3065606153373, 60.4796171838217], + [22.305450374535, 60.4796359468219], + [22.3053190896709, 60.480171539005], + [22.3052212480635, 60.4805084209146], + [22.3049755490792, 60.4808935261734], + [22.3052247049333, 60.4804963760325], + [22.3053716545553, 60.4800106030839], + [22.3055734438911, 60.4790766786728], + [22.3054114281505, 60.4786822532053], + [22.304864564618, 60.4784586634528], + [22.304080364548, 60.4782444335458] + ] + } + } + ] +} \ No newline at end of file From f9f39ac1532de469302dfdb49383b3fc3d1ed983 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:28:30 +0300 Subject: [PATCH 34/66] Add ski trails importer tests --- maintenance/tests/test_import_ski_trails.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 maintenance/tests/test_import_ski_trails.py diff --git a/maintenance/tests/test_import_ski_trails.py b/maintenance/tests/test_import_ski_trails.py new file mode 100644 index 000000000..57d450bec --- /dev/null +++ b/maintenance/tests/test_import_ski_trails.py @@ -0,0 +1,25 @@ +from unittest.mock import patch + +import pytest +from django.contrib.gis.geos import LineString + +from maintenance.models import UnitMaintenanceGeometry +from maintenance.tests.utils import get_test_fixture_data_layer + + +@pytest.mark.django_db(transaction=True) +@patch("maintenance.management.commands.utils.get_data_layer") +def test_import_skitrails( + get_data_layer_mock, +): + from maintenance.management.commands.import_ski_trails import save_trails + + get_data_layer_mock.return_value = get_test_fixture_data_layer("ski_trails.geojson") + num_saved = save_trails(get_data_layer_mock.return_value) + assert num_saved == 1 + assert UnitMaintenanceGeometry.objects.count() == 1 + umg = UnitMaintenanceGeometry.objects.first() + assert umg.geometry_id == 863 + assert umg.geometry.srid == 4326 + assert len(umg.geometry) == 31 + assert isinstance(umg.geometry, LineString) is True From 145d5cbae8870d4674bce2031cea5921bd34004e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:35:20 +0300 Subject: [PATCH 35/66] Add function to get fixture GDAL data layer --- maintenance/tests/utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/maintenance/tests/utils.py b/maintenance/tests/utils.py index 0306c8d79..ddba7a8cb 100644 --- a/maintenance/tests/utils.py +++ b/maintenance/tests/utils.py @@ -1,5 +1,8 @@ +import os from datetime import datetime +from django.contrib.gis.gdal import DataSource + from maintenance.management.commands.constants import DATE_FORMATS, INFRAROAD, YIT @@ -354,3 +357,19 @@ def get_kuntec_units_mock_data(num_elements): assert num_elements <= len(units) data = {"data": {"units": units[:num_elements]}} return data + + +def get_data_source(file_name): + """ + Returns the given file_name as a GDAL Datasource, + the file must be located in /maintenance/tests/data/ + """ + data_path = os.path.join(os.path.dirname(__file__), "data") + file = os.path.join(data_path, file_name) + return DataSource(file) + + +def get_test_fixture_data_layer(file_name): + ds = get_data_source(file_name) + assert len(ds) == 1 + return ds[0] From c77fcafcc974c25e5a3f15bc338b14dbb8e288c0 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:34:11 +0300 Subject: [PATCH 36/66] Add UnitMaintenanceGeometry fixture --- maintenance/tests/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index 76ae687e5..8160a184a 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -16,7 +16,7 @@ KUNTEC, LIUKKAUDENTORJUNTA, ) -from maintenance.models import DEFAULT_SRID, GeometryHistory +from maintenance.models import DEFAULT_SRID, GeometryHistory, UnitMaintenanceGeometry from mobility_data.tests.conftest import TURKU_WKT UTC_TIMEZONE = pytz.timezone("UTC") @@ -96,3 +96,9 @@ def administrative_division_geometry(administrative_division): id=1, division_id=1, boundary=turku_multipoly ) return adm_div_geom + + +@pytest.fixture +def unit_maintenance_geometry(): + geometry = GEOSGeometry("LINESTRING(0 0, 1 1, 2 2)") + return UnitMaintenanceGeometry.objects.create(geometry_id=863, geometry=geometry) From 1782b2d859ab1029be74e2f5b7ee8e221ad5a6ca Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:29:23 +0300 Subject: [PATCH 37/66] Add models for storing Unit maintenance history --- .../migrations/0002_unitmaintenance.py | 70 +++++++++++++++++++ .../0003_unitmaintenancegeometry.py | 49 +++++++++++++ maintenance/models.py | 47 +++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 maintenance/migrations/0002_unitmaintenance.py create mode 100644 maintenance/migrations/0003_unitmaintenancegeometry.py diff --git a/maintenance/migrations/0002_unitmaintenance.py b/maintenance/migrations/0002_unitmaintenance.py new file mode 100644 index 000000000..4816b64af --- /dev/null +++ b/maintenance/migrations/0002_unitmaintenance.py @@ -0,0 +1,70 @@ +# Generated by Django 4.2.15 on 2024-09-24 10:08 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("services", "0103_unit_geometry_3d"), + ("maintenance", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="UnitMaintenance", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "condition", + models.CharField( + choices=[ + ("UNDEFINED", "undefined"), + ("USABLE", "usable"), + ("UNUSABLE", "unusable"), + ], + default="UNDEFINED", + max_length=16, + ), + ), + ( + "target", + models.CharField( + choices=[ + ("SKI_TRAIL", "ski_trail"), + ("ICE_TRACK", "ice_track"), + ], + default="SKI_TRAIL", + max_length=16, + ), + ), + ("maintained_at", models.DateTimeField()), + ( + "last_imported_time", + models.DateTimeField(help_text="Time of last data import"), + ), + ( + "unit", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="maintenance", + to="services.unit", + ), + ), + ], + options={ + "ordering": ["-maintained_at"], + }, + ), + ] diff --git a/maintenance/migrations/0003_unitmaintenancegeometry.py b/maintenance/migrations/0003_unitmaintenancegeometry.py new file mode 100644 index 000000000..fe90ce8f3 --- /dev/null +++ b/maintenance/migrations/0003_unitmaintenancegeometry.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.15 on 2024-09-24 10:08 + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("maintenance", "0002_unitmaintenance"), + ] + + operations = [ + migrations.CreateModel( + name="UnitMaintenanceGeometry", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "geometry_id", + models.IntegerField(blank=True, null=True, unique=True), + ), + ( + "geometry", + django.contrib.gis.db.models.fields.GeometryField( + null=True, srid=4326 + ), + ), + ( + "unit_maintenance", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="geometries", + to="maintenance.unitmaintenance", + ), + ), + ], + ), + ] diff --git a/maintenance/models.py b/maintenance/models.py index ec4f2fc1a..1123d328a 100644 --- a/maintenance/models.py +++ b/maintenance/models.py @@ -2,10 +2,57 @@ from django.contrib.postgres.fields import ArrayField from maintenance.management.commands.constants import PROVIDER_CHOICES +from services.models import Unit DEFAULT_SRID = 4326 +class UnitMaintenance(models.Model): + """ + Model for storing maintenance information for service_unit entities. + """ + + UNDEFINED = "UNDEFINED" + USABLE = "USABLE" + UNUSABLE = "UNUSABLE" + CONDITION_CHOICES = [ + (UNDEFINED, "undefined"), + (USABLE, "usable"), + (UNUSABLE, "unusable"), + ] + SKI_TRAIL = "SKI_TRAIL" + ICE_TRACK = "ICE_TRACK" + TARGET_CHOICES = [(SKI_TRAIL, "ski_trail"), (ICE_TRACK, "ice_track")] + condition = models.CharField( + max_length=16, choices=CONDITION_CHOICES, default=UNDEFINED + ) + target = models.CharField(max_length=16, choices=TARGET_CHOICES, default=SKI_TRAIL) + maintained_at = models.DateTimeField() + last_imported_time = models.DateTimeField(help_text="Time of last data import") + unit = models.ForeignKey( + Unit, + on_delete=models.CASCADE, + related_name="maintenance", + null=True, + blank=True, + ) + + class Meta: + ordering = ["-maintained_at"] + + +class UnitMaintenanceGeometry(models.Model): + geometry_id = models.IntegerField(unique=True, blank=True, null=True) + geometry = models.GeometryField(srid=DEFAULT_SRID, null=True) + unit_maintenance = models.ForeignKey( + UnitMaintenance, + on_delete=models.SET_NULL, + related_name="geometries", + null=True, + blank=True, + ) + + class MaintenanceUnit(models.Model): unit_id = models.CharField(max_length=64, null=True) From d4339309524efcb5904e81cf5cf53a5a5e5a234a Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:40:00 +0300 Subject: [PATCH 38/66] Add Unit fixture --- maintenance/tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index 8160a184a..fc91a63b2 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -18,6 +18,7 @@ ) from maintenance.models import DEFAULT_SRID, GeometryHistory, UnitMaintenanceGeometry from mobility_data.tests.conftest import TURKU_WKT +from services.models import Unit UTC_TIMEZONE = pytz.timezone("UTC") @@ -102,3 +103,11 @@ def administrative_division_geometry(administrative_division): def unit_maintenance_geometry(): geometry = GEOSGeometry("LINESTRING(0 0, 1 1, 2 2)") return UnitMaintenanceGeometry.objects.create(geometry_id=863, geometry=geometry) + + +@pytest.fixture +def unit(): + now = datetime.now(UTC_TIMEZONE) + return Unit.objects.create( + id=801, name="Oriketo-Räntämäki -kuntorata", last_modified_time=now + ) From 702ea7e59e87824ffee3d7064352f5b8ac6f0e3f Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:56:01 +0300 Subject: [PATCH 39/66] Add ski trails maintenance history importer --- .../import_ski_trails_maintenance_history.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 maintenance/management/commands/import_ski_trails_maintenance_history.py diff --git a/maintenance/management/commands/import_ski_trails_maintenance_history.py b/maintenance/management/commands/import_ski_trails_maintenance_history.py new file mode 100644 index 000000000..bd8336f42 --- /dev/null +++ b/maintenance/management/commands/import_ski_trails_maintenance_history.py @@ -0,0 +1,141 @@ +import logging +from datetime import datetime + +import pytz +from django import db +from django.core.management.base import BaseCommand + +from maintenance.models import UnitMaintenance, UnitMaintenanceGeometry +from services.models import Unit + +from .utils import get_json_data + +logger = logging.getLogger(__name__) +DATE_FIELD_FORMAT = "%Y-%m-%d %H:%M" +URL = ( + "https://api.paikannuspalvelu.fi/v1/public/location/lastvisit/" + "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson&max_distance=50" +) +SKITRAIL_TO_UNIT_ID_MAPPINGS = { + "Impivaara-Isosuo": 805, + "Impivaara-Mälikkälä": 804, + "Kuloisten reitti": 803, + "Oriketo-Räntämäki": 801, + "Oriketo-Halinen": 800, + "Ispoinen-Harittu-Lauste": 799, + "Orikedon kuntorata": 790, + "Nunnavuoren kuntoreitti": 789, + "Luolavuori-Ispoinen": 796, + "Ispoisten kuntoreitti": 785, + "Ispoisten kuntorata (Wibeliuksenpuisto)": None, + "Lausteen kuntorata": 786, + "Hirvensalo": 783, + "Härkämäki": 784, + "Maaria": 787, + "Moisio": 788, + "Pansio": 791, + "Suikkila": 792, + "Varissuo": 795, +} +TIMEZONE = pytz.timezone("Europe/Helsinki") + + +def get_unit(name): + try: + return Unit.objects.get(id=SKITRAIL_TO_UNIT_ID_MAPPINGS[name]) + except (Unit.DoesNotExist, KeyError): + return None + + +def save_maintenance_history(json_data): + objs_to_delete = list( + UnitMaintenance.objects.filter(target=UnitMaintenance.SKI_TRAIL).values_list( + "id", flat=True + ) + ) + num_created = 0 + num_updated = 0 + features = json_data.get("features", None) + if not features: + logger.error("No features found in JSON response.") + return + + for feature in features: + is_created = False + properties = feature.get("properties", None) + if not properties: + logger.warning( + f"'properties' not found for feature: {feature}, skipping..." + ) + continue + name = properties.get("name", None) + maintained_at = None + try: + maintained_at = TIMEZONE.localize( + datetime.strptime(properties.get("date", ""), DATE_FIELD_FORMAT) + ) + except ValueError as exp: + logger.error( + f"Skipping feature {feature}, missing or invalid 'date' field, reason {exp}." + ) + continue + + unit = get_unit(name) + if not unit: + logger.warning(f"Unit not found for {name}") + + filter = { + "unit": unit, + "target": UnitMaintenance.SKI_TRAIL, + } + queryset = UnitMaintenance.objects.filter(**filter) + + if queryset.count() == 0: + unit_maintenance = UnitMaintenance(**filter) + queryset = UnitMaintenance.objects.filter(**filter) + is_created = True + else: + unit_maintenance = UnitMaintenance.objects.filter(**filter).first() + if queryset.count() > 1: + logger.warning(f"Found duplicate UnitMaintenance {filter}") + + unit_maintenance.maintained_at = maintained_at + unit_maintenance.last_imported_time = TIMEZONE.localize( + datetime.now().replace(microsecond=0) + ) + + try: + unit_maintenance.save() + except Exception as exp: + logger.error(f"unable to save ski trail maintenance history, reason: {exp}") + continue + + geometry_id = properties.get("location_id", None) + try: + geometry = UnitMaintenanceGeometry.objects.get(geometry_id=geometry_id) + geometry.unit_maintenance = unit_maintenance + geometry.save() + except UnitMaintenanceGeometry.DoesNotExist: + logger.warning( + f"'geometry not set, cause: geometry {geometry_id} not found." + ) + + if is_created: + num_created += 1 + else: + num_updated += 1 + if unit_maintenance.id in objs_to_delete: + objs_to_delete.remove(unit_maintenance.id) + + UnitMaintenance.objects.filter(id__in=objs_to_delete).delete() + logger.info( + f"Created {num_created}, updated {num_updated}, deleted {len(objs_to_delete)} ski trail maintenance histories" + ) + + +class Command(BaseCommand): + + @db.transaction.atomic + def handle(self, *args, **options): + json_data = get_json_data(URL) + save_maintenance_history(json_data) From 039e6ced68036bad6f74a98156457887b6c633b8 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:11:42 +0300 Subject: [PATCH 40/66] Add ski_trails_maintenance_history_mock_data --- maintenance/tests/utils.py | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/maintenance/tests/utils.py b/maintenance/tests/utils.py index ddba7a8cb..453f62d4f 100644 --- a/maintenance/tests/utils.py +++ b/maintenance/tests/utils.py @@ -359,6 +359,87 @@ def get_kuntec_units_mock_data(num_elements): return data +def get_ski_trails_maintenance_history_mock_data(): + null = None + data = { + "__comment": "Paikannuspalvelu.fi API, please contact: info [at] paikannuspalvelu [dot] fi for description.", + "__format": "geojson", + "__generated": "2024-09-25 11:56:17", + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "speed": 11, + "missing_date": "2024-09-24 10:53", + "device_id": 861774052275250, + "location_id": 866, + "name": "Missing date", + "distance": 0, + "hours_ago": 25.1, + "days_ago": 1, + }, + "geometry": { + "coordinates": [22.249045608908, 60.486433601894], + "type": "Point", + }, + }, + { + "type": "Feature", + "properties": { + "speed": 0, + "date": "26-08-2024 11:56", + "device_id": null, + "location_id": 862, + "name": "Invalid date", + "distance": 0, + "hours_ago": 720, + "days_ago": 30, + }, + "geometry": { + "coordinates": [22.322459622776, 60.477908123479], + "type": "Point", + }, + }, + { + "type": "Feature", + "properties": { + "speed": 0, + "date": "26-08-2024 11:56", + "device_id": null, + "location_id": 424242, + "name": "Invalid locatioin id", + "distance": 0, + "hours_ago": 720, + "days_ago": 30, + }, + "geometry": { + "coordinates": [22.322459622776, 60.477908123479], + "type": "Point", + }, + }, + { + "type": "Feature", + "properties": { + "speed": 0, + "date": "2024-08-26 11:56", + "device_id": null, + "location_id": 863, + "name": "Oriketo-Räntämäki", + "distance": 0, + "hours_ago": 720, + "days_ago": 30, + }, + "geometry": { + "coordinates": [22.312831834656, 60.47943016396], + "type": "Point", + }, + }, + ], + } + return data + + def get_data_source(file_name): """ Returns the given file_name as a GDAL Datasource, From 8bda877ee9d0456f97955bf0040697db970ed496 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:17:08 +0300 Subject: [PATCH 41/66] Add ski trail maintenance history importer tests --- ...t_import_ski_trails_maintenance_history.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 maintenance/tests/test_import_ski_trails_maintenance_history.py diff --git a/maintenance/tests/test_import_ski_trails_maintenance_history.py b/maintenance/tests/test_import_ski_trails_maintenance_history.py new file mode 100644 index 000000000..606268665 --- /dev/null +++ b/maintenance/tests/test_import_ski_trails_maintenance_history.py @@ -0,0 +1,29 @@ +from unittest.mock import patch + +import pytest + +from maintenance.models import UnitMaintenance + +from .utils import get_ski_trails_maintenance_history_mock_data + + +@pytest.mark.django_db(transaction=True) +@patch("maintenance.management.commands.utils.get_json_data") +def test_import_ski_trails_maintenance_history( + get_json_data_mock, unit_maintenance_geometry, unit +): + from maintenance.management.commands.import_ski_trails_maintenance_history import ( + save_maintenance_history, + ) + + json_data = get_ski_trails_maintenance_history_mock_data() + get_json_data_mock.return_value = json_data + save_maintenance_history(get_json_data_mock.return_value) + # Note, the mock data contains features with invalid date, missing date, invalid location_id, these are discared + assert len(json_data["features"]) == 4 + assert UnitMaintenance.objects.count() == 1 + um = UnitMaintenance.objects.first() + unit_maintenance_geometry.refresh_from_db() + assert unit_maintenance_geometry.unit_maintenance == um + assert um.unit == unit + assert um.target == UnitMaintenance.SKI_TRAIL From f6a12870f262f9ed07862c6b2a6a948c6de43714 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:37:53 +0300 Subject: [PATCH 42/66] Add import_ski_trails_maintenance_history task --- maintenance/tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/maintenance/tasks.py b/maintenance/tasks.py index 0d5e358f1..29c1dc0fa 100644 --- a/maintenance/tasks.py +++ b/maintenance/tasks.py @@ -3,6 +3,11 @@ from smbackend.utils import shared_task_email +@shared_task_email +def import_ski_trails_maintenance_history(name="import_ski_trails_maintenance_history"): + management.call_command("import_ski_trails_maintenance_history") + + @shared_task_email def import_ski_trails(*args, name="import_ski_trails"): management.call_command("import_ski_trails", *args) From 73b4e98f2868e7bd9c1168755defe5ae5d7c0421 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:38:53 +0300 Subject: [PATCH 43/66] Add view for UnitMaintenance model --- maintenance/api/urls.py | 5 +++-- maintenance/api/views.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/maintenance/api/urls.py b/maintenance/api/urls.py index 9edfae900..aa8d986bd 100644 --- a/maintenance/api/urls.py +++ b/maintenance/api/urls.py @@ -14,11 +14,12 @@ router.register( "maintenance_units", views.MaintenanceUnitViewSet, basename="maintenance_units" ) - router.register( "geometry_history", views.GeometryHitoryViewSet, basename="geometry_history" ) - +router.register( + "unit_maintenance", views.UnitMaintenanceViewSet, basename="unit_maintenance" +) urlpatterns = [ path("", include(router.urls), name="maintenance"), ] diff --git a/maintenance/api/views.py b/maintenance/api/views.py index ef5fc6437..fb2bbb1a4 100644 --- a/maintenance/api/views.py +++ b/maintenance/api/views.py @@ -1,8 +1,10 @@ from datetime import datetime +import django_filters from django.utils.decorators import method_decorator from django.utils.timezone import make_aware from django.views.decorators.cache import cache_page +from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter from rest_framework import mixins, viewsets from rest_framework.exceptions import ParseError @@ -13,13 +15,19 @@ GeometryHistorySerializer, MaintenanceUnitSerializer, MaintenanceWorkSerializer, + UnitMaintenanceSerializer, ) from maintenance.management.commands.constants import ( EVENT_CHOICES, PROVIDERS, START_DATE_TIME_FORMAT, ) -from maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork +from maintenance.models import ( + GeometryHistory, + MaintenanceUnit, + MaintenanceWork, + UnitMaintenance, +) EXAMPLE_TIME_FORMAT = "YYYY-MM-DD HH:MM:SS" EXAMPLE_TIME = "2022-09-18 10:00:00" @@ -64,6 +72,32 @@ class LargeResultsSetPagination(PageNumberPagination): max_page_size = 200_000 +class UnitMaintenaceFilterSet(django_filters.FilterSet): + maintained_at__gte = django_filters.DateTimeFilter( + method="filter_maintained_at__gte" + ) + maintained_at__lte = django_filters.DateTimeFilter( + method="filter_maintained_at__lte" + ) + + class Meta: + model = UnitMaintenance + fields = {"target": ["iexact"], "unit": ["exact"]} + + def filter_maintained_at__gte(self, queryset, fields, maintained_at): + return queryset.filter(maintained_at__gte=maintained_at) + + def filter_maintained_at__lte(self, queryset, fields, maintained_at): + return queryset.filter(maintained_at__lte=maintained_at) + + +class UnitMaintenanceViewSet(viewsets.ReadOnlyModelViewSet): + queryset = UnitMaintenance.objects.all() + serializer_class = UnitMaintenanceSerializer + filter_backends = [DjangoFilterBackend] + filterset_class = UnitMaintenaceFilterSet + + class ActiveEventsViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): queryset = MaintenanceWork.objects.order_by().distinct("events") serializer_class = ActiveEventSerializer From 39df1c90ec3754098ce5c052227485824809c16e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:02:51 +0300 Subject: [PATCH 44/66] Add fixture now --- maintenance/tests/conftest.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index fc91a63b2..e2d8a6134 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -1,8 +1,8 @@ from datetime import datetime, timedelta import pytest -import pytz from django.contrib.gis.geos import GEOSGeometry, LineString +from django.utils import timezone from munigeo.models import ( AdministrativeDivision, AdministrativeDivisionGeometry, @@ -20,18 +20,20 @@ from mobility_data.tests.conftest import TURKU_WKT from services.models import Unit -UTC_TIMEZONE = pytz.timezone("UTC") - @pytest.fixture def api_client(): return APIClient() +@pytest.fixture +def now(): + return datetime.now().replace(tzinfo=timezone.get_default_timezone()) + + @pytest.mark.django_db @pytest.fixture -def geometry_historys(): - now = datetime.now(UTC_TIMEZONE) +def geometry_historys(now): geometry = LineString((0, 0), (0, 50), (50, 50), (50, 0), (0, 0), sird=DEFAULT_SRID) GeometryHistory.objects.create( timestamp=now, @@ -106,8 +108,7 @@ def unit_maintenance_geometry(): @pytest.fixture -def unit(): - now = datetime.now(UTC_TIMEZONE) +def unit(now): return Unit.objects.create( id=801, name="Oriketo-Räntämäki -kuntorata", last_modified_time=now ) From 249da09bbef67fa5b988f9180e6ee5edba616424 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:08:55 +0300 Subject: [PATCH 45/66] Add units fixture --- maintenance/tests/conftest.py | 6 ++++-- .../tests/test_import_ski_trails_maintenance_history.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index e2d8a6134..73cd2fe51 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -108,7 +108,9 @@ def unit_maintenance_geometry(): @pytest.fixture -def unit(now): - return Unit.objects.create( +def units(now): + Unit.objects.create( id=801, name="Oriketo-Räntämäki -kuntorata", last_modified_time=now ) + Unit.objects.create(id=784, name="Härkämäen kuntorata", last_modified_time=now) + return Unit.objects.all() diff --git a/maintenance/tests/test_import_ski_trails_maintenance_history.py b/maintenance/tests/test_import_ski_trails_maintenance_history.py index 606268665..7cd4923ef 100644 --- a/maintenance/tests/test_import_ski_trails_maintenance_history.py +++ b/maintenance/tests/test_import_ski_trails_maintenance_history.py @@ -10,7 +10,7 @@ @pytest.mark.django_db(transaction=True) @patch("maintenance.management.commands.utils.get_json_data") def test_import_ski_trails_maintenance_history( - get_json_data_mock, unit_maintenance_geometry, unit + get_json_data_mock, unit_maintenance_geometry, units ): from maintenance.management.commands.import_ski_trails_maintenance_history import ( save_maintenance_history, @@ -25,5 +25,5 @@ def test_import_ski_trails_maintenance_history( um = UnitMaintenance.objects.first() unit_maintenance_geometry.refresh_from_db() assert unit_maintenance_geometry.unit_maintenance == um - assert um.unit == unit + assert um.unit == units.get(id=801) assert um.target == UnitMaintenance.SKI_TRAIL From 91345a589fa4f4df6c447062ed46c6cb4536cf3f Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:31:12 +0300 Subject: [PATCH 46/66] Add unit_maintenance_geometries fixture --- maintenance/tests/conftest.py | 6 ++++-- .../tests/test_import_ski_trails_maintenance_history.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index 73cd2fe51..fcccb92f8 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -102,9 +102,11 @@ def administrative_division_geometry(administrative_division): @pytest.fixture -def unit_maintenance_geometry(): +def unit_maintenance_geometries(): geometry = GEOSGeometry("LINESTRING(0 0, 1 1, 2 2)") - return UnitMaintenanceGeometry.objects.create(geometry_id=863, geometry=geometry) + UnitMaintenanceGeometry.objects.create(geometry_id=863, geometry=geometry) + UnitMaintenanceGeometry.objects.create(geometry_id=864, geometry=geometry) + return UnitMaintenanceGeometry.objects.all() @pytest.fixture diff --git a/maintenance/tests/test_import_ski_trails_maintenance_history.py b/maintenance/tests/test_import_ski_trails_maintenance_history.py index 7cd4923ef..ed1d18eaf 100644 --- a/maintenance/tests/test_import_ski_trails_maintenance_history.py +++ b/maintenance/tests/test_import_ski_trails_maintenance_history.py @@ -10,7 +10,7 @@ @pytest.mark.django_db(transaction=True) @patch("maintenance.management.commands.utils.get_json_data") def test_import_ski_trails_maintenance_history( - get_json_data_mock, unit_maintenance_geometry, units + get_json_data_mock, unit_maintenance_geometries, units ): from maintenance.management.commands.import_ski_trails_maintenance_history import ( save_maintenance_history, @@ -23,7 +23,7 @@ def test_import_ski_trails_maintenance_history( assert len(json_data["features"]) == 4 assert UnitMaintenance.objects.count() == 1 um = UnitMaintenance.objects.first() - unit_maintenance_geometry.refresh_from_db() + unit_maintenance_geometry = unit_maintenance_geometries.get(geometry_id=863) assert unit_maintenance_geometry.unit_maintenance == um assert um.unit == units.get(id=801) assert um.target == UnitMaintenance.SKI_TRAIL From a2d2af5c7226780a23cd1712cbb6c7928e0de385 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:24:35 +0300 Subject: [PATCH 47/66] Add SKI_TRAILS_DATE_FIELD_FORMAT constants --- maintenance/management/commands/constants.py | 2 ++ .../commands/import_ski_trails_maintenance_history.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/maintenance/management/commands/constants.py b/maintenance/management/commands/constants.py index beb9adde5..a45aafc86 100644 --- a/maintenance/management/commands/constants.py +++ b/maintenance/management/commands/constants.py @@ -191,3 +191,5 @@ KUNTEC: {HISTORY_SIZE: KUNTEC_DEFAULT_WORKS_HISTORY_SIZE}, YIT: {HISTORY_SIZE: YIT_DEFAULT_WORKS_HISTORY_SIZE}, } + +SKI_TRAILS_DATE_FIELD_FORMAT = "%Y-%m-%d %H:%M" diff --git a/maintenance/management/commands/import_ski_trails_maintenance_history.py b/maintenance/management/commands/import_ski_trails_maintenance_history.py index bd8336f42..a9d12def8 100644 --- a/maintenance/management/commands/import_ski_trails_maintenance_history.py +++ b/maintenance/management/commands/import_ski_trails_maintenance_history.py @@ -8,10 +8,10 @@ from maintenance.models import UnitMaintenance, UnitMaintenanceGeometry from services.models import Unit +from .constants import SKI_TRAILS_DATE_FIELD_FORMAT from .utils import get_json_data logger = logging.getLogger(__name__) -DATE_FIELD_FORMAT = "%Y-%m-%d %H:%M" URL = ( "https://api.paikannuspalvelu.fi/v1/public/location/lastvisit/" "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson&max_distance=50" @@ -72,7 +72,9 @@ def save_maintenance_history(json_data): maintained_at = None try: maintained_at = TIMEZONE.localize( - datetime.strptime(properties.get("date", ""), DATE_FIELD_FORMAT) + datetime.strptime( + properties.get("date", ""), SKI_TRAILS_DATE_FIELD_FORMAT + ) ) except ValueError as exp: logger.error( From 3cd24f5d45dcf64ade6389e2cc26c0bdc538d9f5 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:25:14 +0300 Subject: [PATCH 48/66] Add UnitMaintenance fixtures --- maintenance/tests/conftest.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/maintenance/tests/conftest.py b/maintenance/tests/conftest.py index fcccb92f8..19f76ab03 100644 --- a/maintenance/tests/conftest.py +++ b/maintenance/tests/conftest.py @@ -16,7 +16,12 @@ KUNTEC, LIUKKAUDENTORJUNTA, ) -from maintenance.models import DEFAULT_SRID, GeometryHistory, UnitMaintenanceGeometry +from maintenance.models import ( + DEFAULT_SRID, + GeometryHistory, + UnitMaintenance, + UnitMaintenanceGeometry, +) from mobility_data.tests.conftest import TURKU_WKT from services.models import Unit @@ -116,3 +121,20 @@ def units(now): ) Unit.objects.create(id=784, name="Härkämäen kuntorata", last_modified_time=now) return Unit.objects.all() + + +@pytest.fixture +def unit_maintenances(now, units): + UnitMaintenance.objects.create( + target=UnitMaintenance.SKI_TRAIL, + unit=units.get(id=801), + last_imported_time=now, + maintained_at=now + timedelta(days=1), + ) + UnitMaintenance.objects.create( + target=UnitMaintenance.SKI_TRAIL, + unit=units.get(id=784), + last_imported_time=now, + maintained_at=now - timedelta(days=1), + ) + return UnitMaintenance.objects.all() From c1017a2a2c6e325897207c514894d5dc4660c57f Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:25:58 +0300 Subject: [PATCH 49/66] Add viewset for UnitMaintenanceGeometry --- maintenance/api/urls.py | 5 +++++ maintenance/api/views.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/maintenance/api/urls.py b/maintenance/api/urls.py index aa8d986bd..cc1f48637 100644 --- a/maintenance/api/urls.py +++ b/maintenance/api/urls.py @@ -20,6 +20,11 @@ router.register( "unit_maintenance", views.UnitMaintenanceViewSet, basename="unit_maintenance" ) +router.register( + "unit_maintenance_geometry", + views.UnitMaintenanceGeometryViewSet, + basename="unit_maintenance_geometry", +) urlpatterns = [ path("", include(router.urls), name="maintenance"), ] diff --git a/maintenance/api/views.py b/maintenance/api/views.py index fb2bbb1a4..f2c42efc0 100644 --- a/maintenance/api/views.py +++ b/maintenance/api/views.py @@ -15,6 +15,7 @@ GeometryHistorySerializer, MaintenanceUnitSerializer, MaintenanceWorkSerializer, + UnitMaintenanceGeometrySerializer, UnitMaintenanceSerializer, ) from maintenance.management.commands.constants import ( @@ -27,6 +28,7 @@ MaintenanceUnit, MaintenanceWork, UnitMaintenance, + UnitMaintenanceGeometry, ) EXAMPLE_TIME_FORMAT = "YYYY-MM-DD HH:MM:SS" @@ -98,6 +100,11 @@ class UnitMaintenanceViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = UnitMaintenaceFilterSet +class UnitMaintenanceGeometryViewSet(viewsets.ReadOnlyModelViewSet): + queryset = UnitMaintenanceGeometry.objects.all() + serializer_class = UnitMaintenanceGeometrySerializer + + class ActiveEventsViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): queryset = MaintenanceWork.objects.order_by().distinct("events") serializer_class = ActiveEventSerializer From 12277a4e92201f8dc718c9479d0b83bd4616c17e Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:32:10 +0300 Subject: [PATCH 50/66] Add serializers for UnitMaintenance and UnitMaintenanceGeometry --- maintenance/api/serializers.py | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/maintenance/api/serializers.py b/maintenance/api/serializers.py index df17c89a9..b937f637c 100644 --- a/maintenance/api/serializers.py +++ b/maintenance/api/serializers.py @@ -3,7 +3,46 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from maintenance.models import GeometryHistory, MaintenanceUnit, MaintenanceWork +from maintenance.models import ( + GeometryHistory, + MaintenanceUnit, + MaintenanceWork, + UnitMaintenance, + UnitMaintenanceGeometry, +) + + +class UnitMaintenanceGeometrySerializer(serializers.ModelSerializer): + + class Meta: + model = UnitMaintenanceGeometry + fields = ["id", "geometry", "geometry_id", "unit_maintenance"] + + def to_representation(self, instance): + ret = super().to_representation(instance) + # If nested in UnitMaintenanceSerializer + if ( + self.context.get("request", False) + and "/maintenance/unit_maintenance/" in self.context.get("request", "").path + ): + ret.pop("unit_maintenance", None) + return ret + + +class UnitMaintenanceSerializer(serializers.ModelSerializer): + geometries = UnitMaintenanceGeometrySerializer(many=True, read_only=True) + + class Meta: + model = UnitMaintenance + fields = [ + "id", + "unit", + "target", + "condition", + "maintained_at", + "last_imported_time", + "geometries", + ] class GeometryHistorySerializer(serializers.ModelSerializer): From 676d8c4d616d3af09149337af6234edb5523ad29 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:32:47 +0300 Subject: [PATCH 51/66] Add tests for UnitMaintenanceViewSet --- maintenance/tests/test_api.py | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/maintenance/tests/test_api.py b/maintenance/tests/test_api.py index 02ab75b00..a22881c3f 100644 --- a/maintenance/tests/test_api.py +++ b/maintenance/tests/test_api.py @@ -9,8 +9,65 @@ INFRAROAD, KUNTEC, LIUKKAUDENTORJUNTA, + SKI_TRAILS_DATE_FIELD_FORMAT, START_DATE_TIME_FORMAT, ) +from maintenance.models import UnitMaintenance + + +@pytest.mark.django_db +def test_unit_maintenance_list(api_client, unit_maintenances): + url = reverse("maintenance:unit_maintenance-list") + response = api_client.get(url) + assert response.json()["count"] == 2 + unit_maintenance = response.json()["results"][0] + assert unit_maintenance.keys() == { + "id", + "unit", + "target", + "condition", + "maintained_at", + "last_imported_time", + "geometries", + } + + +@pytest.mark.django_db +def test_unit_maintenance_list_unit_parameter(api_client, unit_maintenances): + url = reverse("maintenance:unit_maintenance-list") + "?unit=801" + response = api_client.get(url) + assert response.json()["count"] == 1 + assert response.json()["results"][0]["unit"] == 801 + + +@pytest.mark.django_db +def test_unit_maintenance_list_target_parameter(api_client, unit_maintenances): + url = ( + reverse("maintenance:unit_maintenance-list") + + f"?target={UnitMaintenance.SKI_TRAIL}" + ) + response = api_client.get(url) + assert response.json()["count"] == 2 + assert response.json()["results"][0]["target"] == UnitMaintenance.SKI_TRAIL + assert response.json()["results"][1]["target"] == UnitMaintenance.SKI_TRAIL + + +@pytest.mark.django_db +def test_unit_maintenance_list_maintained_at_parameter( + api_client, now, unit_maintenances +): + url = ( + reverse("maintenance:unit_maintenance-list") + + f"?maintained_at__gte={now.strftime(SKI_TRAILS_DATE_FIELD_FORMAT)}" + ) + response = api_client.get(url) + assert response.json()["results"][0]["unit"] == 801 + url = ( + reverse("maintenance:unit_maintenance-list") + + f"?maintained_at__lte={now.strftime(SKI_TRAILS_DATE_FIELD_FORMAT)}" + ) + response = api_client.get(url) + assert response.json()["results"][0]["unit"] == 784 @pytest.mark.django_db From c905115b90cbfc864705be626f42ba798b08cbcf Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:36:16 +0300 Subject: [PATCH 52/66] Add unit_maintenance/ to DOC_ENDPOINTS --- smbackend/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/smbackend/settings.py b/smbackend/settings.py index 64b3cd5fd..3fa1b96f3 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -353,6 +353,7 @@ def gettext(s): "/maintenance/geometry_history/", "/maintenance/maintenance_works/", "/maintenance/maintenance_units/", + "/maintenance/unit_maintenance/", "/environment_data/api/v1/stations/", "/environment_data/api/v1/parameters/", "/environment_data/api/v1/data/", From e8bc4063bf1ae2dc72da8c385863788acf37770d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:43:04 +0300 Subject: [PATCH 53/66] Add UnitMaintenance and UnitMaintenanceGeometry --- maintenance/admin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/maintenance/admin.py b/maintenance/admin.py index 80ee9fce5..29f0bdcb6 100644 --- a/maintenance/admin.py +++ b/maintenance/admin.py @@ -1,6 +1,13 @@ from django.contrib import admin -from maintenance.models import MaintenanceUnit, MaintenanceWork +from maintenance.models import ( + MaintenanceUnit, + MaintenanceWork, + UnitMaintenance, + UnitMaintenanceGeometry, +) admin.site.register(MaintenanceWork) admin.site.register(MaintenanceUnit) +admin.site.register(UnitMaintenance) +admin.site.register(UnitMaintenanceGeometry) From 4d79211656b9745af34fbe1e60bccd569c3e6b84 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:11:41 +0300 Subject: [PATCH 54/66] Test UnitMaintenanceGeometryViewSet --- maintenance/tests/test_api.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/maintenance/tests/test_api.py b/maintenance/tests/test_api.py index a22881c3f..1b0b2688b 100644 --- a/maintenance/tests/test_api.py +++ b/maintenance/tests/test_api.py @@ -15,6 +15,20 @@ from maintenance.models import UnitMaintenance +@pytest.mark.django_db +def test_unit_maintenance_geometry_list(api_client, unit_maintenance_geometries): + url = reverse("maintenance:unit_maintenance_geometry-list") + response = api_client.get(url) + assert response.json()["count"] == 2 + unit_maintenance = response.json()["results"][0] + assert unit_maintenance.keys() == { + "id", + "geometry", + "geometry_id", + "unit_maintenance", + } + + @pytest.mark.django_db def test_unit_maintenance_list(api_client, unit_maintenances): url = reverse("maintenance:unit_maintenance-list") @@ -44,7 +58,7 @@ def test_unit_maintenance_list_unit_parameter(api_client, unit_maintenances): def test_unit_maintenance_list_target_parameter(api_client, unit_maintenances): url = ( reverse("maintenance:unit_maintenance-list") - + f"?target={UnitMaintenance.SKI_TRAIL}" + + f"?target__iexact={UnitMaintenance.SKI_TRAIL}" ) response = api_client.get(url) assert response.json()["count"] == 2 From 7436e3aaa5e1d01d5d354828807f0a0b1fb9d424 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:14:49 +0300 Subject: [PATCH 55/66] Add information about ski trails --- maintenance/README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/maintenance/README.md b/maintenance/README.md index 08478af70..68de5c832 100644 --- a/maintenance/README.md +++ b/maintenance/README.md @@ -33,12 +33,24 @@ Note, only the MaintenanceWorks and MaintenanceUnits for the given provider from To periodically import data use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#street-maintenance-history-street_maintenancetasksimport_street_maintenance_history). -## Deleting street maintenance history for a provider +### Deleting street maintenance history for a provider It is possible to delete street maintenance history for a provider. e.g., to delete all street maintenance history for provider 'destia': ``` ./manage.py delete_street_maintenance_history destia ``` -## API -See: specificatin.swagger.yaml \ No newline at end of file +## Ski trails +To periodically import ski trails maintenance history use Celery, for more information [see](https://github.com/City-of-Turku/smbackend/wiki/Celery-Tasks#unit-maintenance). + +### Ski trails +Before importing the ski trails maintenance history the ski trails must be imported. To import type: +``` +./manage.py import_ski_trails +``` + +### Ski trails maintenance history +To import the maintenance history, type: +``` +./manage.py import_ski_trails_maintenance_history +``` From cabb95f571d5132f157b610277bfbc87277609eb Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:03:09 +0300 Subject: [PATCH 56/66] Conditionally serialize maintenance information to units --- services/api.py | 9 +++++++++ services/open_api_parameters.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/services/api.py b/services/api.py index d58f1f4bc..c7efe139d 100644 --- a/services/api.py +++ b/services/api.py @@ -28,6 +28,7 @@ from rest_framework.exceptions import ParseError from rest_framework.response import Response +from maintenance.api.views import UnitMaintenanceSerializer from observations.models import Observation from services.accessibility import RULES from services.models import ( @@ -61,6 +62,7 @@ LATITUDE_PARAMETER, LEVEL_PARAMETER, LONGITUDE_PARAMETER, + MAINTENANCE_PARAMETER, MUNICIPALITY_PARAMETER, OCD_ID_PARAMETER, OCD_MUNICIPALITY_PARAMETER, @@ -817,6 +819,12 @@ def to_representation(self, obj): if qparams.get("accessibility_description", "").lower() in ("true", "1"): ret["accessibility_description"] = shortcomings.accessibility_description + + if qparams.get("maintenance", "").lower() in ("true", "1"): + ret["maintenance"] = UnitMaintenanceSerializer( + obj.maintenance, many=True + ).data + return ret class Meta: @@ -890,6 +898,7 @@ def render(self, data, media_type=None, renderer_context=None): LEVEL_PARAMETER, UNIT_GEOMETRY_PARAMETER, UNIT_GEOMETRY_3D_PARAMETER, + MAINTENANCE_PARAMETER, ] ) class UnitViewSet( diff --git a/services/open_api_parameters.py b/services/open_api_parameters.py index 83e57fde4..5c14c09a8 100644 --- a/services/open_api_parameters.py +++ b/services/open_api_parameters.py @@ -196,3 +196,12 @@ required=False, type=bool, ) + +MAINTENANCE_PARAMETER = OpenApiParameter( + name="maintenance", + location=OpenApiParameter.QUERY, + description="Display units maintenance information.", + required=False, + type=bool, + default=False, +) From e2ab6e00a0836fb787dd5063503374601c76f47c Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:54:34 +0300 Subject: [PATCH 57/66] Serialize maintance information by default --- services/api.py | 2 +- services/open_api_parameters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api.py b/services/api.py index c7efe139d..8bd7acd94 100644 --- a/services/api.py +++ b/services/api.py @@ -820,7 +820,7 @@ def to_representation(self, obj): if qparams.get("accessibility_description", "").lower() in ("true", "1"): ret["accessibility_description"] = shortcomings.accessibility_description - if qparams.get("maintenance", "").lower() in ("true", "1"): + if qparams.get("maintenance", "true").lower() in ("true", "1"): ret["maintenance"] = UnitMaintenanceSerializer( obj.maintenance, many=True ).data diff --git a/services/open_api_parameters.py b/services/open_api_parameters.py index 5c14c09a8..a637958ac 100644 --- a/services/open_api_parameters.py +++ b/services/open_api_parameters.py @@ -203,5 +203,5 @@ description="Display units maintenance information.", required=False, type=bool, - default=False, + default=True, ) From f3d5d2fdb86b9f7e2d1a8ffeacf960acca17ba92 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:38:15 +0300 Subject: [PATCH 58/66] Fix SKITRAIL_TO_UNIT_ID_MAPPINGS --- .../commands/import_ski_trails_maintenance_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintenance/management/commands/import_ski_trails_maintenance_history.py b/maintenance/management/commands/import_ski_trails_maintenance_history.py index a9d12def8..b6064522d 100644 --- a/maintenance/management/commands/import_ski_trails_maintenance_history.py +++ b/maintenance/management/commands/import_ski_trails_maintenance_history.py @@ -26,8 +26,8 @@ "Orikedon kuntorata": 790, "Nunnavuoren kuntoreitti": 789, "Luolavuori-Ispoinen": 796, - "Ispoisten kuntoreitti": 785, - "Ispoisten kuntorata (Wibeliuksenpuisto)": None, + "Ispoisten kuntoreitti": None, + "Ispoisten kuntorata (Wibeliuksenpuisto)": 785, "Lausteen kuntorata": 786, "Hirvensalo": 783, "Härkämäki": 784, From 6b15e4acbd8f0126a3243a03d52a84d16a6b7162 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:23:32 +0300 Subject: [PATCH 59/66] Change STREET_MAINTENANCE_LOG_LEVEL to MAINTENANCE_LOG_LEVEL --- config_dev.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_dev.env.example b/config_dev.env.example index dc66fbe57..888b09bee 100644 --- a/config_dev.env.example +++ b/config_dev.env.example @@ -160,7 +160,7 @@ MOBILITY_DATA_LOG_LEVEL= # Bicycle networks APP, default INFO BICYCLE_NETWORK_LOG_LEVEL= # Street maintenance, default INFO -STREET_MAINTENANCE_LOG_LEVEL= +MAINTENANCE_LOG_LEVEL=INFO # Settings needed for enabling Turku area: #ADDITIONAL_INSTALLED_APPS=smbackend_turku,ptv From 9ace48f76fb596d805df4481f817dcc450e7ead8 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:27:42 +0300 Subject: [PATCH 60/66] Fix comment --- config_dev.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_dev.env.example b/config_dev.env.example index 888b09bee..43bfdc00b 100644 --- a/config_dev.env.example +++ b/config_dev.env.example @@ -159,7 +159,7 @@ ECO_COUNTER_LOG_LEVEL= MOBILITY_DATA_LOG_LEVEL= # Bicycle networks APP, default INFO BICYCLE_NETWORK_LOG_LEVEL= -# Street maintenance, default INFO +# Maintenance MAINTENANCE_LOG_LEVEL=INFO # Settings needed for enabling Turku area: From b7558aa016a5fa4f8a67cd07c75135307794e35b Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 2 Oct 2024 07:59:14 +0300 Subject: [PATCH 61/66] Add SKI_TRAILS_URL environment variable --- config_dev.env.example | 3 +++ maintenance/management/commands/import_ski_trails.py | 8 ++------ smbackend/settings.py | 7 +++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/config_dev.env.example b/config_dev.env.example index 43bfdc00b..ab1c1bbba 100644 --- a/config_dev.env.example +++ b/config_dev.env.example @@ -202,3 +202,6 @@ KUNTEC_KEY= # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py) # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api TELRAAM_TOKEN= +# +# SKI_TRAILS_URL=https://api.paikannuspalvelu.fi/v1/public/track/?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson + diff --git a/maintenance/management/commands/import_ski_trails.py b/maintenance/management/commands/import_ski_trails.py index 8853cc093..5bc90ec53 100644 --- a/maintenance/management/commands/import_ski_trails.py +++ b/maintenance/management/commands/import_ski_trails.py @@ -1,5 +1,6 @@ import logging +from django.conf import settings from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos.error import GEOSException from django.core.management.base import BaseCommand @@ -11,11 +12,6 @@ logger = logging.getLogger(__name__) -URL = ( - "https://api.paikannuspalvelu.fi/v1/public/track/" - "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson" -) - def save_trails(layer): num_saved = 0 @@ -55,5 +51,5 @@ def handle(self, *args, **options): UnitMaintenanceGeometry.objects.filter( unit_maintenance__target=UnitMaintenance.SKI_TRAIL ).delete() - layer = get_data_layer(URL) + layer = get_data_layer(settings.SKI_TRAILS_URL) logger.info(f"Saved {save_trails(layer)} ski trails.") diff --git a/smbackend/settings.py b/smbackend/settings.py index 3fa1b96f3..6053bcd03 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -14,6 +14,11 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = str(Path(__file__).resolve().parent.parent) +SKI_TRAILS_DEFAULT_URL = ( + "https://api.paikannuspalvelu.fi/v1/public/track/" + "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson" +) + env = environ.Env( DEBUG=(bool, False), LANGUAGES=(list, ["fi", "sv", "en"]), @@ -80,6 +85,7 @@ MAINTENANCE_LOG_LEVEL=(str, "INFO"), ENVIRONMENT_DATA_LOG_LEVEL=(str, "INFO"), EXCEPTIONAL_SITUATIONS_LOG_LEVEL=(str, "INFO"), + SKI_TRAILS_URL=(str, SKI_TRAILS_DEFAULT_URL) ) @@ -456,3 +462,4 @@ def preprocessing_filter_spec(endpoints): YIT_TOKEN_URL = env("YIT_TOKEN_URL") KUNTEC_KEY = env("KUNTEC_KEY") TELRAAM_TOKEN = env("TELRAAM_TOKEN") +SKI_TRAILS_URL = env("SKI_TRAILS_URL") From 889932a511d7b2479425afd260dcb52dd5ecb908 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 2 Oct 2024 08:20:31 +0300 Subject: [PATCH 62/66] Add SKI_TRAILS_MAINTENANCE_HISTORY_URL environment variable --- config_dev.env.example | 5 +++-- .../import_ski_trails_maintenance_history.py | 8 +++----- smbackend/settings.py | 12 ++++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/config_dev.env.example b/config_dev.env.example index ab1c1bbba..820421e9f 100644 --- a/config_dev.env.example +++ b/config_dev.env.example @@ -202,6 +202,7 @@ KUNTEC_KEY= # Telraam API token, required when fetching Telraam data to csv (import_telraam_to_csv.py) # https://telraam.helpspace-docs.io/article/27/you-wish-more-data-and-statistics-telraam-api TELRAAM_TOKEN= -# +# Optionally, define the urls, from where the ski trails and maintenance histories are retrieved from. +# The default values are in the examples. # SKI_TRAILS_URL=https://api.paikannuspalvelu.fi/v1/public/track/?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson - +# SKI_TRAILS_MAINTENANCE_HISTORY_URL=https://api.paikannuspalvelu.fi/v1/public/location/lastvisit/?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson&max_distance=50 diff --git a/maintenance/management/commands/import_ski_trails_maintenance_history.py b/maintenance/management/commands/import_ski_trails_maintenance_history.py index b6064522d..84c0f3054 100644 --- a/maintenance/management/commands/import_ski_trails_maintenance_history.py +++ b/maintenance/management/commands/import_ski_trails_maintenance_history.py @@ -3,6 +3,7 @@ import pytz from django import db +from django.conf import settings from django.core.management.base import BaseCommand from maintenance.models import UnitMaintenance, UnitMaintenanceGeometry @@ -12,10 +13,7 @@ from .utils import get_json_data logger = logging.getLogger(__name__) -URL = ( - "https://api.paikannuspalvelu.fi/v1/public/location/lastvisit/" - "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson&max_distance=50" -) + SKITRAIL_TO_UNIT_ID_MAPPINGS = { "Impivaara-Isosuo": 805, "Impivaara-Mälikkälä": 804, @@ -139,5 +137,5 @@ class Command(BaseCommand): @db.transaction.atomic def handle(self, *args, **options): - json_data = get_json_data(URL) + json_data = get_json_data(settings.SKI_TRAILS_MAINTENANCE_HISTORY_URL) save_maintenance_history(json_data) diff --git a/smbackend/settings.py b/smbackend/settings.py index 6053bcd03..74c93402a 100644 --- a/smbackend/settings.py +++ b/smbackend/settings.py @@ -18,7 +18,10 @@ "https://api.paikannuspalvelu.fi/v1/public/track/" "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson" ) - +SKI_TRAILS_MAINTENANCE_HISTORY_DEFAULT_URL = ( + "https://api.paikannuspalvelu.fi/v1/public/location/lastvisit/" + "?data_key=cftqHZ8mjwf3uYpXz9HAUH3nXY6IjrvrvYmRMnbZ&author=?&format=geojson&max_distance=50" +) env = environ.Env( DEBUG=(bool, False), LANGUAGES=(list, ["fi", "sv", "en"]), @@ -85,7 +88,11 @@ MAINTENANCE_LOG_LEVEL=(str, "INFO"), ENVIRONMENT_DATA_LOG_LEVEL=(str, "INFO"), EXCEPTIONAL_SITUATIONS_LOG_LEVEL=(str, "INFO"), - SKI_TRAILS_URL=(str, SKI_TRAILS_DEFAULT_URL) + SKI_TRAILS_URL=(str, SKI_TRAILS_DEFAULT_URL), + SKI_TRAILS_MAINTENANCE_HISTORY_URL=( + str, + SKI_TRAILS_MAINTENANCE_HISTORY_DEFAULT_URL, + ), ) @@ -463,3 +470,4 @@ def preprocessing_filter_spec(endpoints): KUNTEC_KEY = env("KUNTEC_KEY") TELRAAM_TOKEN = env("TELRAAM_TOKEN") SKI_TRAILS_URL = env("SKI_TRAILS_URL") +SKI_TRAILS_MAINTENANCE_HISTORY_URL = env("SKI_TRAILS_MAINTENANCE_HISTORY_URL") From 21aeab0ab9e695b94e8a039db0e92f4b8eee6f41 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:45:26 +0300 Subject: [PATCH 63/66] Test ski trails geometry change --- .../data/ski_trails_geometry_change.geojson | 51 +++++++++++++++++++ maintenance/tests/test_import_ski_trails.py | 19 ++++++- 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 maintenance/tests/data/ski_trails_geometry_change.geojson diff --git a/maintenance/tests/data/ski_trails_geometry_change.geojson b/maintenance/tests/data/ski_trails_geometry_change.geojson new file mode 100644 index 000000000..8bc16a838 --- /dev/null +++ b/maintenance/tests/data/ski_trails_geometry_change.geojson @@ -0,0 +1,51 @@ +{ + "__comment": "Paikannuspalvelu.fi API, please contact: info [at] paikannuspalvelu [dot] fi for description.", + "__format": "geojson", + "__generated": "2024-09-23 16:11:44", + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "track_id": 1, + "name": "Oriketo-Räntämäki", + "construction_point_id": 863 + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [22.3149967484872, 60.4802232613912], + [22.3147906754579, 60.4801233028375], + [22.3145429234268, 60.4800857695957], + [22.3143929961208, 60.4800227127172], + [22.3124718920739, 60.4792771008403], + [22.3119577687763, 60.4790840095889], + [22.3114046783066, 60.478895710334], + [22.310957751619, 60.4788360419992], + [22.3103921680913, 60.4788431739947], + [22.3102104886824, 60.4788197788662], + [22.3099897126849, 60.4787270163438], + [22.3097819870241, 60.4786675944077], + [22.3095240191994, 60.4784943624448], + [22.309016838922, 60.4783512594607], + [22.3088082533975, 60.4782784339478], + [22.3085504749605, 60.4782851386108], + [22.3083731514688, 60.4782912508959], + [22.3078829028877, 60.4786656995758], + [22.3077048566751, 60.4787873434822], + [22.3072939854885, 60.4791111791221], + [22.3065606153373, 60.4796171838217], + [22.305450374535, 60.4796359468219], + [22.3053190896709, 60.480171539005], + [22.3052212480635, 60.4805084209146], + [22.3049755490792, 60.4808935261734], + [22.3052247049333, 60.4804963760325], + [22.3053716545553, 60.4800106030839], + [22.3055734438911, 60.4790766786728], + [22.3054114281505, 60.4786822532053], + [22.304080364548, 60.4782444335458] + ] + } + } + ] +} \ No newline at end of file diff --git a/maintenance/tests/test_import_ski_trails.py b/maintenance/tests/test_import_ski_trails.py index 57d450bec..fb6772cd8 100644 --- a/maintenance/tests/test_import_ski_trails.py +++ b/maintenance/tests/test_import_ski_trails.py @@ -15,11 +15,26 @@ def test_import_skitrails( from maintenance.management.commands.import_ski_trails import save_trails get_data_layer_mock.return_value = get_test_fixture_data_layer("ski_trails.geojson") - num_saved = save_trails(get_data_layer_mock.return_value) - assert num_saved == 1 + num_created, num_updated = save_trails(get_data_layer_mock.return_value) + assert num_created == 1 + assert num_updated == 0 assert UnitMaintenanceGeometry.objects.count() == 1 umg = UnitMaintenanceGeometry.objects.first() assert umg.geometry_id == 863 assert umg.geometry.srid == 4326 assert len(umg.geometry) == 31 assert isinstance(umg.geometry, LineString) is True + # Test geometry update + get_data_layer_mock.return_value = get_test_fixture_data_layer( + "ski_trails_geometry_change.geojson" + ) + num_created, num_updated = save_trails(get_data_layer_mock.return_value) + assert num_created == 0 + assert num_updated == 1 + assert UnitMaintenanceGeometry.objects.count() == 1 + umg_updated = UnitMaintenanceGeometry.objects.first() + assert umg.id == umg_updated.id + assert umg_updated.geometry_id == 863 + assert umg_updated.geometry.srid == 4326 + assert len(umg_updated.geometry) == 30 + assert isinstance(umg_updated.geometry, LineString) is True From 028788be3b9bc1dc66133a61e956a78665760bde Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:45:53 +0300 Subject: [PATCH 64/66] Set unit_maintenance deletion to CASCADE --- ...metry_unit_maintenance_cascade_deletion.py | 25 +++++++++++++++++++ maintenance/models.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 maintenance/migrations/0004_alter_unitmaintenancegeometry_unit_maintenance_cascade_deletion.py diff --git a/maintenance/migrations/0004_alter_unitmaintenancegeometry_unit_maintenance_cascade_deletion.py b/maintenance/migrations/0004_alter_unitmaintenancegeometry_unit_maintenance_cascade_deletion.py new file mode 100644 index 000000000..98618facf --- /dev/null +++ b/maintenance/migrations/0004_alter_unitmaintenancegeometry_unit_maintenance_cascade_deletion.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.15 on 2024-10-02 06:41 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("maintenance", "0003_unitmaintenancegeometry"), + ] + + operations = [ + migrations.AlterField( + model_name="unitmaintenancegeometry", + name="unit_maintenance", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="geometries", + to="maintenance.unitmaintenance", + ), + ), + ] diff --git a/maintenance/models.py b/maintenance/models.py index 1123d328a..571d52f4a 100644 --- a/maintenance/models.py +++ b/maintenance/models.py @@ -46,7 +46,7 @@ class UnitMaintenanceGeometry(models.Model): geometry = models.GeometryField(srid=DEFAULT_SRID, null=True) unit_maintenance = models.ForeignKey( UnitMaintenance, - on_delete=models.SET_NULL, + on_delete=models.CASCADE, related_name="geometries", null=True, blank=True, From a6c612f8de0b94e82a869aae271268e760bc42e7 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:46:35 +0300 Subject: [PATCH 65/66] Remove ueless line of code --- .../management/commands/import_ski_trails_maintenance_history.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maintenance/management/commands/import_ski_trails_maintenance_history.py b/maintenance/management/commands/import_ski_trails_maintenance_history.py index 84c0f3054..90e3a042f 100644 --- a/maintenance/management/commands/import_ski_trails_maintenance_history.py +++ b/maintenance/management/commands/import_ski_trails_maintenance_history.py @@ -92,7 +92,6 @@ def save_maintenance_history(json_data): if queryset.count() == 0: unit_maintenance = UnitMaintenance(**filter) - queryset = UnitMaintenance.objects.filter(**filter) is_created = True else: unit_maintenance = UnitMaintenance.objects.filter(**filter).first() From 161b32215a08072c79ea9afcf001727c3bd1279f Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:48:00 +0300 Subject: [PATCH 66/66] Handle geometry change --- .../management/commands/import_ski_trails.py | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/maintenance/management/commands/import_ski_trails.py b/maintenance/management/commands/import_ski_trails.py index 5bc90ec53..af9e16d25 100644 --- a/maintenance/management/commands/import_ski_trails.py +++ b/maintenance/management/commands/import_ski_trails.py @@ -14,27 +14,44 @@ def save_trails(layer): - num_saved = 0 + num_created = 0 + num_updated = 0 for feature in layer: + is_created = False + is_updated = False try: - geometry = UnitMaintenanceGeometry() + geometry = None + try: - geometry.geometry = GEOSGeometry( - feature.geom.wkt, srid=feature.geom.srid - ) + geometry = GEOSGeometry(feature.geom.wkt, srid=feature.geom.srid) except GEOSException: logger.error(f"Invalid geometry {feature.geom.wkt}, skipping...") continue - geometry.geometry_id = feature["construction_point_id"].as_int() + geometry_id = feature["construction_point_id"].as_int() + filter = {"geometry_id": geometry_id} + queryset = UnitMaintenanceGeometry.objects.filter(**filter) + + if queryset.count() == 0: + unit_maintenance_geometry = UnitMaintenanceGeometry(**filter) + unit_maintenance_geometry.geometry = geometry + is_created = True + else: + unit_maintenance_geometry = queryset.first() + if not unit_maintenance_geometry.geometry.equals(geometry): + unit_maintenance_geometry.geometry = geometry + is_updated = True + try: - geometry.save() - num_saved += 1 + unit_maintenance_geometry.save() except IntegrityError: logger.error(f"geometry id {geometry.geometry_id} exists, skipping...") + except Exception as exp: logger.error(f"Could not save ski trail {feature}, reason {exp}") - return num_saved + num_created = num_created + 1 if is_created else num_created + num_updated = num_updated + 1 if is_updated else num_updated + return num_created, num_updated class Command(BaseCommand): @@ -51,5 +68,7 @@ def handle(self, *args, **options): UnitMaintenanceGeometry.objects.filter( unit_maintenance__target=UnitMaintenance.SKI_TRAIL ).delete() + layer = get_data_layer(settings.SKI_TRAILS_URL) - logger.info(f"Saved {save_trails(layer)} ski trails.") + num_created, num_updated = save_trails(layer) + logger.info(f"Created {num_created}, updated {num_updated} ski trails.")