From 39ff3210d0feb1b1d03c125e49102c75889419da Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:55:29 +0300 Subject: [PATCH 1/6] Add freezegun --- requirements.in | 1 + requirements.txt | 3 +++ 2 files changed, 4 insertions(+) diff --git a/requirements.in b/requirements.in index d8abc2146..dd1ee99b2 100644 --- a/requirements.in +++ b/requirements.in @@ -43,3 +43,4 @@ pyshp polyline drf-spectacular xmltodict +freezegun \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3cb5a9011..f0b2cb181 100644 --- a/requirements.txt +++ b/requirements.txt @@ -108,6 +108,8 @@ flake8==3.9.2 # pep8-naming flake8-polyfill==1.0.2 # via pep8-naming +freezegun==1.5.1 + # via -r requirements.in idna==3.7 # via requests inflection==0.5.1 @@ -191,6 +193,7 @@ python-crontab==2.6.0 python-dateutil==2.8.2 # via # -r requirements.in + # freezegun # pandas # python-crontab pytz==2021.3 From bc786310df6227da4420b7470007b0fe81e69e72 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:56:24 +0300 Subject: [PATCH 2/6] Add command to delete all situation --- .../commands/delete_all_situations.py | 20 +++++++++++++++++++ exceptional_situations/tasks.py | 5 +++++ 2 files changed, 25 insertions(+) create mode 100644 exceptional_situations/management/commands/delete_all_situations.py diff --git a/exceptional_situations/management/commands/delete_all_situations.py b/exceptional_situations/management/commands/delete_all_situations.py new file mode 100644 index 000000000..c0bc022d7 --- /dev/null +++ b/exceptional_situations/management/commands/delete_all_situations.py @@ -0,0 +1,20 @@ +import logging + +from django.core.management import BaseCommand + +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, +) + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + def handle(self, *args, **options): + for situation in Situation.objects.all(): + SituationAnnouncement.objects.filter(situation=situation).delete() + situation.delete() + SituationLocation.objects.all().delete() + logger.info("Deleted all situations.") diff --git a/exceptional_situations/tasks.py b/exceptional_situations/tasks.py index 3dcc90208..2391cf8dd 100644 --- a/exceptional_situations/tasks.py +++ b/exceptional_situations/tasks.py @@ -11,3 +11,8 @@ def import_traffic_situations(name="import_traffic_situations"): @shared_task_email def delete_inactive_situations(name="delete_inactive_situations"): management.call_command("delete_inactive_situations") + + +@shared_task_email +def delete_all_situations(name="delete_all_situations"): + management.call_command("delete_all_situations") From 80f24eda8766199a9ab58c3abea1b23561de7bc3 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:57:39 +0300 Subject: [PATCH 3/6] Add import_traffic_situations tests --- .../tests/test_import_traffic_situations.py | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 exceptional_situations/tests/test_import_traffic_situations.py diff --git a/exceptional_situations/tests/test_import_traffic_situations.py b/exceptional_situations/tests/test_import_traffic_situations.py new file mode 100644 index 000000000..4bcd9389d --- /dev/null +++ b/exceptional_situations/tests/test_import_traffic_situations.py @@ -0,0 +1,375 @@ +from io import StringIO + +import pytest +from django.core.management import call_command +from freezegun import freeze_time + +from exceptional_situations.models import ( + Situation, + SituationAnnouncement, + SituationLocation, + SituationType, +) + +data = [ + { + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [ + [ + [22.317385, 60.488663], + [22.317304, 60.488723], + [22.317274, 60.488745], + [22.316534, 60.489268], + [22.316126, 60.489559], + [22.315814, 60.489772], + [22.314938, 60.490371], + [22.314302, 60.490777], + [22.313629, 60.491174], + [22.313013, 60.491516], + [22.312365, 60.491868], + [22.31148, 60.492311], + [22.310859, 60.492607], + [22.310689, 60.492689], + [22.310625, 60.492717], + [22.309978, 60.493002], + [22.309196, 60.493336], + [22.308259, 60.4937], + [22.307417, 60.494012], + [22.306937, 60.494182], + [22.306443, 60.494348], + [22.30568, 60.494597], + [22.304983, 60.49482], + [22.304693, 60.494903], + [22.304345, 60.495002], + [22.303642, 60.495193], + [22.303402, 60.495258], + ] + ], + }, + "properties": { + "situationId": "GUID50430207", + "situationType": "ROAD_WORK", + "trafficAnnouncementType": None, + "version": 1, + "releaseTime": "2024-06-06T05:38:01.991Z", + "versionTime": "2024-06-06T05:38:01.99Z", + "announcements": [ + { + "language": "FI", + "title": "Tie 40, eli Turun kehätie, Turku. Tietyö. ", + "location": { + "countryCode": 6, + "locationTableNumber": 17, + "locationTableVersion": "1.11.44", + "description": "Tie 40,.. vaikutusalue 1,1 km, suuntaan Kärsämäen risteyssilta.", + }, + "locationDetails": { + "roadAddressLocation": { + "primaryPoint": { + "municipality": "Turku", + "province": "Varsinais-Suomi", + "country": "Suomi", + "roadAddress": { + "road": 40, + "roadSection": 4, + "distance": 2098, + }, + "roadName": "Turun kehätie", + "alertCLocation": { + "locationCode": 2676, + "name": "Orikedon risteyssilta", + "distance": 268, + }, + }, + "secondaryPoint": { + "municipality": "Turku", + "province": "Varsinais-Suomi", + "country": "Suomi", + "roadAddress": { + "road": 40, + "roadSection": 4, + "distance": 1025, + }, + "roadName": "Turun kehätie", + "alertCLocation": { + "locationCode": 2675, + "name": "Kärsämäen risteyssilta", + "distance": 1025, + }, + }, + "direction": "NEG", + "directionDescription": "Piikkiö", + } + }, + "features": [], + "roadWorkPhases": [ + { + "id": "GUID50432255", + "location": { + "countryCode": 6, + "locationTableNumber": 17, + "locationTableVersion": "1.11.44", + "description": "Tie 40, ...vaikutusalue 1,1 km, suuntaan Kärsämäen risteyssilta.", + }, + "locationDetails": { + "roadAddressLocation": { + "primaryPoint": { + "municipality": "Turku", + "province": "Varsinais-Suomi", + "country": "Suomi", + "roadAddress": { + "road": 40, + "roadSection": 4, + "distance": 2098, + }, + "roadName": "Turun kehätie", + "alertCLocation": { + "locationCode": 2676, + "name": "Orikedon risteyssilta", + "distance": 268, + }, + }, + "secondaryPoint": { + "municipality": "Turku", + "province": "Varsinais-Suomi", + "country": "Suomi", + "roadAddress": { + "road": 40, + "roadSection": 4, + "distance": 1025, + }, + "roadName": "Turun kehätie", + "alertCLocation": { + "locationCode": 2675, + "name": "Kärsämäen risteyssilta", + "distance": 1025, + }, + }, + "direction": "NEG", + "directionDescription": "Piikkiö", + } + }, + "workingHours": [ + { + "weekday": "TUESDAY", + "startTime": "09:00:00", + "endTime": "15:00:00", + } + ], + "timeAndDuration": { + "startTime": "2024-06-10T21:00:00Z", + "endTime": "2024-06-11T20:59:59.999Z", + }, + "workTypes": [ + {"type": "BRIDGE", "description": "Siltatyö"}, + {"type": "OTHER", "description": ""}, + ], + "restrictions": [ + { + "type": "SINGLE_LANE_CLOSED", + "restriction": {"name": "Yksi ajokaista suljettu"}, + }, + { + "type": "SPEED_LIMIT", + "restriction": { + "name": "Nopeusrajoitus", + "quantity": 60.0, + "unit": "km/h", + }, + }, + { + "type": "SPEED_LIMIT_LENGTH", + "restriction": { + "name": "Matka, jolla nopeusrajoitus voimassa", + "quantity": 500.0, + "unit": "m", + }, + }, + ], + "restrictionsLiftable": True, + "severity": "HIGH", + "slowTrafficTimes": [], + "queuingTrafficTimes": [], + } + ], + "timeAndDuration": { + "startTime": "2024-06-10T21:00:00Z", + "endTime": "2024-06-11T20:59:59.999Z", + }, + "additionalInformation": "Liikenne- ja kelitiedot verkossa: https://liikennetilanne.fintraffic.fi/", + "sender": "Fintraffic Tieliikennekeskus Turku", + } + ], + "contact": { + "phone": "02002100", + "email": "turku.liikennekeskus@fintraffic.fi", + }, + "dataUpdatedTime": "2024-06-06T05:38:04Z", + }, + } +] + +data_outside_southwest_finland = [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [25.693552, 62.190616]}, + "properties": { + "situationId": "GUID50428325", + "situationType": "ROAD_WORK", + "trafficAnnouncementType": None, + "version": 1, + "releaseTime": "2024-05-08T07:21:39.52Z", + "versionTime": "2024-05-08T07:21:39.519Z", + "announcements": [ + { + "language": "FI", + "title": "Tie 6110, Jyväskylä. Tietyö. ", + "location": { + "countryCode": 6, + "locationTableNumber": 17, + "locationTableVersion": "1.11.43", + "description": "Tie 6110 välillä Keljonkangas - Säynätsalo, Jyväskylä.", + }, + "roadWorkPhases": [ + { + "id": "GUID50430172", + "location": { + "countryCode": 6, + "locationTableNumber": 17, + "locationTableVersion": "1.11.43", + "description": "Tie 6110 välillä Keljonkangas - Säynätsalo, Jyväskylä.", + }, + "locationDetails": { + "roadAddressLocation": { + "primaryPoint": { + "municipality": "Jyväskylä", + "province": "Keski-Suomi", + "country": "Suomi", + "roadAddress": { + "road": 6110, + "roadSection": 1, + "distance": 0, + }, + "alertCLocation": { + "locationCode": 21413, + "name": "Takakeljon tienhaara", + }, + }, + "direction": "UNKNOWN", + } + }, + "workingHours": [ + { + "weekday": "TUESDAY", + "startTime": "07:00:00", + "endTime": "16:00:00", + }, + { + "weekday": "MONDAY", + "startTime": "07:00:00", + "endTime": "16:00:00", + }, + { + "weekday": "FRIDAY", + "startTime": "07:00:00", + "endTime": "16:00:00", + }, + { + "weekday": "THURSDAY", + "startTime": "07:00:00", + "endTime": "16:00:00", + }, + { + "weekday": "WEDNESDAY", + "startTime": "07:00:00", + "endTime": "16:00:00", + }, + ], + "timeAndDuration": { + "startTime": "2024-05-12T21:00:00Z", + "endTime": "2024-08-30T20:59:59.999Z", + }, + "workTypes": [ + { + "type": "OTHER", + "description": "Pysäköintialueen rakentaminen", + } + ], + "restrictions": [], + "restrictionsLiftable": False, + "severity": "LOW", + "slowTrafficTimes": [], + "queuingTrafficTimes": [], + } + ], + "timeAndDuration": { + "startTime": "2024-05-12T21:00:00Z", + "endTime": "2024-08-30T20:59:59.999Z", + }, + "additionalInformation": "Liikenne- ja kelitiedot verkossa: https://liikennetilanne.fintraffic.fi/", + "sender": "Fintraffic Tieliikennekeskus Tampere", + } + ], + "contact": { + "phone": "02002100", + "email": "tampere.liikennekeskus@fintraffic.fi", + }, + "dataUpdatedTime": "2024-05-08T07:21:41Z", + }, + } +] + + +def import_command(*args, **kwargs): + out = StringIO() + call_command( + "import_traffic_situations", + *args, + stdout=out, + stderr=StringIO(), + **kwargs, + ) + return out.getvalue() + + +@pytest.mark.django_db +@freeze_time("2024-06-11 12:00:00", tz_offset=2) +def test_import_traffic_situation(): + import_command(test_importer=data) + assert SituationType.objects.count() == 1 + assert SituationType.objects.first().type_name == "ROAD_WORK" + assert Situation.objects.count() == 1 + situation = Situation.objects.first() + assert situation.situation_id == "GUID50430207" + assert situation.is_active is True + assert SituationLocation.objects.count() == 1 + location = SituationLocation.objects.first() + assert "MULTILINESTRING" in location.geometry.wkt + assert location.location is None + assert SituationAnnouncement.objects.count() == 1 + assert location.details["primaryPoint"]["roadName"] == "Turun kehätie" + announcement = SituationAnnouncement.objects.first() + assert announcement in situation.announcements.all() + assert announcement.title == "Tie 40, eli Turun kehätie, Turku. Tietyö. " + assert "aikutusalue 1,1 km, suuntaan Kärsämäen " in announcement.description + assert ( + announcement.additional_info["sender"] == "Fintraffic Tieliikennekeskus Turku" + ) + assert announcement.location == location + # Test that no duplicates are created + import_command(test_importer=data) + assert Situation.objects.count() == 1 + assert SituationType.objects.count() == 1 + assert SituationAnnouncement.objects.count() == 1 + assert SituationLocation.objects.count() == 1 + + +@pytest.mark.django_db +def test_import_traffic_situation_outside_southwest_finland(): + import_command(test_importer=data_outside_southwest_finland) + assert Situation.objects.count() == 0 + assert SituationType.objects.count() == 0 + assert SituationAnnouncement.objects.count() == 0 + assert SituationLocation.objects.count() == 0 From 982160b1197ec8cf7a859549d71560bcbf46956d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:58:40 +0300 Subject: [PATCH 4/6] Fix location bugs and refactor --- .../commands/import_traffic_situations.py | 148 ++++++++++-------- 1 file changed, 84 insertions(+), 64 deletions(-) diff --git a/exceptional_situations/management/commands/import_traffic_situations.py b/exceptional_situations/management/commands/import_traffic_situations.py index 0ddcc7c7e..3d9671ffe 100644 --- a/exceptional_situations/management/commands/import_traffic_situations.py +++ b/exceptional_situations/management/commands/import_traffic_situations.py @@ -41,11 +41,19 @@ ) +def get_or_create(model, filter): + obj = model.objects.filter(**filter).first() + if obj: + return obj + else: + return model.objects.create(**filter) + + class Command(BaseCommand): def get_geos_geometry(self, feature_data): return GEOSGeometry(str(feature_data["geometry"]), srid=PROJECTION_SRID) - def create_location(self, geometry, announcement_data, announcement): + def create_location(self, geometry, announcement_data): location = None details = announcement_data["locationDetails"].get("roadAddressLocation", None) if details: @@ -54,12 +62,10 @@ def create_location(self, geometry, announcement_data, announcement): "geometry": geometry, "location": location, "details": details, - "announcement": announcement, } - situation_location = SituationLocation.objects.create(**filter) - return situation_location + return get_or_create(SituationLocation, filter) - def create_announcement(self, announcement_data): + def create_announcement(self, announcement_data, location): title = announcement_data.get("title", "") description = announcement_data["location"].get("description", "") additional_info = {} @@ -89,69 +95,83 @@ def create_announcement(self, announcement_data): "additional_info": additional_info, "start_time": start_time, "end_time": end_time, + "location": location, } - situation_announcement = SituationAnnouncement.objects.create(**filter) - return situation_announcement + return get_or_create(SituationAnnouncement, filter) - def handle(self, *args, **options): + def save_features(self, features): num_imported = 0 - for url in URLS: - try: - response = requests.get(url) - assert response.status_code == 200 - except AssertionError: + for feature_data in features: + geometry = self.get_geos_geometry(feature_data) + if not SOUTHWEST_FINLAND_POLYGON.intersects(geometry): continue - features = response.json()["features"] - for feature_data in features: - geometry = self.get_geos_geometry(feature_data) - if not SOUTHWEST_FINLAND_POLYGON.intersects(geometry): - continue - - properties = feature_data.get("properties", None) - if not properties: - continue - situation_id = properties.get("situationId", None) - release_time_str = properties.get("releaseTime", None) - if release_time_str: - for format_str in DATETIME_FORMATS: - try: - release_time = datetime.strptime( - release_time_str, format_str - ) - except ValueError: - pass - else: - break - - if release_time.microsecond != 0: - release_time.replace(microsecond=0) - release_time = timezone.make_aware(release_time, timezone.utc) - - type_name = properties.get("situationType", None) - sub_type_name = properties.get("trafficAnnouncementType", None) - - situation_type, _ = SituationType.objects.get_or_create( - type_name=type_name, sub_type_name=sub_type_name + properties = feature_data.get("properties", None) + if not properties: + continue + situation_id = properties.get("situationId", None) + release_time_str = properties.get("releaseTime", None) + if release_time_str: + for format_str in DATETIME_FORMATS: + try: + release_time = datetime.strptime(release_time_str, format_str) + except ValueError: + pass + else: + break + + if release_time.microsecond != 0: + release_time.replace(microsecond=0) + release_time = timezone.make_aware(release_time, timezone.utc) + + type_name = properties.get("situationType", None) + sub_type_name = properties.get("trafficAnnouncementType", None) + + situation_type, _ = SituationType.objects.get_or_create( + type_name=type_name, sub_type_name=sub_type_name + ) + + filter = { + "situation_id": situation_id, + "situation_type": situation_type, + } + situation, created = Situation.objects.get_or_create(**filter) + situation.release_time = release_time + situation.save() + if not created: + SituationAnnouncement.objects.filter(situation=situation).delete() + situation.announcements.clear() + for announcement_data in properties.get("announcements", []): + situation_location = self.create_location(geometry, announcement_data) + situation_announcement = self.create_announcement( + deepcopy(announcement_data), situation_location ) + situation.announcements.add(situation_announcement) + num_imported += 1 + return num_imported + + def add_arguments(self, parser): + parser.add_argument( + "--test-importer", + type=list, + default=[], + nargs="*", + help="Test importing of data.", + ) - filter = { - "situation_id": situation_id, - "situation_type": situation_type, - } - situation, created = Situation.objects.get_or_create(**filter) - situation.release_time = release_time - situation.save() - if not created: - SituationAnnouncement.objects.filter(situation=situation).delete() - situation.announcements.clear() - for announcement_data in properties.get("announcements", []): - situation_announcement = self.create_announcement( - deepcopy(announcement_data) - ) - self.create_location( - geometry, announcement_data, situation_announcement - ) - situation.announcements.add(situation_announcement) - num_imported += 1 - logger.info(f"Imported/updated {num_imported} traffic situations.") + def handle(self, *args, **options): + num_imported = 0 + if options.get("test_importer", False): + features = [options["test_importer"][0]] + self.save_features(features) + else: + for url in URLS: + try: + response = requests.get(url) + assert response.status_code == 200 + except AssertionError: + continue + features = response.json()["features"] + breakpoint() + num_imported += self.save_features(features) + logger.info(f"Imported/updated {num_imported} traffic situations.") From 8732e6da6d7ab7f2c2ae4ab8480ac9f7af51aa60 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:14:33 +0300 Subject: [PATCH 5/6] Remove breakpoint --- .../management/commands/import_traffic_situations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exceptional_situations/management/commands/import_traffic_situations.py b/exceptional_situations/management/commands/import_traffic_situations.py index 3d9671ffe..d7123a514 100644 --- a/exceptional_situations/management/commands/import_traffic_situations.py +++ b/exceptional_situations/management/commands/import_traffic_situations.py @@ -172,6 +172,5 @@ def handle(self, *args, **options): except AssertionError: continue features = response.json()["features"] - breakpoint() num_imported += self.save_features(features) logger.info(f"Imported/updated {num_imported} traffic situations.") From 1b6ada8e5d93ccc73200e0d468d7b90206021e0d Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:38:16 +0300 Subject: [PATCH 6/6] Remove useless loop --- .../management/commands/delete_all_situations.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/exceptional_situations/management/commands/delete_all_situations.py b/exceptional_situations/management/commands/delete_all_situations.py index c0bc022d7..ca3573ef9 100644 --- a/exceptional_situations/management/commands/delete_all_situations.py +++ b/exceptional_situations/management/commands/delete_all_situations.py @@ -13,8 +13,7 @@ class Command(BaseCommand): def handle(self, *args, **options): - for situation in Situation.objects.all(): - SituationAnnouncement.objects.filter(situation=situation).delete() - situation.delete() - SituationLocation.objects.all().delete() + SituationLocation.objects.all().delete() + SituationAnnouncement.objects.all().delete() + Situation.objects.all().delete() logger.info("Deleted all situations.")