From fd153092cd17d4ba9b0cd48f28596930f9b44eb5 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 25 Jun 2024 11:21:13 +0300 Subject: [PATCH 1/4] Add lru_cache to functions --- mobility_data/importers/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobility_data/importers/utils.py b/mobility_data/importers/utils.py index 131b8a9d3..431297b46 100644 --- a/mobility_data/importers/utils.py +++ b/mobility_data/importers/utils.py @@ -5,6 +5,7 @@ import tempfile import zipfile from enum import Enum +from functools import lru_cache import requests import yaml @@ -171,6 +172,7 @@ def get_postal_code(point): return postal_code_area.postal_code +@lru_cache() def get_street_name_translations(name, municipality): """ Returns a dict where the key is the language and the value is @@ -197,6 +199,7 @@ def get_street_name_translations(name, municipality): return names +@lru_cache() def get_municipality_name(point): """ Returns the string name of the municipality in which the point From 3812bf4c916b5921464feb23e678bbe1fc768cdb Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Tue, 25 Jun 2024 13:42:40 +0300 Subject: [PATCH 2/4] Update GML fields --- mobility_data/importers/bicycle_stands.py | 29 ++++++++++++++--------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/mobility_data/importers/bicycle_stands.py b/mobility_data/importers/bicycle_stands.py index df59b8c2b..782f99b21 100644 --- a/mobility_data/importers/bicycle_stands.py +++ b/mobility_data/importers/bicycle_stands.py @@ -148,16 +148,18 @@ def set_gml_feature(self, feature): self.extra["model"] = model_elem.as_string() else: self.extra["model"] = None - num_stands_elem = feature["Lukumaara"] - if num_stands_elem is not None: - num = num_stands_elem.as_int() - # for bicycle stands that are Not maintained by Turku - # the number of stands is set to 0 in the input data - # but in reality there is no data so None is set. - if num == 0 and not self.extra["maintained_by_turku"]: - self.extra["number_of_stands"] = None - else: - self.extra["number_of_stands"] = num + + # For some reason the field "lukumaara" has been removed + # num_stands_elem = feature["lukumaara"] + # if num_stands_elem is not None: + # num = num_stands_elem.as_int() + # # for bicycle stands that are Not maintained by Turku + # # the number of stands is set to 0 in the input data + # # but in reality there is no data so None is set. + # if num == 0 and not self.extra["maintained_by_turku"]: + # self.extra["number_of_stands"] = None + # else: + # self.extra["number_of_stands"] = num num_places_elem = feature["Pyorapaikkojen_lukumaara"].as_string() if num_places_elem: @@ -219,7 +221,12 @@ def get_bicycle_stand_objects(): for feature in data_source[1][0]: bicycle_stand = BicyleStand() if data_source[0] == "gml": - bicycle_stand.set_gml_feature(feature) + try: + bicycle_stand.set_gml_feature(feature) + except IndexError as err: + # Handle IndexError gracefully, as the field names can change in the source data + # and it would prevent importing the GEOJSON data. + logger.warning(f"Discarding {feature}, reason: {err}") elif data_source[0] == "geojson": bicycle_stand.set_geojson_feature(feature) if ( From 3bbb23ecfa4b9f0282342008d1d989debb9693f7 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:48:49 +0300 Subject: [PATCH 3/4] Add prefix "Bicycle parking" to names --- mobility_data/importers/bicycle_stands.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mobility_data/importers/bicycle_stands.py b/mobility_data/importers/bicycle_stands.py index 782f99b21..9b1cfbf6f 100644 --- a/mobility_data/importers/bicycle_stands.py +++ b/mobility_data/importers/bicycle_stands.py @@ -56,7 +56,6 @@ class BicyleStand(MobileUnitDataBase): def __init__(self): super().__init__() - self.prefix_name = {} self.related_unit = None self.extra = {f: None for f in self.EXTRA_FIELDS} @@ -110,7 +109,7 @@ def set_geojson_feature(self, feature): else: self.name["sv"] = name self.name["en"] = name - self.prefix_name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()} + self.name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()} addr = feature["osoite"].as_string().split(",") # Some addresses are in format:"Pyhän Henrikin aukio, Kupittaankatu 8, 20520 Turku" # Then remove the prefix part. @@ -188,8 +187,8 @@ def set_gml_feature(self, feature): full_names = get_closest_address_full_name(self.geometry) self.name[FI_KEY] = full_names[FI_KEY] self.name[SV_KEY] = full_names[SV_KEY] - self.name[EN_KEY] = full_names[EN_KEY] - self.prefix_name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()} + self.name[EN_KEY] = full_names[FI_KEY] + self.name = {k: f"{NAME_PREFIX[k]} {v}" for k, v in self.name.items()} def get_data_sources(): From 78e68259f1012368b9f3b821af75289817883ea1 Mon Sep 17 00:00:00 2001 From: juuso-j <68938778+juuso-j@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:49:22 +0300 Subject: [PATCH 4/4] Update tests --- .../tests/test_import_bicycle_stands.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/mobility_data/tests/test_import_bicycle_stands.py b/mobility_data/tests/test_import_bicycle_stands.py index bd67c8624..5677febe1 100644 --- a/mobility_data/tests/test_import_bicycle_stands.py +++ b/mobility_data/tests/test_import_bicycle_stands.py @@ -47,11 +47,15 @@ def test_geojson_import( assert num_created == 3 assert num_deleted == 0 assert MobileUnit.objects.all().count() == 3 - kupittaan_maauimala = MobileUnit.objects.get(name="Kupittaan maauimala") + kupittaan_maauimala = MobileUnit.objects.get( + name="Pyöräpysäköinti Kupittaan maauimala" + ) assert kupittaan_maauimala - kupittaan_palloiluhalli = MobileUnit.objects.get(name="Kupittaan palloiluhalli") + kupittaan_palloiluhalli = MobileUnit.objects.get( + name="Pyöräpysäköinti Kupittaan palloiluhalli" + ) assert kupittaan_palloiluhalli - turun_amk = MobileUnit.objects.get(name="Turun AMK") + turun_amk = MobileUnit.objects.get(name="Pyöräpysäköinti Turun AMK") assert turun_amk assert kupittaan_maauimala.extra["hull_lockable"] is False assert kupittaan_maauimala.extra["covered"] is False @@ -100,8 +104,9 @@ def test_gml_importer( stand_covered_hull_lockable = MobileUnit.objects.all()[1] # 319490982 in fixture xml stand_external = MobileUnit.objects.all()[2] - assert stand_normal.name_fi == "Linnanpuisto" - assert stand_normal.name_sv == "Slottsparken" + assert stand_normal.name_fi == "Pyöräpysäköinti Linnanpuisto" + assert stand_normal.name_sv == "Cykelparkering Slottsparken" + assert stand_normal.name_en == "Bicycle parking Linnanpuisto" assert stand_normal.municipality.name == "Turku" extra = stand_normal.extra assert extra["model"] == "Normaali" @@ -109,19 +114,19 @@ def test_gml_importer( assert extra["covered"] is False assert extra["hull_lockable"] is False assert extra["number_of_places"] == 24 - assert extra["number_of_stands"] == 2 - assert stand_covered_hull_lockable.name == "Pitkäpellonkatu 7" - assert stand_covered_hull_lockable.name_sv == "Långåkersgatan 7" + assert extra["number_of_stands"] is None + assert stand_covered_hull_lockable.name == "Pyöräpysäköinti Pitkäpellonkatu 7" + assert stand_covered_hull_lockable.name_sv == "Cykelparkering Långåkersgatan 7" extra = stand_covered_hull_lockable.extra assert extra["maintained_by_turku"] is True assert extra["covered"] is True assert extra["hull_lockable"] is True assert extra["number_of_places"] == 18 - assert extra["number_of_stands"] == 1 + assert extra["number_of_stands"] is None # external stand has no street name, so the closest street name # and address number is assigned as name and that is "Kupittaankatu 8". - assert stand_external.name == "Kupittaankatu 8" - assert stand_external.name_sv == "Kuppisgatan 8" + assert stand_external.name == "Pyöräpysäköinti Kupittaankatu 8" + assert stand_external.name_sv == "Cykelparkering Kuppisgatan 8" extra = stand_external.extra assert extra["maintained_by_turku"] is False # As there are no info for stand that are not maintained by turku