Feature/mobility data search

mobility_data/api/serializers/content_type.py

@@ -16,3 +16,12 @@ class Meta:
             "description_sv",
             "description_en",
         ]
+
+
+class ContentTypeSerializerTrimmed(serializers.ModelSerializer):
+    class Meta:
+        model = ContentType
+        fields = [
+            "id",
+            "type_name",
+        ]

mobility_data/importers/data/content_types.yml

@@ -374,7 +374,7 @@ content_types:
     name:
       fi: Yleinen pöytä
       sv: Allmän bord
-      eb: Public table
+      en: Public table
 
   - content_type_name: PublicBench
     name:
@@ -405,4 +405,16 @@ content_types:
       fi: Nopeusrajoitusalue
       sv: Hastighetsbegränsningszon
       en: Speed limit zone
+
+  - content_type_name: Underpass
+    name:
+      fi: Alikulku
+      sv: Underfartsbro
+      en: Underpass
+
+  - content_Type_name: Overpass
+    name:
+      fi: Ylikulu
+      sv: Överfartsbro
+      en: Overpass
   # End of WFS importer content types

mobility_data/importers/utils.py

@@ -332,6 +332,24 @@ def log_imported_message(logger, content_type, num_created, num_deleted):
     logger.info(f"Deleted {num_deleted} obsolete {name} items")
 
 
+def set_content_type_names(mobile_unit):
+    content_type_names_fi = []
+    content_type_names_sv = []
+    content_type_names_en = []
+    for content_type in mobile_unit.content_types.all():
+        if content_type.name_fi:
+            content_type_names_fi.append(content_type.name_fi)
+        if content_type.name_sv:
+            content_type_names_sv.append(content_type.name_sv)
+        if content_type.name_en:
+            content_type_names_en.append(content_type.name_en)
+
+    setattr(mobile_unit, "content_type_names_fi", content_type_names_fi)
+    setattr(mobile_unit, "content_type_names_sv", content_type_names_sv)
+    setattr(mobile_unit, "content_type_names_en", content_type_names_en)
+    mobile_unit.save()
+
+
 @db.transaction.atomic
 def save_to_database(objects, content_types, logger=logger):
     if type(content_types) != list:
@@ -381,6 +399,7 @@ def save_to_database(objects, content_types, logger=logger):
         ):
             for content_type in content_types:
                 mobile_unit.content_types.add(content_type)
+        set_content_type_names(mobile_unit)
 
     MobileUnit.objects.filter(id__in=objs_to_delete).delete()
     return num_created, len(objs_to_delete)

mobility_data/management/commands/import_mobility_data.py

@@ -25,6 +25,7 @@
     "foli_stops",
     "outdoor_gym_devices",
     "foli_parkandride_stops",
+    "parking_machines",
 ]
 # Read the content type names to be imported
 wfs_content_type_names = get_configured_cotent_type_names()

mobility_data/migrations/0042_add_mobileunit_multilingual_search_columns.py

@@ -0,0 +1,55 @@
+# Generated by Django 4.1.2 on 2023-03-20 08:57
+
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "mobility_data",
+            "0041_contenttype_grouptype_make_name_and_description_multilingual",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="mobileunit",
+            options={},
+        ),
+        migrations.AddField(
+            model_name="mobileunit",
+            name="search_column_en",
+            field=django.contrib.postgres.search.SearchVectorField(null=True),
+        ),
+        migrations.AddField(
+            model_name="mobileunit",
+            name="search_column_fi",
+            field=django.contrib.postgres.search.SearchVectorField(null=True),
+        ),
+        migrations.AddField(
+            model_name="mobileunit",
+            name="search_column_sv",
+            field=django.contrib.postgres.search.SearchVectorField(null=True),
+        ),
+        migrations.AddIndex(
+            model_name="mobileunit",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_column_fi"], name="mobility_da_search__0deecc_gin"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="mobileunit",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_column_sv"], name="mobility_da_search__f31d07_gin"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="mobileunit",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_column_en"], name="mobility_da_search__563f0a_gin"
+            ),
+        ),
+    ]

mobility_data/migrations/0043_add_mobileunit_multilingual_content_type_names.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.2 on 2023-03-20 11:14
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("mobility_data", "0042_add_mobileunit_multilingual_search_columns"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="mobileunit",
+            name="content_type_names_en",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=200), default=list, size=None
+            ),
+        ),
+        migrations.AddField(
+            model_name="mobileunit",
+            name="content_type_names_fi",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=200), default=list, size=None
+            ),
+        ),
+        migrations.AddField(
+            model_name="mobileunit",
+            name="content_type_names_sv",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=200), default=list, size=None
+            ),
+        ),
+    ]

mobility_data/migrations/0044_add_mobileunit_syllables_fi_column.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.2 on 2023-03-21 06:46
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("mobility_data", "0043_add_mobileunit_multilingual_content_type_names"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="mobileunit",
+            name="syllables_fi",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=16), default=list, size=None
+            ),
+        ),
+    ]

mobility_data/migrations/0045_mobileunit_inherit_options_from_basemodel.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.2 on 2023-03-21 12:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("mobility_data", "0044_add_mobileunit_syllables_fi_column"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="mobileunit",
+            options={"ordering": ["-created_time"]},
+        ),
+    ]

mobility_data/models/mobile_unit.py

@@ -2,6 +2,11 @@
 
 from django.conf import settings
 from django.contrib.gis.db import models
+from django.contrib.postgres.fields import ArrayField
+from django.contrib.postgres.indexes import (  # add the Postgres recommended GIN index
+    GinIndex,
+)
+from django.contrib.postgres.search import SearchVectorField
 from munigeo.models import Municipality
 
 from . import ContentType, GroupType
@@ -46,6 +51,13 @@ class MobileUnit(BaseUnit):
     data that are specific for a data source.
     """
 
+    class Meta(BaseUnit.Meta):
+        indexes = (
+            GinIndex(fields=["search_column_fi"]),
+            GinIndex(fields=["search_column_sv"]),
+            GinIndex(fields=["search_column_en"]),
+        )
+
     geometry = models.GeometryField(srid=settings.DEFAULT_SRID, null=True)
     address = models.CharField(max_length=100, null=True)
     municipality = models.ForeignKey(
@@ -65,3 +77,56 @@ class MobileUnit(BaseUnit):
         related_name="mobile_units",
     )
     extra = models.JSONField(null=True)
+    search_column_fi = SearchVectorField(null=True)
+    search_column_sv = SearchVectorField(null=True)
+    search_column_en = SearchVectorField(null=True)
+
+    content_type_names_fi = ArrayField(models.CharField(max_length=200), default=list)
+    content_type_names_sv = ArrayField(models.CharField(max_length=200), default=list)
+    content_type_names_en = ArrayField(models.CharField(max_length=200), default=list)
+
+    syllables_fi = ArrayField(models.CharField(max_length=16), default=list)
+
+    @classmethod
+    def get_search_column_indexing(cls, lang):
+        """
+        Defines the columns to be to_tsvector to the search_column
+        ,config language and weight.
+        """
+        if lang == "fi":
+            return [
+                ("name_fi", "finnish", "A"),
+                ("syllables_fi", "finnish", "A"),
+                ("content_type_names_fi", "finnish", "A"),
+                ("description_fi", "finnish", "B"),
+                ("extra", None, "C"),
+                ("address_zip", None, "D"),
+            ]
+        elif lang == "sv":
+            return [
+                ("name_sv", "swedish", "A"),
+                ("content_type_names_sv", "swedish", "A"),
+                ("description_sv", "swedish", "B"),
+                ("extra", None, "C"),
+                ("address_zip", None, "D"),
+            ]
+        elif lang == "en":
+            return [
+                ("name_en", "english", "A"),
+                ("content_type_names_en", "english", "A"),
+                ("description_en", "english", "B"),
+                ("extra", None, "C"),
+                ("address_zip", None, "D"),
+            ]
+        else:
+            return []
+
+    @classmethod
+    def get_syllable_fi_columns(cls):
+        """
+        Defines the columns that will be used when populating
+        finnish syllables to syllables_fi column. The content
+        will be tokenized to lexems(to_tsvector) and added to
+        the search_column.
+        """
+        return ["name_fi", "content_type_names_fi"]

services/management/commands/empty_search_columns.py

@@ -3,11 +3,12 @@
 from django.core.management.base import BaseCommand
 from munigeo.models import Address, AdministrativeDivision
 
+from mobility_data.models.mobile_unit import MobileUnit
 from services.models import Service, ServiceNode, Unit
 
 logger = logging.getLogger("search")
 
-MODELS = [Address, AdministrativeDivision, Unit, Service, ServiceNode]
+MODELS = [Address, AdministrativeDivision, Unit, Service, ServiceNode, MobileUnit]
 
 
 class Command(BaseCommand):

services/management/commands/index_search_columns.py

@@ -4,7 +4,9 @@
 from django.core.management.base import BaseCommand
 from munigeo.models import Address, AdministrativeDivision
 
+from mobility_data.models.mobile_unit import MobileUnit
 from services.models import Service, ServiceNode, Unit
+from services.search.constants import HYPHENATE_MODELS
 from services.search.utils import hyphenate
 
 logger = logging.getLogger("search")
@@ -33,7 +35,7 @@ def generate_syllables(model):
     """
     # Disable sending of signals
     model._meta.auto_created = True
-    num_populated = 0
+    num_generated_syllables = 0
     for row in model.objects.all():
         row.syllables_fi = []
         for column in model.get_syllable_fi_columns():
@@ -47,11 +49,11 @@ def generate_syllables(model):
                     syllables = hyphenate(word)
                     for s in syllables:
                         row.syllables_fi.append(s)
+                num_generated_syllables += len(row_content)
             row.save()
-            num_populated += 1
     # Enable sending of signals
     model._meta.auto_created = False
-    return num_populated
+    return num_generated_syllables
 
 
 def index_servicenodes(lang):
@@ -91,17 +93,16 @@ def handle(self, *args, **kwargs):
             # Only generate syllables for the finnish language
             if lang == "fi":
                 logger.info(f"Generating syllables for language: {lang}.")
-                logger.info(f"Syllables generated for {generate_syllables(Unit)} Units")
-                logger.info(
-                    f"Syllables generated for {generate_syllables(Service)} Services"
-                )
-                logger.info(
-                    f"Syllables generated for {generate_syllables(ServiceNode)} ServiceNodes"
-                )
-
+                for model in HYPHENATE_MODELS:
+                    logger.info(
+                        f"Generated {generate_syllables(model)} syllables for model {model.__name__}"
+                    )
             logger.info(
                 f"{lang} Units indexed: {Unit.objects.update(**{key: get_search_column(Unit, lang)})}"
             )
+            logger.info(
+                f"{lang} MobileUnits indexed: {MobileUnit.objects.update(**{key: get_search_column(MobileUnit, lang)})}"
+            )
             logger.info(
                 f"{lang} Services indexed: {Service.objects.update(**{key: get_search_column(Service, lang)})}"
             )

services/migrations/0095_add_mobility_data_mobileunit_to_search_view.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.1.2 on 2023-03-20 09:11
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("services", "0094_create_syllables_fi_columns"),
+    ]
+    operations = [
+        migrations.RunSQL(
+            sql="""
+            CREATE OR REPLACE VIEW search_view as
+            SELECT concat('unit_', services_unit.id) AS id, name_fi, name_sv, name_en, search_column_fi, search_column_sv, search_column_en, 'Unit' AS type_name from services_unit
+            UNION
+            SELECT concat('service_', id) AS id, name_fi, name_sv, name_en, search_column_fi, search_column_sv, search_column_en, 'Service' AS type_name from services_service
+            UNION        
+            SELECT concat('servicenode_', string_agg(id::text, '_')) AS ids, name_fi, name_sv, name_en, search_column_fi, search_column_sv, search_column_en, 'ServiceNode' AS type_name from services_servicenode group by 2,3,4,5,6,7,8 
+            UNION
+            SELECT concat('administrativedivision_', id) AS id,  name_fi, name_sv, name_en, search_column_fi, search_column_sv, search_column_en, 'AdministrativeDivision' AS type_name from munigeo_administrativedivision
+            UNION
+            SELECT concat('address_', id) AS id,  full_name_fi as name_fi, full_name_sv as name_sv, full_name_en as name_en, search_column_fi, search_column_sv, search_column_en, 'Address' AS type_name from munigeo_address
+            UNION
+            SELECT concat('mobileunit_', id) AS id,  name_fi ,name_sv, name_en, search_column_fi, search_column_sv, search_column_en, 'MobileUnit' AS type_name from mobility_data_mobileunit;
+            """,
+            reverse_sql="""
+            DROP VIEW search_view;
+            """,
+        ),
+    ]