Merge pull request #720 from City-of-Helsinki/UHF-9201_elastic_streets

UHF-9201 elasticsearh streets data

conf/cmi/search_api.index.street_data.yml

@@ -0,0 +1,51 @@
+uuid: 0f434f1f-19a0-45a2-84bc-9f81e52ade45
+langcode: fi
+status: true
+dependencies:
+  config:
+    - search_api.server.elastic_kymp
+  module:
+    - search_api
+    - helfi_react_search
+    - helfi_kymp_content
+id: street_data
+name: 'Street data'
+description: 'Street data for street maintenance class search.'
+read_only: false
+field_settings:
+  id:
+    label: id
+    datasource_id: helfi_street_data_source
+    property_path: id
+    type: integer
+  maintenance_class:
+    label: 'Maintenance class'
+    datasource_id: helfi_street_data_source
+    property_path: maintenance_class
+    type: integer
+  street_name:
+    label: 'Street name'
+    datasource_id: helfi_street_data_source
+    property_path: street_name
+    type: string
+datasource_settings:
+  helfi_street_data_source: {  }
+processor_settings:
+  add_url: {  }
+  aggregated_field: {  }
+  custom_value: {  }
+  district_image_absolute_url: {  }
+  entity_type: {  }
+  language_with_fallback: {  }
+  project_execution_schedule: {  }
+  project_image_absolute_url: {  }
+  project_plan_schedule: {  }
+  rendered_item: {  }
+tracker_settings:
+  default:
+    indexing_order: fifo
+options:
+  cron_limit: 50
+  index_directly: false
+  track_changes_in_references: true
+server: elastic_kymp

public/modules/custom/helfi_kymp_content/helfi_kymp_content.services.yml

@@ -0,0 +1,11 @@
+services:
+  logger.channel.helfi_kymp_content:
+    parent: logger.channel_base
+    arguments:
+      - 'helfi_kymp_content'
+
+  helfi_kymp_content.search_api_subscriber:
+    class: Drupal\helfi_kymp_content\EventSubscriber\SearchApiSubscriber
+    arguments: ['@logger.channel.helfi_kymp_content']
+    tags:
+      - { name: event_subscriber }

public/modules/custom/helfi_kymp_content/src/EventSubscriber/SearchApiSubscriber.php

@@ -0,0 +1,59 @@
+<?php
+
+namespace Drupal\helfi_kymp_content\EventSubscriber;
+
+use Drupal\search_api\Event\ReindexScheduledEvent;
+use Drupal\search_api\Event\SearchApiEvents;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+/**
+ * Subscribe to search api events.
+ */
+class SearchApiSubscriber implements EventSubscriberInterface {
+
+  /**
+   * The constructor.
+   *
+   * @param Psr\Log\LoggerInterface $logger
+   *   The logger.
+   */
+  public function __construct(private LoggerInterface $logger) {
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public static function getSubscribedEvents(): array {
+    return [
+      SearchApiEvents::REINDEX_SCHEDULED => ['reindex'],
+    ];
+  }
+
+  /**
+   * Tell tracker which IDs to index on next indexing.
+   *
+   * @param Drupal\search_api\Event\ReindexScheduledEvent $event
+   *   The event.
+   *
+   * @throws \Drupal\search_api\SearchApiException
+   */
+  public function reindex(ReindexScheduledEvent $event): void {
+    $index = $event->getIndex();
+    if ($index->id() == 'street_data') {
+      $source = $index->getDatasource('helfi_street_data_source');
+      try {
+        $data = $source->loadMultiple([]);
+        if (!$data) {
+          return;
+        }
+      }
+      catch (\Exception $e) {
+        $this->logger->error('unable to fetch data while running reindex event');
+      }
+
+      $index->trackItemsInserted($source->getPluginId(), array_keys($data));
+    }
+  }
+
+}

public/modules/custom/helfi_kymp_content/src/Plugin/DataType/StreetData.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace Drupal\helfi_kymp_content\Plugin\DataType;
+
+use Drupal\Core\TypedData\DataDefinition;
+use Drupal\Core\TypedData\Plugin\DataType\Map;
+
+/**
+ * Street data type.
+ *
+ * @DataType(
+ *   id = "street_data",
+ *   label = @Translation("Street data"),
+ *   constraints = {},
+ *   definition_class = "\Drupal\helfi_kymp_content\TypedData\StreetDataDefinition"
+ * )
+ */
+class StreetData extends Map {
+
+  /**
+   * Get property definition.
+   */
+  public static function propertyDefinitions(): array {
+    $properties = [];
+
+    $properties['id'] = DataDefinition::create('integer')
+      ->setLabel('id')
+      ->setRequired(TRUE);
+
+    $properties['street_name'] = DataDefinition::create('string')
+      ->setLabel('Street name')
+      ->addConstraint('Range', ['min' => 0, 'max' => 255])
+      ->setRequired(TRUE);
+
+    $properties['maintenance_class'] = DataDefinition::create('integer')
+      ->setLabel('Maintenance class')
+      ->addConstraint('Range', ['min' => 0, 'max' => 5])
+      ->setRequired(TRUE);
+
+    return $properties;
+  }
+
+}

public/modules/custom/helfi_kymp_content/src/Plugin/search_api/datasource/HelfiStreetDataSource.php

@@ -0,0 +1,169 @@
+<?php
+
+namespace Drupal\helfi_kymp_content\Plugin\search_api\datasource;
+
+use Drupal\Core\Language\LanguageInterface;
+use Drupal\Core\TypedData\ComplexDataInterface;
+use Drupal\Core\TypedData\DataDefinition;
+use Drupal\Core\TypedData\TypedDataTrait;
+use Drupal\search_api\Datasource\DatasourceInterface;
+use Drupal\search_api\Datasource\DatasourcePluginBase;
+use GuzzleHttp\ClientInterface;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Provides a datasource for kartta.hel.fi.
+ *
+ * @SearchApiDatasource(
+ *   id = "helfi_street_data_source",
+ *   label = @Translation("Helfi street datasource"),
+ *   description = @Translation("Datasource for street data from kartta.hel.fi."),
+ * )
+ */
+class HelfiStreetDataSource extends DatasourcePluginBase implements DatasourceInterface {
+
+  use TypedDataTrait;
+
+  public const API_URL = 'https://kartta.hel.fi/ws/geoserver/avoindata/wfs';
+
+  /**
+   * The client.
+   *
+   * @var GuzzleHttp\ClientInterface
+   */
+  protected ClientInterface $client;
+
+  /**
+   * The logger.
+   *
+   * @var Psr\Log\LoggerInterface
+   */
+  protected LoggerInterface $logger;
+
+  /**
+   * {@inheritDoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
+    $instance = parent::create($container, $configuration, $plugin_id, $plugin_definition);
+    $instance->client = $container->get('http_client');
+    $instance->logger = $container->get('logger.channel.helfi_kymp_content');
+    return $instance;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getItemId(ComplexDataInterface $item): null|string {
+    return $item['id'];
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function load($id) {
+    $items = $this->loadMultiple([$id]);
+    return $items ? reset($items) : NULL;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function loadMultiple(array $ids): array {
+    $query = http_build_query([
+      'request' => 'GetFeature',
+      'service' => 'WFS',
+      'version' => '1.1.0',
+      'typeName' => 'avoindata:YLRE_Katualue_alue',
+      'propertyname' => 'avoindata:kadun_nimi,avoindata:kayttotarkoitus,avoindata:yllapitoluokka,avoindata:pituus',
+    ]);
+    $uri = sprintf('%s?%s', 'https://kartta.hel.fi/ws/geoserver/avoindata/wfs', $query);
+
+    try {
+      $content = $this->client->request('GET', $uri);
+      $xmlResult = $content->getBody()->getContents();
+      if (!$xmlResult) {
+        return [];
+      }
+    }
+    catch (\Exception $e) {
+      $this->logger->error("Errors while fetching street data from kartta.hel.fi: {$e->getMessage()}");
+      return [];
+    }
+
+    libxml_use_internal_errors(TRUE);
+    $doc = new \DOMDocument(encoding: 'UTF-8');
+    $doc->loadXML($xmlResult);
+    $errors = libxml_get_errors();
+
+    if ($errors) {
+      $this->logger->error('Errors while parsing street data xml string.');
+      return [];
+    }
+
+    $data = [];
+    foreach ($doc->firstChild->firstChild->childNodes->getIterator() as $street_data) {
+      foreach ($street_data->childNodes->getIterator() as $field) {
+        switch ($field->nodeName) {
+          case 'avoindata:katualue_id':
+            $id = $field->nodeValue;
+            $single_street['id'] = $ids && $id && in_array($id, $ids) ? $id : NULL;
+            break;
+
+          case 'avoindata:kadun_nimi':
+            $single_street['street_name'] = $field->nodeValue;
+            break;
+
+          case 'avoindata:yllapitoluokka':
+            // Turn field value from III or II to 3 or 2 etc.
+            $single_street['maintenance_class'] = strlen($field->nodeValue);
+            break;
+        }
+
+        if (!$id) {
+          continue;
+        }
+      }
+
+      if ($ids && $id && !in_array($id, $ids)) {
+        continue;
+      }
+
+      $street_data_definition = $this->getTypedDataManager()->createDataDefinition('street_data');
+      $street_data = $this->getTypedDataManager()->create($street_data_definition);
+      $street_data->setValue($single_street);
+      $data[$id] = $street_data;
+    }
+
+    return $data;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getItemLanguage(ComplexDataInterface $item): string {
+    return LanguageInterface::LANGCODE_NOT_SPECIFIED;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getPropertyDefinitions(): array {
+    $property_definition = [];
+
+    $property_definition['id'] = DataDefinition::create('integer')
+      ->setLabel('id')
+      ->setRequired(TRUE);
+    $property_definition['street_name'] = DataDefinition::create('string')
+      ->setLabel('Street name')
+      ->addConstraint('Range', ['min' => 0, 'max' => 255])
+      ->setRequired(TRUE);
+    $property_definition['maintenance_class'] = DataDefinition::create('integer')
+      ->setLabel('Maintenance class')
+      ->addConstraint('Range', ['min' => 0, 'max' => 5])
+      ->setRequired(TRUE);
+
+    return $property_definition;
+  }
+
+}

public/modules/custom/helfi_kymp_content/src/TypedData/StreetDataDefinition.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace Drupal\helfi_kymp_content\TypedData;
+
+use Drupal\Core\TypedData\ComplexDataDefinitionBase;
+use Drupal\Core\TypedData\DataDefinition;
+
+/**
+ * The street data definition.
+ */
+class StreetDataDefinition extends ComplexDataDefinitionBase {
+
+  /**
+   * {@inheritDoc}
+   */
+  public function getPropertyDefinitions() {
+    if (!isset($this->propertyDefinitions)) {
+      $this->propertyDefinitions['id'] = DataDefinition::create('integer')
+        ->setLabel('id')
+        ->setRequired(TRUE);
+      $this->propertyDefinitions['street_name'] = DataDefinition::create('string')
+        ->setLabel('Street name')
+        ->addConstraint('Range', ['min' => 0, 'max' => 255])
+        ->setRequired(TRUE);
+      $this->propertyDefinitions['maintenance_class'] = DataDefinition::create('integer')
+        ->setLabel('Maintenance class')
+        ->addConstraint('Range', ['min' => 0, 'max' => 5])
+        ->setRequired(TRUE);
+    }
+    return $this->propertyDefinitions;
+  }
+
+}