Merge branch 'dev' into sync-dev-staging-dec-12

wri · Dec 12, 2024 · bbcf363 · bbcf363
2 parents db4a04c + fc2d07b
commit bbcf363
Show file tree

Hide file tree

Showing 26 changed files with 755 additions and 467 deletions.
diff --git a/ckan-backend-dev/docker-compose.dev.yml b/ckan-backend-dev/docker-compose.dev.yml
@@ -81,23 +81,35 @@ services:
       retries: 3
       start_period: 60s
 
-  #my-datapusher:
-  #  container_name: my-datapusher
-  #  build:
-  #    context: ../datapusher
-  #  restart: always
-  #  environment:
-  #    - PREFECT_API_URL=http://prefect:4200/api
-  #    - DEPLOYMENT_NAME=datapusher-dev
-  #    - CKAN_DATASTORE_READ_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
-  #    - CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
-  #    - S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
-  #    - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
-  #    - S3_BUCKET_NAME=ckan
-  #    - S3_BUCKET_REGION=us-east-1
-  #  depends_on:
-  #    prefect:
-  #      condition: service_healthy
+  my-datapusher:
+    container_name: my-datapusher
+    build:
+      context: ../datapusher
+    restart: always
+    environment:
+      - PREFECT_API_URL=http://prefect:4200/api
+      - DEPLOYMENT_NAME=datapusher-dev
+      - CKAN_DATASTORE_READ_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
+      - CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
+      - S3_ACCESS_KEY_ID=${MINIO_ROOT_USER}
+      - S3_SECRET_KEY_ID=${MINIO_ROOT_PASSWORD}
+      - S3_BUCKET_NAME=ckan
+      - S3_BUCKET_REGION=us-east-1
+    depends_on:
+      prefect:
+        condition: service_healthy
+
+  my-migration:
+    container_name: my-migration
+    build:
+      context: ../migration
+    restart: always
+    environment:
+      - FLOW_DEPLOYMENT_ENV=dev
+      - PREFECT_API_URL=http://prefect:4200/api
+    depends_on:
+      prefect:
+        condition: service_healthy
 
   db:
     container_name: ${POSTGRESQL_CONTAINER_NAME}
@@ -124,6 +136,8 @@ services:
   solr:
     container_name: ${SOLR_CONTAINER_NAME}
     image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    build:
+      context: solr/
     volumes:
       - solr_data:/var/solr
       - ./ckan/setup/schema.xml:/opt/solr/server/solr/configsets/ckan/conf/managed-schema

diff --git a/ckan-backend-dev/docker-compose.test.yml b/ckan-backend-dev/docker-compose.test.yml
@@ -146,7 +146,9 @@ services:
 
   solr:
     container_name: ${SOLR_CONTAINER_NAME}
-    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+      #image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    build:
+      context: solr/
     volumes:
       - solr_data:/var/solr
       - ./ckan/setup/schema.xml:/opt/solr/server/solr/configsets/ckan/conf/managed-schema

diff --git a/ckan-backend-dev/solr/Dockerfile b/ckan-backend-dev/solr/Dockerfile
@@ -0,0 +1,57 @@
+FROM solr:8
+
+EXPOSE 8983
+
+ARG CKAN_BRANCH="dev-v2.10"
+
+ENV SOLR_INSTALL="/opt/solr"
+ENV SOLR_CONFIG_DIR="$SOLR_INSTALL/server/solr/configsets"
+ENV SOLR_SCHEMA_FILE="$SOLR_CONFIG_DIR/ckan/conf/managed-schema"
+
+ARG JTS_VERSION="1.16.0"
+ARG JTS_JAR_FILE="$SOLR_INSTALL/server/solr-webapp/webapp/WEB-INF/lib/jts-core-$JTS_VERSION.jar"
+
+USER root
+
+# Create a CKAN configset by copying the default one
+RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan
+
+# Update the schema
+ADD https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml $SOLR_SCHEMA_FILE
+
+# Install JTS JAR file
+ADD https://repo1.maven.org/maven2/org/locationtech/jts/jts-core/$JTS_VERSION/jts-core-$JTS_VERSION.jar \
+    $JTS_JAR_FILE
+RUN chmod 644 $JTS_JAR_FILE
+
+# Add the spatial field type definitions and fields
+
+## RPT
+ENV SOLR_RPT_FIELD_DEFINITION '<fieldType name="location_rpt"   class="solr.SpatialRecursivePrefixTreeFieldType" \
+    spatialContextFactory="JTS"     \
+    autoIndex="true"                \
+    validationRule="repairBuffer0"  \
+    distErrPct="0.025"              \
+    maxDistErr="0.001"              \
+    distanceUnits="kilometers" />'
+
+ENV SOLR_RPT_FIELD '<field name="spatial_geom" type="location_rpt" indexed="true" multiValued="true" />'
+
+RUN sed -i "/<types>/a $SOLR_RPT_FIELD_DEFINITION" $SOLR_SCHEMA_FILE
+RUN sed -i "/<fields>/a $SOLR_RPT_FIELD" $SOLR_SCHEMA_FILE
+
+## BBox
+ENV SOLR_BBOX_FIELDS '<field name="bbox_area" type="float" indexed="true" stored="true" /> \
+    <field name="maxx" type="float" indexed="true" stored="true" /> \
+    <field name="maxy" type="float" indexed="true" stored="true" /> \
+    <field name="minx" type="float" indexed="true" stored="true" /> \
+    <field name="miny" type="float" indexed="true" stored="true" />'
+
+RUN sed -i "/<fields>/a $SOLR_BBOX_FIELDS" $SOLR_SCHEMA_FILE
+
+
+RUN chmod 644 $SOLR_SCHEMA_FILE
+
+USER solr
+
+CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"]
diff --git a/ckan-backend-dev/src/ckanext-wri/ckanext/wri/plugin.py b/ckan-backend-dev/src/ckanext-wri/ckanext/wri/plugin.py
@@ -78,6 +78,11 @@
 import ckanext.issues.logic.action as issue_action
 import queue
 import logging
+import shapely
+from shapely.geometry import mapping
+import shapely.geometry
+import shapely.ops
+import json
 
 log = logging.getLogger(__name__)
 
@@ -409,10 +414,22 @@ def before_dataset_index(self, pkg_dict):
         if any(key in pkg_dict for key in ("authors", "maintainers")):
             pkg_dict = stringify_actor_objects(pkg_dict)
 
-        if not pkg_dict.get("spatial"):
+        if not pkg_dict.get("spatial") and pkg_dict.get('spatial_type') != 'derived_from_resources':
             return pkg_dict
 
-        pkg_dict = SolrSpatialFieldSearchBackend().index_dataset(pkg_dict)
+        if pkg_dict.get("spatial_type") == "derived_from_resources":
+            dataset = logic.get_action("package_show")(
+                {"ignore_auth": True}, {"id": pkg_dict.get('id')}
+            )
+            print("Derived from resources", flush=True)
+            print(f"{len(dataset.get('resources'))} Resources", flush=True)
+            geometries = [shapely.geometry.shape(resource["spatial_geom"]["geometry"]) for resource in dataset.get("resources", []) if resource.get("spatial_geom") and resource.get("spatial_geom").get("geometry")]
+            if len(geometries) == 0:
+                return pkg_dict
+            merged_geom = shapely.ops.unary_union(geometries)
+            pkg_dict["spatial_geom"] = merged_geom.wkt
+        else:
+            pkg_dict = SolrSpatialFieldSearchBackend().index_dataset(pkg_dict)
 
         # Coupled resources are URL -> uuid links, they are not needed in SOLR
         # and might be huge if there are lot of coupled resources
@@ -423,6 +440,8 @@ def before_dataset_index(self, pkg_dict):
         pkg_dict.pop("extras_spatial", None)
         pkg_dict.pop("spatial", None)
 
+        print("PACKAGE DICT", flush=True)
+        print(pkg_dict, flush=True)
         return pkg_dict
 
     def before_dataset_search(self, search_params):

diff --git a/ckan-backend-dev/src/ckanext-wri/ckanext/wri/schema/ckan_dataset.yaml b/ckan-backend-dev/src/ckanext-wri/ckanext/wri/schema/ckan_dataset.yaml
@@ -38,6 +38,19 @@ dataset_fields:
   label: Spatial address
   validators: ignore_missing
 
+- field_name: spatial_type
+  label: Spatial type
+  preset: select
+  choices:
+  - value: address
+    label: Address
+  - value: geom
+    label: Geom
+  - value: global
+    label: Global
+  - value: derived_from_resources
+    label: Derived From Resources
+
 - field_name: spatial
   label: Spatial
   validators: ignore_missing