Baseline

ES/Dockerfile

@@ -0,0 +1,76 @@
+# =================================================================
+#
+# Authors: Just van den Broecke <[email protected]>>
+#          Jorge Samuel Mendes de Jesus <[email protected]>
+#          Tom Kralidis <[email protected]>
+#
+# Copyright (c) 2019 Just van den Broecke
+# Copyright (c) 2019 Jorge Samuel Mendes de Jesus
+# Copyright (c) 2020 Tom Kralidis
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.5.1
+
+LABEL maintainer="[email protected] [email protected]"
+ARG DATA_FOLDER=/usr/share/elasticsearch/data
+
+USER root
+
+COPY docker-entrypoint.sh  /docker-entrypoint.sh
+COPY add_data.sh /add_data.sh
+
+#COPY ./data/* ${DATA_FOLDER}/
+
+RUN yum install -y wget
+
+RUN wget https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -O bin/wait-for-it.sh
+RUN chmod +x bin/wait-for-it.sh
+# RUN wget https://raw.githubusercontent.com/geopython/pygeoapi/master/tests/data/ne_110m_populated_places_simple.geojson -O ${DATA_FOLDER}/ne_110m_populated_places_simple.geojson
+RUN wget https://raw.githubusercontent.com/geopython/pygeoapi/master/tests/load_es_data.py -O /load_es_data.py
+
+RUN echo "xpack.security.enabled: false" >> config/elasticsearch.yml
+RUN echo "http.host: 0.0.0.0" >> config/elasticsearch.yml
+RUN echo "discovery.type: single-node" >> config/elasticsearch.yml
+
+RUN yum --enablerepo=extras -y install epel-release \
+        && yum install -y python3 python3-pip python3-setuptools python-typing \
+        && pip3 install --upgrade pip elasticsearch==7.17.1 elasticsearch-dsl \
+	&& yum clean packages
+
+USER elasticsearch
+
+CMD ["/usr/share/elasticsearch/bin/elasticsearch"]
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+# we need to run this on host
+#sudo sysctl -w vm.max_map_count=262144
+#check indices
+#http://localhost:9200/_cat/indices?v
+#check spatial data
+#http://localhost:9200/ne_110m_populated_places_simple/
+#This docker compose was inspired on:
+#https://discuss.elastic.co/t/best-practice-for-creating-an-index-when-an-es-docker-container-starts/126651
+#docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" es:latest

ES/add_data.sh

@@ -0,0 +1,83 @@
+#!/bin/sh
+# =================================================================
+#
+# Authors: Just van den Broecke <[email protected]>>
+#          Jorge Samuel Mendes de Jesus <[email protected]>
+#
+# Copyright (c) 2019 Just van den Broecke
+# Copyright (c) 2019 Jorge Samuel Mendes de Jesus
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+
+echo "Starting script to add geojson"
+# move to the directory of this setup script
+cd "$(dirname "$0")"
+
+# for some reason even when port 9200 is open Elasticsearch is unable to be accessed as authentication fails
+# a few seconds later it works
+# incresing to 50s for wait in a slow system /_cluster/health?wait_for_status=yellow&timeout=50s
+until $(curl -sSf -XGET --insecure 'http://localhost:9200/_cluster/health?wait_for_status=yellow' > /dev/null); do
+    printf 'No status yellow from ES, trying again in 10 seconds \n'
+    sleep 10
+done
+
+dataset="${DATASET:-obs}"
+
+## ADDING DATA *************************************************
+
+echo "Elasticsearch seems to be working - Adding $dataset to ES"
+
+python3 /load_es_data.py /in/"$dataset".geojson fid
+
+# Waiting for geopackage_pusher container to create the file
+while ! test -f "/in/activity_level_ldn.geojson"; do
+  sleep 10
+  echo "Still waiting"
+done
+python3 /load_es_data.py /in/activity_level_ldn.geojson GSS_CODE
+
+# Waiting for geopackage_pusher container to create the file
+while ! test -f "/in/cardivasular_disease_ldn.geojson"; do
+  sleep 10
+  echo "Still waiting"
+done
+python3 /load_es_data.py /in/cardivasular_disease_ldn.geojson GSS_CODE
+
+# Waiting for geopackage_pusher container to create the file
+while ! test -f "/in/tweet_count_sample.geojson"; do
+  sleep 10
+  echo "Still waiting"
+done
+python3 /load_es_data.py /in/tweet_count_sample.geojson id
+
+echo "Pushing EC metadata"
+python3 /load_es_data.py /in/metadata/ec_catalog.geojson id
+
+echo "Seems that data was loaded"
+
+## *************************************************************
+
+# create a new index with the settings in es_index_config.json
+#curl -v --insecure --user elastic:changeme -XPUT '0.0.0.0:9200/test?pretty' -H 'Content-Type: application/json' -d @es_index_config.json