From 711863fa9e750b8c62e12f5e7891d242aa3b2db1 Mon Sep 17 00:00:00 2001 From: Vivek Subedi Date: Thu, 12 Jul 2018 10:49:51 -0400 Subject: [PATCH 1/4] - updated clipToWorld() method which seems to have bug and returning empty tiles for West coast of USA - added elasticsearch connection timeout on rest client --- .DS_Store | Bin 0 -> 6148 bytes .gitignore | 1 + .../ElasticDataStoreFactory.java | 15 ++- .../elasticsearch/FilterToElasticHelper.java | 89 +++++++++++++++--- 4 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..de2a1cc049f135c6eb10ee7148e31439b161bdd3 GIT binary patch literal 6148 zcmeHK!A=4(5Pe-qfNFB=@+k9iw(&XkiYr-iEg9FfF0aSz1>?1F#oUyu?=pKqOvy_F)?>h`;{+odV~8`1F$}U_ zD%mT}ag81xSmx#INd5(%6>8#~;~rsvQ(}*dU^UTmz+h8N6;K6Kfge|ZXSP_Q6fm{H zc+lq~`Lnh1aB|kh^a))|@(PC=3M=*~X09B?9lABfW2q1mkA*`wWza$u_*Dho E00QrIWdHyG literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore index 8b8025c..39c2981 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ *.log *.iml **idea/ +/.metadata/ diff --git a/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/ElasticDataStoreFactory.java b/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/ElasticDataStoreFactory.java index a72f92b..7c548d0 100644 --- a/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/ElasticDataStoreFactory.java +++ b/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/ElasticDataStoreFactory.java @@ -5,6 +5,7 @@ package mil.nga.giat.data.elasticsearch; import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig.Builder; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.ssl.SSLContextBuilder; import org.elasticsearch.client.RestClient; @@ -52,6 +53,8 @@ public class ElasticDataStoreFactory implements DataStoreFactorySpi { public static final Param SCROLL_SIZE = new Param("scroll_size", Long.class, "Scroll size (ignored if scroll_enabled=false)", false, 20); + public static final Param CONNECTION_TIMEOUT_MILISECONDS = new Param("connection_timeout", Integer.class, "ElasticSearch connection timeout (default 5000 miliseconds)", false, 5000); + public static final Param SCROLL_TIME_SECONDS = new Param("scroll_time", Integer.class, "Time to keep the scroll open in seconds (ignored if scroll_enabled=false)", false, 120); public static final Param ARRAY_ENCODING = new Param("array_encoding", String.class, "Array encoding strategy. Allowed values are \"JSON\" (keep arrays) " @@ -72,6 +75,7 @@ public class ElasticDataStoreFactory implements DataStoreFactorySpi { SOURCE_FILTERING_ENABLED, SCROLL_ENABLED, SCROLL_SIZE, + CONNECTION_TIMEOUT_MILISECONDS, SCROLL_TIME_SECONDS, ARRAY_ENCODING, GRID_SIZE, @@ -132,10 +136,17 @@ public DataStore createDataStore(Map params) throws IOExce final String arrayEncoding = (String) getValue(ARRAY_ENCODING, params); final Boolean sslEnabled = (Boolean) getValue(SSL_ENABLED, params); final Boolean sslRejectUnauthorized = (Boolean) getValue(SSL_REJECT_UNAUTHORIZED, params); + final Integer connectionTimeout = (Integer) getValue(CONNECTION_TIMEOUT_MILISECONDS, params); - final String scheme = sslEnabled ? "https" : "http"; - final RestClientBuilder builder = RestClient.builder(new HttpHost(searchHost, hostPort, scheme)); + final String scheme = sslEnabled ? "https" : "http"; + final RestClientBuilder builder = RestClient.builder(new HttpHost(searchHost, hostPort, scheme)).setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + public Builder customizeRequestConfig(Builder requestConfigBuilder) { + return requestConfigBuilder.setConnectTimeout(connectionTimeout); + } + }); + if (sslEnabled) { builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { @Override diff --git a/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java b/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java index f12c76f..57db973 100644 --- a/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java +++ b/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java @@ -48,10 +48,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.vividsolutions.jts.geom.Coordinate; -import com.vividsolutions.jts.geom.CoordinateFilter; import com.vividsolutions.jts.geom.Envelope; import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.geom.GeometryComponentFilter; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.MultiPolygon; import com.vividsolutions.jts.geom.Polygon; class FilterToElasticHelper { @@ -288,28 +289,90 @@ boolean isCurrentGeography() { } protected Literal clipToWorld(Literal geometry) { - if(geometry != null) { + if (geometry != null) { Geometry g = geometry.evaluate(null, Geometry.class); - if(g != null) { - g.apply(new GeometryComponentFilter() { - @Override - public void filter(Geometry geom) { - geom.apply(new CoordinateFilter() { - @Override - public void filter(Coordinate coord) { - coord.setCoordinate(new Coordinate(clipLon(coord.x),clipLat(coord.y))); + if (g != null) { + Envelope env = g.getEnvelopeInternal(); + // first, limit to world + if (!WORLD.contains(env)) { + g = sanitizePolygons(g.intersection(JTS.toGeometry(WORLD))); + } + + // second, elasticsearch will always use the shortest distance between two + // points, if an arc is longer than 180 degrees the opposite will + // be used instead, so we have to slice the geometry in parts + env = g.getEnvelopeInternal(); + if (Math.sqrt(env.getWidth() * env.getWidth() + env.getHeight() * env.getHeight()) + >= 180) { + // slice in 90x90 degrees quadrants, none of them has a diagonal longer than 180 + final List polygons = new ArrayList(); + for (double lon = Math.floor(env.getMinX()); lon < env.getMaxX(); lon += 90) { + for (double lat = Math.floor(env.getMinY()); + lat < env.getMaxY(); + lat += 90) { + Geometry quadrant = + JTS.toGeometry(new Envelope(lon, lon + 90, lat, lat + 90)); + Geometry cut = sanitizePolygons(g.intersection(quadrant)); + if (!cut.isEmpty()) { + if (cut instanceof Polygon) { + polygons.add((Polygon) cut); + } else { + for (int i = 0; i < cut.getNumGeometries(); i++) { + polygons.add((Polygon) cut.getGeometryN(i)); + } + } } - }); + } } - }); - geometry = CommonFactoryFinder.getFilterFactory(null).literal(g); + g = toPolygon(g.getFactory(), polygons); + } + geometry = CommonFactoryFinder.getFilterFactory(null).literal(g); } } return geometry; } + /** + * Given a geometry that might contain heterogeneous components extracts only the polygonal ones + * + * @param geometry + * @return + */ + private Geometry sanitizePolygons(Geometry geometry) { + // already sane? + if (geometry == null || geometry instanceof Polygon || geometry instanceof MultiPolygon) { + return geometry; + } + + // filter out only polygonal parts + final List polygons = new ArrayList(); + geometry.apply( + new GeometryComponentFilter() { + + public void filter(Geometry geom) { + if (geom instanceof Polygon) { + polygons.add((Polygon) geom); + } + } + }); + + // turn filtered selection into a geometry + return toPolygon(geometry.getFactory(), polygons); + } + + private Geometry toPolygon(GeometryFactory gf, final List polygons) { + if (polygons.size() == 0) { + return gf.createGeometryCollection(null); + } else if (polygons.size() == 1) { + return polygons.get(0); + } else { + return gf.createMultiPolygon( + (Polygon[]) polygons.toArray(new Polygon[polygons.size()])); + } + } + protected double clipLon(double lon) { double x = Math.signum(lon)*(Math.abs(lon)%360); return x = x>180 ? x-360 : (x<-180 ? x+360 : x); From ad088e231e8be520a86defc7df3d93ae8474f134 Mon Sep 17 00:00:00 2001 From: Vivek Subedi Date: Thu, 12 Jul 2018 10:54:19 -0400 Subject: [PATCH 2/4] - just a clean up --- .../mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java b/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java index 57db973..8d86fa6 100644 --- a/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java +++ b/gt-elasticsearch/src/main/java/mil/nga/giat/data/elasticsearch/FilterToElasticHelper.java @@ -375,7 +375,7 @@ private Geometry toPolygon(GeometryFactory gf, final List polygons) { protected double clipLon(double lon) { double x = Math.signum(lon)*(Math.abs(lon)%360); - return x = x>180 ? x-360 : (x<-180 ? x+360 : x); + return x>180 ? x-360 : (x<-180 ? x+360 : x); } protected double clipLat(double lat) { From bb479b10bcc6ea33035525e15fa49ed9263bb367 Mon Sep 17 00:00:00 2001 From: Vivek Subedi Date: Wed, 18 Jul 2018 13:23:31 -0400 Subject: [PATCH 3/4] Updated tests and added one more test to get the point on west coast of USA which is on wifiAccessPoint.json --- .../data/elasticsearch/ElasticGeometryFilterIT.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java b/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java index 17de0df..c5808cb 100644 --- a/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java +++ b/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java @@ -308,7 +308,16 @@ public void testBBOXCoveringDateline() throws Exception { FilterFactory ff = dataStore.getFilterFactory(); BBOX bbox = ff.bbox("geo", 178, -98, 182, 98, "EPSG:" + SOURCE_SRID); SimpleFeatureCollection features = featureSource.getFeatures(bbox); - assertEquals(2, features.size()); + assertEquals(1, features.size()); + } + + @Test + public void testBBOXCoveringDatelineOnWestOfUSA() throws Exception { + init("not-active","geo"); + FilterFactory ff = dataStore.getFilterFactory(); + BBOX bbox = ff.bbox("geo", -180, 0, -90, 66, "EPSG:" + SOURCE_SRID); + SimpleFeatureCollection features = featureSource.getFeatures(bbox); + assertEquals(1, features.size()); } @Test @@ -317,7 +326,7 @@ public void testBBOXBeyondDateline() throws Exception { FilterFactory ff = dataStore.getFilterFactory(); BBOX bbox = ff.bbox("geo", 180.5, -98, 182, 98, "EPSG:" + SOURCE_SRID); SimpleFeatureCollection features = featureSource.getFeatures(bbox); - assertEquals(1, features.size()); + assertEquals(0, features.size()); } @Test From 0b51c16a341011bd5653bf5ebffe8d76dab8e413 Mon Sep 17 00:00:00 2001 From: Vivek Subedi Date: Wed, 18 Jul 2018 13:28:59 -0400 Subject: [PATCH 4/4] just updated the name of test --- .../nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java b/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java index c5808cb..71b97e4 100644 --- a/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java +++ b/gt-elasticsearch/src/test/java/mil/nga/giat/data/elasticsearch/ElasticGeometryFilterIT.java @@ -312,7 +312,7 @@ public void testBBOXCoveringDateline() throws Exception { } @Test - public void testBBOXCoveringDatelineOnWestOfUSA() throws Exception { + public void testBBOXCoveringDatelineOnWestCoastOfUSA() throws Exception { init("not-active","geo"); FilterFactory ff = dataStore.getFilterFactory(); BBOX bbox = ff.bbox("geo", -180, 0, -90, 66, "EPSG:" + SOURCE_SRID);