Merge branch 'master' into suppo-1256-readme

rosette-api · Jul 28, 2020 · 65fcd17 · 65fcd17
2 parents b6382dd + 64db3c6
commit 65fcd17
Show file tree

Hide file tree

Showing 12 changed files with 263 additions and 52 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -13,5 +13,6 @@ sudo: required
 
 notifications:
   slack:
-    rooms:
-      - secure: 4FRaTAAiYyeUvgw2RhmblgbNiJO4wmd34OBgWcwURjP9oVmFfSwR9r1LNCdUGxrPOghexSY2DjXIuvIrfTfi/xYbhHb3Kw7PEAyB8IuBMlKtY4NSFou62S2VhYpxyg58T+C7P2zi0eDnDE06pwTCoGPaimxMZQY91yQ0yPYDPVXbwe5SjEgamzlwGBxlS/0A6w1iCPHg27/iO2hXtdW3oLS2I0F/Q8Q95RBkX9hpg6yqHlTV7jRbSqvQ9OFBqk/tXMHQvhoPDGgCgQDuykJuaAYx7g9d0YL0eEYYOh9B/TJ/kNOwdRFBu5kuQ2/nFS5Z0S3Y3UIhdYjUmm9gSMnwIbYnrW22EqDJLoT9Zi3Gv7Prg/8/fSkWsof7BJTMSuXUqO1AxDGKIxFv9uSF1daZoY+AC1ooU1xDu1nNvWVYPlkwEdDxxmHpFkGT3ESTZYccPovQl8Z5K0I1BBAVdJKDzm07lE6VHbxkKcvK6gG0TN3uLxnSlQtjkfJ+aVMq1kxeVsB9lEsKs9oezsKzzbftMm525aXPg+OAv+31CUFWxvT/p4ps8Q+AV6aZpoPHkpK8VryyNirUeZ/m4m4ebDHhD9vcN+JqE9gzshT+0U3g19SvLiUMQtbuZ2BUvrq2hh2LEGs03AFZaNg9AEUVA1PQRhV5NILyoS/lbiBYJPT39Sg=
+    on_failure: always
+    on_success: always
+    secure: YUeHcLIxXFLOokSc8ljuRWfA8q02+1Us8dalaRLWatJXqnGTGasbPc7PPeg0/hesjJA6j4KTohbtLCjt0Eto+wR5ENG5Yeyy7tP3FlWPA2AQJ4Xl71zqrKO9q8C/FDw6dKJxXeEbACN7HFvBETrQ6643bNbz559FQA2iQOLi6fk4fMJs04GQBGj1zosvCVJTafzRuReYau5kfyBGqnVBDTnmoNYirgMFAyeg8mTM5NAUWPfDdl4r6DzdXKIGV05wgLzDOdDwhJmBcrFAKss5xv49aIbPTzCyQbOqNgOBUy1sXjNI8RMRgkUtFnU+FRQXFx5jsCJwvwdWBHyFjxKaJvz14ZYqKb8i4GRLdUj95FbWKvJnmexaoCkA9MbyqO6SLtviqhTl8oZjLUdfMX7htJgJtjM0u1A+ZbaHH7NsTNVtfdlCYTQ4M1ZZHy/cTPQ08OIe62nrbcQiCWuOTxukhZnRO5pctmI9BZGVhGNSxssDlM5vFMacQMXBEHGKq8PqtyLP7bzJriX1rSUFXwsCp5SYqozvcrvsy7wOK5mHhiUDQmadXNQXb3evha7uJ+u402AlkCObLULLoAs6yRZzL1g9C8hNRp8gplj7+ni9rvGtr4vZi7Gp/HnENhqUcP7/LbY6RKH9npFm6QlYoaLDOcFC6Ky4JnK749NtHqP3+bY=
diff --git a/docker/test.sh b/docker/test.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 code=0
+command_counter=0
 set -x
 
 # uses '|| code=$?' to make sure that even if there is an error,
@@ -19,12 +20,17 @@ curl -fsSL -H 'Content-Type: application/json' -XPUT "$1:$2/_ingest/pipeline/my_
   ]
 }
 ' || ((code++))
+echo
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 curl -fsSL -H 'Content-Type: application/json' -XPOST "$1:$2/indexname/mappingName?pipeline=my_pipeline&pretty" -d'
 {
   "text" : "This is a document containing English text"
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/_ingest/pipeline/rosapi?pretty" -d'
 {
@@ -77,6 +83,8 @@ curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/_ingest/pipeline/ros
   ]
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 #Pipeline without categories since it only supports English
 curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/_ingest/pipeline/rosapi_jpn?pretty" -d'
@@ -124,6 +132,8 @@ curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/_ingest/pipeline/ros
   ]
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/test_idx?include_type_name=true&pretty" -d'
 {
@@ -142,25 +152,33 @@ curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/test_idx?include_typ
   }
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/test_idx/rosette/1?pretty&refresh=true&pipeline=rosapi" -d'
 {
   "text": "Original Ghostbuster Dan Aykroyd, who also co-wrote the 1984 Ghostbusters film, couldn’t be more pleased with the new all-female Ghostbusters cast, telling The Hollywood Reporter, “The Aykroyd family is delighted by this inheritance of the Ghostbusters torch by these most magnificent women in comedy.”"
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/test_idx/rosette/2?pretty&refresh=true&pipeline=rosapi_jpn" -d'
 {
   "text": "バングラデシュ政府、ロヒンギャ難民の島への移動を計画 \nバングラデシュ政府、ロヒンギャ難民の島への移動を計画\n\nテンガール・チャール島は約10年前に、メグナ川の堆積土で形成され、高潮の際には数十センチの水に囲まれてしまう。道路や堤防などは築かれておらず、島を記載する地図はあまりない。\n\n約30キロ西には60万人が住むハティｱ島があり、現在の難民キャンプからの移動には9時間かかる。\n\nある地元政府関係者はAFP通信に対し、テンガール・チャール島について、「島に行けるのは冬のみで、海賊たちの隠れ家になっている」と語った。島を洪水から守るため植樹が行われているが、完了するまでには少なくとも10年がかかるという。同関係者は、「モンスーンの季節には完全に水浸しになってしまう」と話し、「あそこに住まわせるというのは、ひどいアイデアだ」と指摘した。\n\nImage caption 移住が計画されているテンガール・チャール島はハティア（Hatiya）島の近くにある\n\nミャンマーでは、ロヒンギャの人々は国境を接するバングラデシュからの不法移民として扱われており、国籍の取得ができずにいる。\n\n"
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 curl -fsSL -H "Content-Type: application/json" -XPUT "$1:$2/test_idx/rosette/3?pretty&refresh=true&pipeline=rosapi" -d'
 {
   "text" : "Vladimir Vladimirovich Nabokov was a Russian-American novelist and entomologist. His first nine novels were in Russian, and he achieved international prominence after he began writing English prose.",
   "name" : "Vladimir Nabokov"
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
+command_counter=$((command_counter + 1))
 
 sleep 3
 
@@ -175,6 +193,7 @@ curl -fsSL -H "Content-Type: application/json" -XPOST "$1:$2/test_idx/_search?pr
   }
 }
 ' || ((code++))
+echo "After command ${command_counter} our error code count is ${code}."
 
 set +x
 echo "exit: $code"
diff --git a/plugin/pom.xml b/plugin/pom.xml
@@ -39,13 +39,32 @@
         </site>
     </distributionManagement>
     <properties>
-        <mockserver.version>3.9.1</mockserver.version>
+        <commons-codec.version>1.14</commons-codec.version>
+        <commons-lang3.version>3.11</commons-lang3.version>
+        <findbugs.version>3.0.2</findbugs.version>
+        <joda-time.version>2.10.6</joda-time.version>
+        <jopt-simple.version>5.0.4</jopt-simple.version>
+        <json-schema-validator.version>2.2.14</json-schema-validator.version>
+        <mockserver.version>5.11.1</mockserver.version>
         <mockserver.baseurl>/rest/worker/v1/</mockserver.baseurl>
+        <hamcrest.version>2.2</hamcrest.version>
+        <slf4j-ext.version>1.7.30</slf4j-ext.version>
     </properties>
     <dependencies>
         <dependency>
             <groupId>com.basistech.rosette</groupId>
             <artifactId>rosette-api</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>${commons-codec.version}</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -60,6 +79,20 @@
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
             <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>net.sf.jopt-simple</groupId>
+                    <artifactId>jopt-simple</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>joda-time</groupId>
+                    <artifactId>joda-time</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.hamcrest</groupId>
+                    <artifactId>hamcrest</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
@@ -80,23 +113,139 @@
                     <groupId>commons-codec</groupId>
                     <artifactId>commons-codec</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.hamcrest</groupId>
+                    <artifactId>hamcrest</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mock-server</groupId>
+            <artifactId>mockserver-client-java</artifactId>
+            <version>${mockserver.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion> <!-- This conflicts with a library from rosette api binding. -->
+                    <groupId>jakarta.validation</groupId>
+                    <artifactId>jakarta.validation-api</artifactId>
+                </exclusion>
+                <exclusion> <!-- Conflicts with rosette api binding -->
+                    <groupId>jakarta.xml.bind</groupId>
+                    <artifactId>jakarta.xml.bind-api</artifactId>
+                </exclusion>
+                <exclusion> <!-- Java 11 -->
+                    <groupId>javax.xml.bind</groupId>
+                    <artifactId>jaxb-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.github.java-json-tools</groupId>
+                    <artifactId>json-schema-validator</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-lang3</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-ext</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.mock-server</groupId>
             <artifactId>mockserver-netty</artifactId>
             <version>${mockserver.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>ch.qos.logback</groupId>
                     <artifactId>logback-classic</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.github.java-json-tools</groupId>
+                    <artifactId>json-schema-validator</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>joda-time</groupId>
+                    <artifactId>joda-time</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>net.sf.jopt-simple</groupId>
+                    <artifactId>jopt-simple</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-lang3</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.hamcrest</groupId>
+                    <artifactId>hamcrest</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-ext</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.github.java-json-tools</groupId>
+            <artifactId>json-schema-validator</artifactId>
+            <version>${json-schema-validator.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>joda-time</groupId>
+                    <artifactId>joda-time</artifactId>
+                </exclusion>
             </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+            <version>${findbugs.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.sf.jopt-simple</groupId>
+            <artifactId>jopt-simple</artifactId>
+            <version>${jopt-simple.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>${commons-lang3.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest</artifactId>
+            <version>${hamcrest.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-ext</artifactId>
+            <version>${slf4j-ext.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>${joda-time.version}</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -170,6 +319,13 @@
                         <configuration>
                             <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
                             <createSourcesJar>true</createSourcesJar>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" />
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer" />
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                    <resource>META-INF/maven/dependencies.properties</resource>
+                                </transformer>
+                            </transformers>
                             <artifactSet>
                                 <!-- have to include all dependencies that use jackson due to relocation -->
                                 <includes>
@@ -179,6 +335,11 @@
                                     <include>com.basistech:common-api-jackson</include>
                                     <include>com.fasterxml.jackson.core:*</include>
                                 </includes>
+                                <!-- This blob clears the shade plugin warning, but breaks the runtime classpath -->
+                                <!-- <excludes>
+                                    <artifact>*:*</artifact>
+                                    <exclude>META-INF/MANIFEST.MF</exclude>
+                                </excludes> -->
                             </artifactSet>
                             <relocations>
                                 <relocation>

diff --git a/plugin/src/test/java/com/rosette/elasticsearch/CategoriesProcessorTest.java b/plugin/src/test/java/com/rosette/elasticsearch/CategoriesProcessorTest.java
@@ -1,5 +1,5 @@
 /*
-* Copyright 2017 Basis Technology Corp.
+* Copyright 2020 Basis Technology Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
 
 import java.util.HashMap;
@@ -35,7 +36,7 @@ public void testCategories() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
 
-        assertThat(ingestDocument.getSourceAndMetadata().get("text"), Matchers.equalTo(inputText));
-        assertThat(ingestDocument.getSourceAndMetadata().get("category"), Matchers.equalTo("SPORTS"));
+        MatcherAssert.assertThat(ingestDocument.getSourceAndMetadata().get("text"), Matchers.equalTo(inputText));
+        MatcherAssert.assertThat(ingestDocument.getSourceAndMetadata().get("category"), Matchers.equalTo("SPORTS"));
     }
 }