fix(elasticsearch): add test to spark 3 from snapshot JAR

.travis.yml

@@ -39,7 +39,7 @@ before_script:
   - export -f travis_time_start
   - export -f travis_time_finish
   - export -f travis_nanoseconds
-  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || echo "Logging in to docker"
 jobs:
   include:
     - stage: "Build"

README.md

@@ -582,7 +582,7 @@ Metorikku supports reading/writing with [Apache Hudi](https://github.com/apache/
 Hudi is a very exciting project that basically allows upserts and deletes directly on top of partitioned parquet data.
 
 In order to use Hudi with Metorikku you need to add to your classpath (via ```--jars``` or if running locally with ```-cp```) 
-an external JAR from here: https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.5.3/hudi-spark-bundle_2.12-0.5.3.jar
+an external JAR from here: https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar
 
 To run Hudi jobs you also have to make sure you have the following spark configuration (pass with ```--conf``` or ```-D```):
 ```properties

e2e/cdc/docker-compose.yml

@@ -85,13 +85,13 @@ services:
       - mysql
   # Spark Resources
   spark-master:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     entrypoint:
       - /scripts/entrypoint-master.sh
     logging:
       driver: none
   spark-worker:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     entrypoint:
       - /scripts/entrypoint-worker.sh
     logging:
@@ -108,9 +108,9 @@ services:
       - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081
   # Spark job: Read from CDC Kafka topic, Deserialize according to schema registry, Write to Hudi output
   spark-submit:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --repositories http://packages.confluent.io/maven/ --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar,https://repo1.maven.org/maven2/za/co/absa/abris_2.11/3.2.2/abris_2.11-3.2.2.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --packages io.confluent:kafka-schema-registry-client:5.3.0,io.confluent:kafka-avro-serializer:5.3.0 --conf spark.sql.warehouse.dir=/warehouse --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/kafka/kafka_example_cdc.yaml
+      - SUBMIT_COMMAND=spark-submit --repositories http://packages.confluent.io/maven/ --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar,https://repo1.maven.org/maven2/za/co/absa/abris_2.12/3.2.2/abris_2.12-3.2.2.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --packages io.confluent:kafka-schema-registry-client:5.3.0,io.confluent:kafka-avro-serializer:5.3.0 --conf spark.sql.warehouse.dir=/warehouse --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/kafka/kafka_example_cdc.yaml
       - HIVE_METASTORE_URI=hive:9083
     entrypoint:
       - /scripts/entrypoint-submit.sh
@@ -142,9 +142,9 @@ services:
 #      - 9083:9083
   # Hive test: Select from hive table and assert over the result
   hive-tester:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --conf spark.sql.warehouse.dir=/warehouse --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings /test_metrics/hive_test.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --conf spark.sql.warehouse.dir=/warehouse --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings /test_metrics/hive_test.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/

e2e/elasticsearch/docker-compose.yml

@@ -2,22 +2,22 @@
 version: '3'
 services:
   spark-submit:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --packages commons-httpclient:commons-httpclient:3.0.1 --jars https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch-hadoop/6.8.10/elasticsearch-hadoop-6.8.10.jar --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/elasticsearch/movies.yaml
+      - SUBMIT_COMMAND=spark-submit --packages commons-httpclient:commons-httpclient:3.0.1 --jars https://yotpo-public.s3.amazonaws.com/elasticsearch-hadoop/elasticsearch-spark-30_2.12-8.0.0-SNAPSHOT.jar --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/elasticsearch/movies.yaml
     entrypoint:
       - /scripts/entrypoint-submit.sh
     depends_on:
       - spark-master
       - spark-worker
   spark-master:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     entrypoint:
       - /scripts/entrypoint-master.sh
     logging:
       driver: none
   spark-worker:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     entrypoint:
       - /scripts/entrypoint-worker.sh
     logging:

e2e/hudi/docker-compose.yml

@@ -1,9 +1,9 @@
 version: '3'
 services:
   spark-submit:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/movies.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/movies.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/
@@ -13,9 +13,9 @@ services:
       - spark-master
       - spark-worker
   hive-tester:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/movies_test.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/movies_test.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/
@@ -25,9 +25,9 @@ services:
       - spark-master
       - spark-worker
   spark-submit-manual-hive-sync:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/manual_hive_sync_config.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/manual_hive_sync_config.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/
@@ -37,9 +37,9 @@ services:
       - spark-master
       - spark-worker
   hive-tester-manual-hive-sync:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/manual_hive_sync_test.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/manual_hive_sync_test.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/
@@ -49,9 +49,9 @@ services:
       - spark-master
       - spark-worker
   spark-submit-manual-hive-sync-non-partition:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/manual_hive_sync_no_partitions_config.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.Metorikku metorikku.jar -c examples/hudi/manual_hive_sync_no_partitions_config.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/
@@ -61,9 +61,9 @@ services:
       - spark-master
       - spark-worker
   hive-tester-manual-hive-sync-no-partition:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     environment:
-      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.3/hudi-spark-bundle_2.11-0.5.3.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/manual_hive_sync_no_partitions_test.yaml
+      - SUBMIT_COMMAND=spark-submit --jars https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.12/0.7.0/hudi-spark-bundle_2.12-0.7.0.jar --conf spark.hadoop.mapreduce.input.pathFilter.class=org.apache.hudi.hadoop.HoodieROTablePathFilter --class com.yotpo.metorikku.MetorikkuTester metorikku.jar --test-settings examples/hudi/manual_hive_sync_no_partitions_test.yaml
       - HIVE_METASTORE_URI=hive:9083
     volumes:
       - ./output/:/examples/output/
@@ -73,13 +73,13 @@ services:
       - spark-master
       - spark-worker
   spark-master:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     entrypoint:
       - /scripts/entrypoint-master.sh
     logging:
       driver: none
   spark-worker:
-    image: metorikku/metorikku:spark2_standalone
+    image: metorikku/metorikku:standalone
     entrypoint:
       - /scripts/entrypoint-worker.sh
     volumes: