feat(Hudi): bump version to 0.5.1 (#281)

* feat(Hudi): bump version to 0.5.1 & bump hive version to 2.3.3
YotpoLtd · Mar 23, 2020 · 6fd161e · 6fd161e
1 parent 849da0e
commit 6fd161e
Show file tree

Hide file tree

Showing 25 changed files with 551 additions and 65 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,8 @@ derby.log
 metastore_db/
 out/
 metorikku.iml
-examples/output
+examples/output
+e2e/*/output
+e2e/*/*/output
+e2e/*/warehouse
+e2e/*/*/warehouse
diff --git a/.travis.yml b/.travis.yml
@@ -15,9 +15,12 @@ branches:
     - /^v[\d\.]+$/
 env:
   global:
-    - SPARK_VERSION=2.4.4
+    - SPARK_VERSION=2.4.5
     - HADOOP_VERSION=2.9.2
-    - HIVE_VERSION=1.2.2
+    - HIVE1_VERSION=1.2.2
+    - HUDI_HIVE1_VERSION=0.4.7
+    - HIVE_VERSION=2.3.3
+    - HUDI_VERSION=0.5.1-incubating
 before_script:
   - docker pull $(grep -ioP '(?<=^from)\s+\S+' docker/spark/Dockerfile)
   - docker pull metorikku/spark:latest
@@ -45,7 +48,8 @@ script:
   # Create all relevant docker images
   - travis_fold start "docker"
   - docker build -t metorikku/spark:$SPARK_VERSION --cache-from metorikku/spark:latest -t metorikku/spark:latest --build-arg SPARK_VERSION=$SPARK_VERSION --build-arg HADOOP_VERSION=$HADOOP_VERSION -f docker/spark/Dockerfile docker/spark || travis_terminate 1;
-  - docker build -t metorikku/hive:$HIVE_VERSION --cache-from metorikku/hive:latest -t metorikku/hive:latest --build-arg HIVE_VERSION=$HIVE_VERSION -f docker/hive/Dockerfile docker/hive || travis_terminate 1;
+  - docker build -t metorikku/hive:1 --cache-from metorikku/hive:1 -t metorikku/hive:HIVE1_VERSION --build-arg HIVE_VERSION=$HIVE1_VERSION --build-arg HUDI_HIVE1_VERSION=$HUDI_HIVE1_VERSION -f docker/hive1/Dockerfile docker/hive1 || travis_terminate 1;
+  - docker build -t metorikku/hive --cache-from metorikku/hive -t metorikku/hive:HIVE_VERSION --build-arg HIVE_VERSION=$HIVE_VERSION --build-arg HUDI_VERSION=$HUDI_VERSION -f docker/hive/Dockerfile docker/hive || travis_terminate 1;
   - docker build -t metorikku/metorikku:latest -f docker/metorikku/Dockerfile . || travis_terminate 1;
   - travis_fold end "docker"
   # Kafka to Kafka E2E
@@ -56,10 +60,14 @@ script:
   - travis_fold start "file_to_influx_e2e"
   - (cd e2e/influxdb && ./test.sh) || travis_terminate 1;
   - travis_fold end "file_to_influx_e2e"
-  # File to hive E2E
+  # File to hive2 E2E
   - travis_fold start "file_to_hive_e2e"
   - (cd e2e/hive && ./test.sh) || travis_terminate 1;
   - travis_fold end "file_to_hive_e2e"
+  # File to hive1 E2E
+  - travis_fold start "file_to_hive1_e2e"
+  - (cd e2e/hive1 && ./test.sh) || travis_terminate 1;
+  - travis_fold end "file_to_hive1_e2e"
   # File to Elasticsearch E2E
   - travis_fold start "file_to_elasticsearch_e2e"
   - (cd e2e/elasticsearch && ./test.sh) || travis_terminate 1;
@@ -81,7 +89,7 @@ deploy:
       condition: ($TRAVIS_BRANCH = master) || ($TRAVIS_TAG =~ ^v.*)
   - provider: script
     skip_cleanup: true
-    script: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin && docker tag metorikku/metorikku metorikku/metorikku:"$TRAVIS_TAG"_spark_"$SPARK_VERSION" && docker push metorikku/spark && docker push metorikku/hive && docker push metorikku/metorikku
+    script: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin && docker tag metorikku/metorikku metorikku/metorikku:"$TRAVIS_TAG"_spark_"$SPARK_VERSION" && docker push metorikku/spark && docker push metorikku/hive && docker push metorikku/hive:1 && docker push metorikku/metorikku
     on:
       all_branches: true
       condition: $TRAVIS_TAG =~ ^v.*

diff --git a/README.md b/README.md
@@ -265,7 +265,7 @@ This will commit the offsets to kafka, as a new dummy consumer group.
 * we use ABRiS as a provided jar In order to deserialize your kafka stream messages (https://github.com/AbsaOSS/ABRiS), add the  ```schemaRegistryUrl``` option to the kafka input config
 spark-submit command should look like so:
 
-```spark-submit --repositories http://packages.confluent.io/maven/ --jars https://repo1.maven.org/maven2/za/co/absa/abris_2.11/3.1.1/abris_2.11-3.1.1.jar --packages org.apache.spark:spark-avro_2.11:2.4.4,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.4,io.confluent:kafka-schema-registry-client:5.3.0,io.confluent:kafka-avro-serializer:5.3.0 --class com.yotpo.metorikku.Metorikku metorikku.jar```
+```spark-submit --repositories http://packages.confluent.io/maven/ --jars https://repo1.maven.org/maven2/za/co/absa/abris_2.11/3.1.1/abris_2.11-3.1.1.jar --packages org.apache.spark:spark-avro_2.11:2.4.5,org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.5,io.confluent:kafka-schema-registry-client:5.3.0,io.confluent:kafka-avro-serializer:5.3.0 --class com.yotpo.metorikku.Metorikku metorikku.jar```
 
 * If your subject schema name is not ```<TOPIC NAME>-value``` (e.g. if the topic is a regex pattern) you can specify the schema subject in the ```schemaSubject``` section
 
@@ -447,7 +447,7 @@ Metorikku supports reading/writing with [Apache Hudi](https://github.com/apache/
 Hudi is a very exciting project that basically allows upserts and deletes directly on top of partitioned parquet data.
 
 In order to use Hudi with Metorikku you need to add to your classpath (via ```--jars``` or if running locally with ```-cp```) 
-an external JAR from here: https://repo1.maven.org/maven2/com/uber/hoodie/hoodie-spark-bundle/0.4.7/hoodie-spark-bundle-0.4.7.jar
+an external JAR from here: https://repo1.maven.org/maven2/org/apache/hudi/hudi-spark-bundle_2.11/0.5.1-incubating/hudi-spark-bundle_2.11-0.5.1-incubating.jar
 
 To run Hudi jobs you also have to make sure you have the following spark configuration (pass with ```--conf``` or ```-D```):
 ```properties

diff --git a/build.sbt b/build.sbt
@@ -18,15 +18,16 @@ developers := List(
 )
 
 scalaVersion := "2.11.12"
-val sparkVersion = Option(System.getProperty("sparkVersion")).getOrElse("2.4.4")
+val sparkVersion = Option(System.getProperty("sparkVersion")).getOrElse("2.4.5")
 val jacksonVersion = "2.9.9"
 
 lazy val excludeJpountz = ExclusionRule(organization = "net.jpountz.lz4", name = "lz4")
 lazy val excludeNetty = ExclusionRule(organization = "io.netty", name = "netty")
 lazy val excludeNettyAll = ExclusionRule(organization = "io.netty", name = "netty-all")
-lazy val excludeHoodieTimeLineService = ExclusionRule(organization = "com.uber.hoodie", name = "hoodie-timeline-service")
 lazy val excludeAvro = ExclusionRule(organization = "org.apache.avro", name = "avro")
 lazy val excludeSpark = ExclusionRule(organization = "org.apache.spark")
+lazy val excludeFasterXML = ExclusionRule(organization = "com.fasterxml.jackson.module", name= "jackson-module-scala_2.12")
+lazy val excludeMetricsCore = ExclusionRule(organization = "io.dropwizard.metrics", name= "metrics-core")
 
 libraryDependencies ++= Seq(
   "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
@@ -45,7 +46,7 @@ libraryDependencies ++= Seq(
   "io.netty" % "netty" % "3.10.6.Final",
   "com.google.guava" % "guava" % "16.0.1",
   "com.typesafe.play" %% "play-json" % "2.6.2",
-  "com.databricks" %% "spark-redshift" % "3.0.0-preview1",
+  "com.databricks" %% "spark-redshift" % "3.0.0-preview1" excludeAll excludeAvro,
   "com.amazon.redshift" % "redshift-jdbc42" % "1.2.1.1001",
   "com.segment.analytics.java" % "analytics" % "2.0.0",
   "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.6",
@@ -56,15 +57,15 @@ libraryDependencies ++= Seq(
   "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
   "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
   "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion,
-  "com.groupon.dse" % "spark-metrics" % "2.0.0",
+  "com.groupon.dse" % "spark-metrics" % "2.0.0" excludeAll excludeMetricsCore,
   "org.apache.commons" % "commons-text" % "1.6",
   "org.influxdb" % "influxdb-java" % "2.14",
   "org.apache.kafka" %% "kafka" % "2.2.0" % "provided",
   "za.co.absa" % "abris_2.11" % "3.1.1"  % "provided" excludeAll(excludeAvro, excludeSpark),
-  "com.uber.hoodie" % "hoodie-spark" % "0.4.7" % "provided" excludeAll(excludeHoodieTimeLineService),
-  "com.uber.hoodie" % "hoodie-common" % "0.4.7" % "provided" excludeAll(excludeHoodieTimeLineService),
+  "org.apache.hudi" %% "hudi-spark-bundle" % "0.5.1-incubating" % "provided" excludeAll excludeFasterXML,
+  "org.apache.parquet" % "parquet-avro" % "1.10.1" % "provided",
   "org.apache.avro" % "avro" % "1.8.2" % "provided",
-  "org.apache.hive" % "hive-jdbc" % "1.2.2" % "provided" excludeAll(excludeNetty, excludeNettyAll)
+  "org.apache.hive" % "hive-jdbc" % "2.3.3" % "provided" excludeAll(excludeNetty, excludeNettyAll)
 )
 
 // Temporary fix for https://github.com/databricks/spark-redshift/issues/315#issuecomment-285294306

diff --git a/docker/hive/Dockerfile b/docker/hive/Dockerfile
@@ -5,13 +5,15 @@ ENV ATLAS_HOME=/opt/atlas
 RUN mkdir -p $ATLAS_HOME/hook/hive
 ENV HADOOP_HOME=/opt/hadoop
 ENV HADOOP_VERSION=2.7.4
+RUN apt-get update && apt-get install -y ant
+
 RUN wget -q https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz \
     && tar -xzf hadoop-$HADOOP_VERSION.tar.gz \
     && mv hadoop-$HADOOP_VERSION $HADOOP_HOME \
     && rm hadoop-$HADOOP_VERSION.tar.gz
 
 ENV HIVE_HOME=/opt/hive
-ENV HIVE_VERSION=1.2.2
+ENV HIVE_VERSION=2.3.3
 RUN wget -q https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz \
 	&& tar -xzf apache-hive-$HIVE_VERSION-bin.tar.gz \
 	&& mv apache-hive-$HIVE_VERSION-bin $HIVE_HOME \
@@ -21,10 +23,12 @@ ENV MYSQL_CONNECTOR_VERSION=5.1.47
 RUN wget -q https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar \
     && mv mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar $HIVE_HOME/lib
 
-ENV HUDI_VERSION=0.4.5
-RUN apt-get update && apt-get install -y ant 
-RUN wget -q https://repo1.maven.org/maven2/com/uber/hoodie/hoodie-hive-bundle/$HUDI_VERSION/hoodie-hive-bundle-$HUDI_VERSION.jar \
-    && mv hoodie-hive-bundle-$HUDI_VERSION.jar $HIVE_HOME/lib
+ENV HUDI_VERSION=0.5.1-incubating
+RUN wget -q https://repo1.maven.org/maven2/org/apache/hudi/hudi-hive-bundle/$HUDI_VERSION/hudi-hive-bundle-$HUDI_VERSION.jar \
+    && mv hudi-hive-bundle-$HUDI_VERSION.jar $HIVE_HOME/lib
+RUN wget -q https://repo1.maven.org/maven2/org/apache/hudi/hudi-hadoop-mr-bundle/$HUDI_VERSION/hudi-hadoop-mr-bundle-$HUDI_VERSION.jar \
+    && mv hudi-hadoop-mr-bundle-$HUDI_VERSION.jar $HIVE_HOME/lib
+
 RUN wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/$HADOOP_VERSION/hadoop-aws-$HADOOP_VERSION.jar \
     && mv hadoop-aws-$HADOOP_VERSION.jar $HIVE_HOME/lib
 
@@ -86,4 +90,4 @@ RUN mkdir -p $ATLAS_HOME/hook-bin/
 COPY atlas/import_hive.sh $ATLAS_HOME/hook-bin/
 RUN chmod +x /$ATLAS_HOME/hook-bin/import_hive.sh
 
-CMD /wait && /start-hive.sh
+CMD /wait && /start-hive.sh
diff --git a/docker/hive/start-hive.sh b/docker/hive/start-hive.sh
@@ -72,6 +72,10 @@ cat >${HIVE_HOME}/conf/hive-site.xml <<EOL
         <name>fs.s3n.awsSecretAccessKey</name>
         <value>${AWS_SECRET_KEY}</value>
      </property>
+       <property>
+        <name>hive.security.authorization.enabled</name>
+        <value>false</value>
+     </property>
 EOL
 
 if [[ ! -z ${USE_ATLAS} ]] ; then
@@ -109,4 +113,5 @@ fi
 $HIVE_HOME/bin/schematool -dbType ${DB_TYPE} -initSchema
 
 nohup ${HIVE_HOME}/bin/hive --service metastore -p ${METASTORE_PORT} &
+sleep 10s
 ${HIVE_HOME}/bin/hiveserver2 --hiveconf hive.root.logger=INFO,console
diff --git a/docker/hive1/Dockerfile b/docker/hive1/Dockerfile
@@ -0,0 +1,89 @@
+FROM openjdk:8u212-b04-jre-stretch
+
+RUN mkdir /opt/atlas
+ENV ATLAS_HOME=/opt/atlas
+RUN mkdir -p $ATLAS_HOME/hook/hive
+ENV HADOOP_HOME=/opt/hadoop
+ENV HADOOP_VERSION=2.7.4
+RUN wget -q https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz \
+    && tar -xzf hadoop-$HADOOP_VERSION.tar.gz \
+    && mv hadoop-$HADOOP_VERSION $HADOOP_HOME \
+    && rm hadoop-$HADOOP_VERSION.tar.gz
+
+ENV HIVE_HOME=/opt/hive
+ENV HIVE_VERSION=1.2.2
+RUN wget -q https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz \
+	&& tar -xzf apache-hive-$HIVE_VERSION-bin.tar.gz \
+	&& mv apache-hive-$HIVE_VERSION-bin $HIVE_HOME \
+	&& rm apache-hive-$HIVE_VERSION-bin.tar.gz
+
+ENV MYSQL_CONNECTOR_VERSION=5.1.47
+RUN wget -q https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar \
+    && mv mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar $HIVE_HOME/lib
+
+ENV HUDI_VERSION=0.4.7
+RUN apt-get update && apt-get install -y ant
+RUN wget -q https://repo1.maven.org/maven2/com/uber/hoodie/hoodie-hive-bundle/$HUDI_VERSION/hoodie-hive-bundle-$HUDI_VERSION.jar \
+    && mv hoodie-hive-bundle-$HUDI_VERSION.jar $HIVE_HOME/lib
+RUN wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/$HADOOP_VERSION/hadoop-aws-$HADOOP_VERSION.jar \
+    && mv hadoop-aws-$HADOOP_VERSION.jar $HIVE_HOME/lib
+
+ENV AWS_JAVA_SDK_VERSION=1.7.4
+RUN wget -q https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/$AWS_JAVA_SDK_VERSION/aws-java-sdk-$AWS_JAVA_SDK_VERSION.jar \
+    && mv aws-java-sdk-$AWS_JAVA_SDK_VERSION.jar $HIVE_HOME/lib
+
+RUN wget -q https://repo1.maven.org/maven2/net/logstash/log4j/jsonevent-layout/1.7/jsonevent-layout-1.7.jar \
+    && mv jsonevent-layout-1.7.jar $HIVE_HOME/lib
+RUN wget -q https://repo1.maven.org/maven2/net/minidev/json-smart/1.1.1/json-smart-1.1.1.jar \
+    && mv json-smart-1.1.1.jar $HIVE_HOME/lib
+
+# Apache Atlas HiveHook installation
+ENV ATLAS_VERSION=2.0.0
+ENV HBASE_VERSION=2.0.2
+ENV JACKSON_VERSION=2.9.9
+ENV JERSEY_VERSION=1.19
+ENV JSR311_VERSION=1.1
+ENV KAFKA_2_1_1_VERSION=2.0.0
+ENV SCALA_LIBRARY_VERSION=2.11.12
+ENV COMMONS_CONFIG_VERSION=1.10
+
+RUN mkdir -p $ATLAS_HOME/hook/hive/atlas-hive-plugin-impl
+
+RUN wget -P ${ATLAS_HOME}/hook/hive/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-plugin-classloader/$ATLAS_VERSION/atlas-plugin-classloader-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/ https://repo1.maven.org/maven2/org/apache/atlas/hive-bridge-shim/$ATLAS_VERSION/hive-bridge-shim-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-client-common/$ATLAS_VERSION/atlas-client-common-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-client-v1/$ATLAS_VERSION/atlas-client-v1-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-client-v2/$ATLAS_VERSION/atlas-client-v2-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-common/$ATLAS_VERSION/atlas-common-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-intg/$ATLAS_VERSION/atlas-intg-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/atlas-notification/$ATLAS_VERSION/atlas-notification-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/hdfs-model/$ATLAS_VERSION/hdfs-model-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/hive-bridge/$ATLAS_VERSION/hive-bridge-$ATLAS_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/hbase/hbase-common/$HBASE_VERSION/hbase-common-$HBASE_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/hbase/hbase-server/$HBASE_VERSION/hbase-server-$HBASE_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/$JACKSON_VERSION/jackson-annotations-$JACKSON_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/$JACKSON_VERSION/jackson-core-$JACKSON_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/$JACKSON_VERSION/jackson-databind-$JACKSON_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/$JACKSON_VERSION/jackson-module-jaxb-annotations-$JACKSON_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/$JACKSON_VERSION/jackson-jaxrs-base-$JACKSON_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/$JACKSON_VERSION/jackson-jaxrs-json-provider-$JACKSON_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/sun/jersey/jersey-json/$JERSEY_VERSION/jersey-json-$JERSEY_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/com/sun/jersey/contribs/jersey-multipart/$JERSEY_VERSION/jersey-multipart-$JERSEY_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/$KAFKA_2_1_1_VERSION/kafka-clients-$KAFKA_2_1_1_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/kafka/kafka_2.12/$KAFKA_2_1_1_VERSION/kafka_2.12-$KAFKA_2_1_1_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/scala-lang/scala-library/$SCALA_LIBRARY_VERSION/scala-library-$SCALA_LIBRARY_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/commons-configuration/commons-configuration/$COMMONS_CONFIG_VERSION/commons-configuration-$COMMONS_CONFIG_VERSION.jar
+RUN wget -P ${ATLAS_HOME}/hook/hive/atlas-hive-plugin-impl/ https://repo1.maven.org/maven2/org/apache/atlas/hdfs-model/$ATLAS_VERSION/hdfs-model-$ATLAS_VERSION.jar
+
+
+ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.5.0/wait /wait
+RUN chmod +x /wait
+
+COPY start-hive.sh /
+COPY log4j.json.properties .
+
+RUN mkdir -p $ATLAS_HOME/hook-bin/
+COPY atlas/import_hive.sh $ATLAS_HOME/hook-bin/
+RUN chmod +x /$ATLAS_HOME/hook-bin/import_hive.sh
+
+CMD /wait && /start-hive.sh