fix hadoop provided dependencies

Signed-off-by: chenxu <[email protected]>
lakesoul-io · Aug 5, 2024 · 005d4e0 · 005d4e0
1 parent 70a9eef
commit 005d4e0
Show file tree

Hide file tree

Showing 6 changed files with 9 additions and 6 deletions.
diff --git a/.github/workflows/presto-cdc-test.yml b/.github/workflows/presto-cdc-test.yml
@@ -100,7 +100,7 @@ jobs:
           echo "FLINK_TEST_JAR_NAME=$(python script/get_jar_name.py lakesoul-flink | sed -e 's/.jar/-tests.jar/g')" >> $GITHUB_ENV
           echo "SPARK_JAR_NAME=$(python script/get_jar_name.py lakesoul-spark)" >> $GITHUB_ENV
           echo "SPARK_TEST_JAR_NAME=$(python script/get_jar_name.py lakesoul-spark | sed -e 's/.jar/-tests.jar/g')" >> $GITHUB_ENV
-          echo "PRESTO_JAR_NAME=$(python script/get_jar_name.py lakesoul-presto | sed -e 's/.jar/-jar-with-dependencies.jar/g')" >> $GITHUB_ENV
+          echo "PRESTO_JAR_NAME=$(python script/get_jar_name.py lakesoul-presto)" >> $GITHUB_ENV
           echo "PRESTO_TEST_JAR_NAME=$(python script/get_jar_name.py lakesoul-presto | sed -e 's/.jar/-tests.jar/g')" >> $GITHUB_ENV
       - name: Copy built jar to work-dir
         run: |

diff --git a/lakesoul-common/pom.xml b/lakesoul-common/pom.xml
@@ -332,7 +332,6 @@ SPDX-License-Identifier: Apache-2.0
             <version>3.22.0</version>
         </dependency>
 
-
         <!-- jnr-ffi deps-->
         <dependency>
             <groupId>com.github.jnr</groupId>
@@ -342,7 +341,7 @@ SPDX-License-Identifier: Apache-2.0
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client-api</artifactId>
-            <version>3.3.6</version>
+            <version>3.4.0</version>
             <scope>${local.scope}</scope>
         </dependency>
     </dependencies>

diff --git a/lakesoul-flink/pom.xml b/lakesoul-flink/pom.xml
@@ -347,6 +347,12 @@ SPDX-License-Identifier: Apache-2.0
             <version>${flink.version}</version>
             <scope>${local.scope}</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client-api</artifactId>
+            <version>3.4.0</version>
+            <scope>${local.scope}</scope>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/lakesoul-presto/pom.xml b/lakesoul-presto/pom.xml
@@ -97,7 +97,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client-api</artifactId>
-            <version>3.3.2</version>
+            <version>3.4.0</version>
             <scope>${local.scope}</scope>
         </dependency>
     </dependencies>

diff --git a/lakesoul-spark/src/test/scala/org/apache/spark/sql/lakesoul/TableCreationTests.scala b/lakesoul-spark/src/test/scala/org/apache/spark/sql/lakesoul/TableCreationTests.scala
@@ -685,7 +685,6 @@ trait TableCreationTests
 
   test("CTAS a managed table with the existing empty directory") {
     val tableLoc = new File(getDefaultTablePath("tab1").stripPrefix("file:"))
-    println(tableLoc.getAbsolutePath)
     try {
       tableLoc.mkdirs()
       withTable("tab1") {

diff --git a/lakesoul-spark/src/test/scala/org/apache/spark/sql/lakesoul/commands/CompactionSuite.scala b/lakesoul-spark/src/test/scala/org/apache/spark/sql/lakesoul/commands/CompactionSuite.scala
@@ -27,7 +27,6 @@ class CompactionSuite extends QueryTest
 
   override def sparkConf: SparkConf = {
     super.sparkConf
-      .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager")
       .set("spark.network.timeout", "10000000")
       .set("spark.sql.catalog.lakesoul", classOf[LakeSoulCatalog].getName)
       .set(SQLConf.DEFAULT_CATALOG.key, LakeSoulCatalog.CATALOG_NAME)