[Java] Upgrade substrait for antlr4 compatibility. reduce binary size (…

…#523) Signed-off-by: chenxu <[email protected]> Co-authored-by: chenxu <[email protected]>
lakesoul-io · Aug 7, 2024 · 3670bf3 · 3670bf3
1 parent 86cd629
commit 3670bf3
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 12 deletions.
diff --git a/native-io/lakesoul-io-java/pom.xml b/native-io/lakesoul-io-java/pom.xml
@@ -27,7 +27,7 @@ SPDX-License-Identifier: Apache-2.0
         <maven.compiler.target>8</maven.compiler.target>
         <arrow.version>15.0.2</arrow.version>
         <scalatest.version>3.1.0</scalatest.version>
-        <substrait.version>0.28.0</substrait.version>
+        <substrait.version>0.30.0</substrait.version>
         <protobuf.version>3.22.0</protobuf.version>
         <protobuf.plugin.version>0.6.1</protobuf.plugin.version>
     </properties>
@@ -630,6 +630,9 @@ SPDX-License-Identifier: Apache-2.0
                         <includes>
                             <include>**</include>
                         </includes>
+                        <excludes>
+                            <exclude>org.apache.spark:*</exclude>
+                        </excludes>
                     </artifactSet>
                     <filters>
                         <filter>
@@ -646,6 +649,7 @@ SPDX-License-Identifier: Apache-2.0
                             <excludes>
                                 <exclude>org/sparkproject/jetty/**</exclude>
                                 <exclude>META-INF/maven/org.eclipse.jetty/**</exclude>
+                                <exclude>META-INF/maven/org.spark-project.spark/**</exclude>
                                 <exclude>jetty-dir.css</exclude>
                                 <exclude>META-INF/services/org.eclipse.jetty.http.HttpFieldPreEncoder</exclude>
                             </excludes>
@@ -744,10 +748,6 @@ SPDX-License-Identifier: Apache-2.0
                             <pattern>com.google.flatbuffers</pattern>
                             <shadedPattern>com.lakesoul.shaded.com.google.flatbuffers</shadedPattern>
                         </relocation>
-                        <relocation>
-                            <pattern>org.apache.spark</pattern>
-                            <shadedPattern>com.lakesoul.shaded.org.apache.spark</shadedPattern>
-                        </relocation>
                         <relocation>
                             <pattern>io.substrait</pattern>
                             <shadedPattern>com.lakesoul.shaded.io.substrait</shadedPattern>

diff --git a/native-io/lakesoul-io-java/src/main/scala/com/dmetasoul/lakesoul/LakeSoulArrowReader.scala b/native-io/lakesoul-io-java/src/main/scala/com/dmetasoul/lakesoul/LakeSoulArrowReader.scala
@@ -5,14 +5,9 @@
 package com.dmetasoul.lakesoul
 
 import com.dmetasoul.lakesoul.lakesoul.io.NativeIOReader
-import org.apache.arrow.c.{ArrowArray, ArrowSchema, CDataDictionaryProvider, Data}
+import org.apache.arrow.c.{ArrowArray, CDataDictionaryProvider, Data}
 import org.apache.arrow.vector.VectorSchemaRoot
 
-import java.io.IOException
-import scala.concurrent.duration.DurationInt
-import scala.concurrent.{Await, Future, Promise}
-import scala.util.Success
-
 case class LakeSoulArrowReader(reader: NativeIOReader,
                                timeout: Int = 10000) extends AutoCloseable {
 

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -14,6 +14,9 @@ members = [
 resolver = "2"
 
 [profile.release]
+strip = true
+opt-level = "z"
+lto = true
 
 [workspace.dependencies]
 datafusion = { git = "https://github.com/lakesoul-io/arrow-datafusion.git", branch = "datafusion-33-parquet-prefetch" }
@@ -47,4 +50,4 @@ bytes = "1.4.0"
 half = "^2.1"
 tracing = "0.1.40"
 thiserror = "1.0"
-anyhow = "1.0.79"
+anyhow = "1.0.79"