fix after rebase main

Signed-off-by: zenghua <[email protected]>
lakesoul-io · Aug 8, 2023 · d6dfcbc · d6dfcbc
1 parent 7316a5c
commit d6dfcbc
Show file tree

Hide file tree

Showing 22 changed files with 154 additions and 136 deletions.
diff --git a/.github/workflows/flink-cdc-test.yml b/.github/workflows/flink-cdc-test.yml
@@ -61,6 +61,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./native-io -> target"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./native-metadata -> target"
       - name: Pull images
         run: |
           docker pull -q bitnami/spark:3.3.1
@@ -69,10 +72,17 @@ jobs:
           use-cross: true
           command: build
           args: '--manifest-path native-io/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: '--manifest-path native-metadata/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
       - name: Build with Maven
         run: |
           mkdir -p native-io/target/release
           cp native-io/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so native-io/target/release
+          mkdir -p native-metadata/target/release
+          cp native-metadata/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so native-metadata/target/release
           MAVEN_OPTS="-Xmx4000m" mvn -q -B package -f pom.xml -Pcross-build -DskipTests
       - name: Get jar names
         run: |

diff --git a/.github/workflows/maven-test.yml b/.github/workflows/maven-test.yml
@@ -41,16 +41,12 @@ jobs:
           profile: minimal
           toolchain: nightly-2023-05-20
           default: true
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v2
-        with:
-          version: "23.x"
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./native-metadata -> target"
       - uses: actions-rs/cargo@v1
         with:
-          #          use-cross: true
+          use-cross: true
           command: build
           args: '--manifest-path native-metadata/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
       - uses: actions/upload-artifact@master
@@ -222,7 +218,7 @@ jobs:
 
   spark-test-rbac:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -265,6 +261,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -355,7 +355,7 @@ jobs:
 
   flink-test-rbac:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -406,6 +406,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test

diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml
@@ -117,10 +117,6 @@ jobs:
           java-version: '8'
           distribution: 'temurin'
           cache: maven
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v2
-        with:
-          version: "23.x"
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -131,7 +127,7 @@ jobs:
           workspaces: "./native-metadata -> target"
       - uses: actions-rs/cargo@v1
         with:
-          #          use-cross: true
+          use-cross: true
           command: build
           args: '--manifest-path native-metadata/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
       - uses: actions/upload-artifact@master

diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml
@@ -33,6 +33,10 @@ jobs:
           toolchain: nightly-2023-05-20
           components: clippy
           default: true
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./native-io -> target"

diff --git a/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBManager.java b/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBManager.java
@@ -112,7 +112,6 @@ public void createNewTable(String tableId, String namespace, String tableName, S
 
         String domain = getNameSpaceDomain(namespace);
 
-        System.out.println("[debug]createNewTable " + tableInfo);
         if (StringUtils.isNotBlank(tableName)) {
             tableNameIdDao.insert(TableNameIdDao.newTableNameId(tableName, tableId, namespace, domain));
         }
@@ -706,7 +705,7 @@ private PartitionInfo getOrCreateCurPartitionInfo(Map<String, PartitionInfo> cur
                     .setVersion(-1)
                     .setDomain(getTableDomain(tableId))
                     .build();
-        }else{
+        } else {
             curPartitionInfo = curPartitionInfo.toBuilder()
                     .setDomain(getTableDomain(tableId))
                     .build();
@@ -773,23 +772,23 @@ public void rollbackPartitionByVersion(String tableId, String partitionDesc, int
                         .build());
     }
 
-    private String getTableDomain(String tableId){
-        if(!AuthZEnforcer.authZEnabled()){
+    private String getTableDomain(String tableId) {
+        if (!AuthZEnforcer.authZEnabled()) {
             return "public";
         }
         TableInfo tableInfo = this.getTableInfoByTableId(tableId);
-        if(tableInfo == null){
+        if (tableInfo == null) {
             throw new IllegalStateException("target tableinfo does not exists");
         }
         return getNameSpaceDomain(tableInfo.getTableNamespace());
     }
 
-    private String getNameSpaceDomain(String namespace){
-        if(!AuthZEnforcer.authZEnabled()){
+    private String getNameSpaceDomain(String namespace) {
+        if (!AuthZEnforcer.authZEnabled()) {
             return "public";
         }
         Namespace namespaceInfo = getNamespaceByNamespace(namespace);
-        if(namespaceInfo == null) {
+        if (namespaceInfo == null) {
             throw new IllegalStateException("target namespace does not exists");
         }
         return namespaceInfo.getDomain();
@@ -875,10 +874,13 @@ public void deleteNamespace(String namespace) {
     public void cleanMeta() {
         if (NativeUtils.NATIVE_METADATA_UPDATE_ENABLED) {
             NativeMetadataJavaClient.cleanMeta();
+            if (!AuthZEnforcer.authZEnabled()) {
+                namespaceDao.insert(NamespaceDao.DEFAULT_NAMESPACE);
+            }
             return;
         }
         namespaceDao.clean();
-        if(!AuthZEnforcer.authZEnabled()){
+        if (!AuthZEnforcer.authZEnabled()) {
             namespaceDao.insert(NamespaceDao.DEFAULT_NAMESPACE);
         }
         dataCommitInfoDao.clean();

diff --git a/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBUtil.java b/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBUtil.java
@@ -14,6 +14,8 @@
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.sql.Connection;
@@ -30,17 +32,11 @@ public class DBUtil {
     private static final String urlDefault = "jdbc:postgresql://127.0.0.1:5432/lakesoul_test?stringtype=unspecified";
     private static final String usernameDefault = "lakesoul_test";
     private static final String passwordDefault = "lakesoul_test";
-    private static final String hostDefault = "127.0.0.1";
-    private static final String portDefault = "5432";
-    private static final String dbNameDefault = "lakesoul_test";
 
     private static final String driverNameKey = "lakesoul.pg.driver";
     private static final String urlKey = "lakesoul.pg.url";
-    private static final String usernameKey = "lakesoul.pg.username";
-    private static final String passwordKey = "lakesoul.pg.password";
-    private static final String hostKey = "lakesoul.pg.host";
-    private static final String portKey = "lakesoul.pg.port";
-    private static final String dbNameKey = "lakesoul.pg.dbName";
+    public static final String usernameKey = "lakesoul.pg.username";
+    public static final String passwordKey = "lakesoul.pg.password";
 
     private static final String driverNameEnv = "LAKESOUL_PG_DRIVER";
     private static final String urlEnv = "LAKESOUL_PG_URL";
@@ -96,16 +92,20 @@ public static DataBaseProperty getDBInfo() {
             properties.setProperty(urlKey, getConfigValue(urlEnv, urlKey, urlDefault));
             properties.setProperty(usernameKey, getConfigValue(usernameEnv, usernameKey, usernameDefault));
             properties.setProperty(passwordKey, getConfigValue(passwordEnv, passwordKey, passwordDefault));
-            properties.setProperty(dbNameKey, getConfigValue(dbNameKey, dbNameKey, dbNameDefault));
         }
         DataBaseProperty dataBaseProperty = new DataBaseProperty();
         dataBaseProperty.setDriver(properties.getProperty(driverNameKey, driverNameDefault));
         dataBaseProperty.setUrl(properties.getProperty(urlKey, urlDefault));
         dataBaseProperty.setUsername(properties.getProperty(usernameKey, usernameDefault));
         dataBaseProperty.setPassword(properties.getProperty(passwordKey, passwordDefault));
-        dataBaseProperty.setDbName(properties.getProperty(dbNameKey, dbNameDefault));
-        dataBaseProperty.setHost(properties.getProperty(hostKey, hostDefault));
-        dataBaseProperty.setPort(properties.getProperty(portKey, portDefault));
+        try {
+            URL url = new URL(properties.getProperty(urlKey, urlDefault).replaceFirst("jdbc:postgresql", "http"));
+            dataBaseProperty.setDbName(url.getPath().substring(1));
+            dataBaseProperty.setHost(url.getHost());
+            dataBaseProperty.setPort(String.valueOf(url.getPort()));
+        } catch (MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
         return dataBaseProperty;
     }
 

diff --git a/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/dao/NamespaceDao.java b/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/dao/NamespaceDao.java
@@ -10,6 +10,9 @@
 import com.dmetasoul.lakesoul.meta.entity.Namespace;
 import com.dmetasoul.lakesoul.meta.jnr.NativeMetadataJavaClient;
 import com.dmetasoul.lakesoul.meta.jnr.NativeUtils;
+import com.dmetasoul.lakesoul.meta.rbac.AuthZContext;
+import com.dmetasoul.lakesoul.meta.rbac.AuthZEnforcer;
+import dev.failsafe.internal.util.Lists;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -167,10 +170,11 @@ public void clean() {
     }
 
     public static Namespace namespaceFromResultSet(ResultSet rs) throws SQLException {
+        String comment = rs.getString("comment");
         return Namespace.newBuilder()
                 .setNamespace(rs.getString("namespace"))
                 .setProperties(rs.getString("properties"))
-                .setComment(rs.getString("comment"))
+                .setComment(comment == null ? "" : comment)
                 .setDomain(rs.getString("domain"))
                 .build();
     }
@@ -180,6 +184,7 @@ public static Namespace namespaceFromResultSet(ResultSet rs) throws SQLException
                     .setNamespace(DBConfig.LAKESOUL_DEFAULT_NAMESPACE)
                     .setProperties("{}")
                     .setComment("")
+                    .setDomain(AuthZContext.getInstance().getDomain())
                     .build();
 
 }
diff --git a/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/JnrLoader.java b/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/JnrLoader.java
@@ -25,7 +25,6 @@ public class JnrLoader {
 
     public static LibLakeSoulMetaData get() {
         JnrLoader.tryLoad();
-        System.out.println(INSTANCE.libLakeSoulMetaData);
         return INSTANCE.libLakeSoulMetaData;
     }
 
@@ -59,7 +58,6 @@ public synchronized static void tryLoad() {
             libraryOptions.put(LibraryOption.LoadNow, true);
             libraryOptions.put(LibraryOption.IgnoreError, true);
 
-            System.out.println(finalPath);
             JnrLoader.INSTANCE.libLakeSoulMetaData = LibraryLoader.loadLibrary(
                     LibLakeSoulMetaData.class,
                     libraryOptions,

diff --git a/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/NativeMetadataJavaClient.java b/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/NativeMetadataJavaClient.java
@@ -60,6 +60,12 @@ public class NativeMetadataJavaClient implements AutoCloseable {
 
     private final ReentrantReadWriteLock lock;
 
+    private static DataBaseProperty dataBaseProperty = null;
+
+    public static void setDataBaseProperty(DataBaseProperty dataBaseProperty) {
+        NativeMetadataJavaClient.dataBaseProperty = dataBaseProperty;
+    }
+
     public NativeMetadataJavaClient() {
         this(5000L, 1 << 12, 1 << 16);
     }
@@ -85,6 +91,7 @@ public static NativeMetadataJavaClient getInstance() {
         return instance;
     }
 
+
     public Pointer getTokioPostgresClient() {
         return tokioPostgresClient;
     }
@@ -183,9 +190,11 @@ public void close() {
     }
 
     private void initialize() {
-        DataBaseProperty dataBaseProperty = DBUtil.getDBInfo();
+        DataBaseProperty dataBaseProperty = NativeMetadataJavaClient.dataBaseProperty;
+        if (dataBaseProperty == null) {
+            dataBaseProperty = DBUtil.getDBInfo();
+        }
         tokioRuntime = libLakeSoulMetaData.create_tokio_runtime();
-        System.out.println("create_tokio_runtime success");
 
         String config = String.format(
                 "host=%s port=%s dbname=%s user=%s password=%s",
@@ -200,14 +209,13 @@ private void initialize() {
                     if (msg.isEmpty()) {
                         future.complete(bool);
                     } else {
-                        System.out.println(msg);
+                        System.err.println(msg);
                         future.completeExceptionally(new IOException(msg));
                     }
                 }, getbooleanCallbackObjectReferenceManager()),
                 config,
                 tokioRuntime
         );
-        System.out.println("create_tokio_postgres_client success");
         preparedStatement = libLakeSoulMetaData.create_prepared_statement();
         try {
             future.get(timeout, TimeUnit.MILLISECONDS);
@@ -431,7 +439,7 @@ public static int cleanMeta() {
     }
 
     @Override
-    public void close() throws Exception {
+    public void close() {
         if (tokioRuntime != null) {
             libLakeSoulMetaData.free_tokio_runtime(tokioRuntime);
             tokioRuntime = null;
@@ -445,4 +453,11 @@ public void close() throws Exception {
             preparedStatement = null;
         }
     }
+
+    public static void closeAll() {
+        if (instance != null) {
+            instance.close();
+            instance = null;
+        }
+    }
 }
diff --git a/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/NativeUtils.java b/lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/NativeUtils.java
@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2023 LakeSoul Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package com.dmetasoul.lakesoul.meta.jnr;
 
 public class NativeUtils {