fix after rebase main

Signed-off-by: zenghua <[email protected]>

.github/workflows/maven-test.yml

@@ -222,7 +222,7 @@ jobs:
 
   spark-test-rbac:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -265,6 +265,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -355,7 +359,7 @@ jobs:
 
   flink-test-rbac:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -406,6 +410,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test

.github/workflows/rust-clippy.yml

@@ -33,6 +33,10 @@ jobs:
           toolchain: nightly-2023-05-20
           components: clippy
           default: true
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./native-io -> target"

lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBManager.java

@@ -112,7 +112,6 @@ public void createNewTable(String tableId, String namespace, String tableName, S
 
         String domain = getNameSpaceDomain(namespace);
 
-        System.out.println("[debug]createNewTable " + tableInfo);
         if (StringUtils.isNotBlank(tableName)) {
             tableNameIdDao.insert(TableNameIdDao.newTableNameId(tableName, tableId, namespace, domain));
         }
@@ -706,7 +705,7 @@ private PartitionInfo getOrCreateCurPartitionInfo(Map<String, PartitionInfo> cur
                     .setVersion(-1)
                     .setDomain(getTableDomain(tableId))
                     .build();
-        }else{
+        } else {
             curPartitionInfo = curPartitionInfo.toBuilder()
                     .setDomain(getTableDomain(tableId))
                     .build();
@@ -773,23 +772,23 @@ public void rollbackPartitionByVersion(String tableId, String partitionDesc, int
                         .build());
     }
 
-    private String getTableDomain(String tableId){
-        if(!AuthZEnforcer.authZEnabled()){
+    private String getTableDomain(String tableId) {
+        if (!AuthZEnforcer.authZEnabled()) {
             return "public";
         }
         TableInfo tableInfo = this.getTableInfoByTableId(tableId);
-        if(tableInfo == null){
+        if (tableInfo == null) {
             throw new IllegalStateException("target tableinfo does not exists");
         }
         return getNameSpaceDomain(tableInfo.getTableNamespace());
     }
 
-    private String getNameSpaceDomain(String namespace){
-        if(!AuthZEnforcer.authZEnabled()){
+    private String getNameSpaceDomain(String namespace) {
+        if (!AuthZEnforcer.authZEnabled()) {
             return "public";
         }
         Namespace namespaceInfo = getNamespaceByNamespace(namespace);
-        if(namespaceInfo == null) {
+        if (namespaceInfo == null) {
             throw new IllegalStateException("target namespace does not exists");
         }
         return namespaceInfo.getDomain();
@@ -875,10 +874,13 @@ public void deleteNamespace(String namespace) {
     public void cleanMeta() {
         if (NativeUtils.NATIVE_METADATA_UPDATE_ENABLED) {
             NativeMetadataJavaClient.cleanMeta();
+            if (!AuthZEnforcer.authZEnabled()) {
+                namespaceDao.insert(NamespaceDao.DEFAULT_NAMESPACE);
+            }
             return;
         }
         namespaceDao.clean();
-        if(!AuthZEnforcer.authZEnabled()){
+        if (!AuthZEnforcer.authZEnabled()) {
             namespaceDao.insert(NamespaceDao.DEFAULT_NAMESPACE);
         }
         dataCommitInfoDao.clean();

lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBUtil.java

@@ -36,8 +36,8 @@ public class DBUtil {
 
     private static final String driverNameKey = "lakesoul.pg.driver";
     private static final String urlKey = "lakesoul.pg.url";
-    private static final String usernameKey = "lakesoul.pg.username";
-    private static final String passwordKey = "lakesoul.pg.password";
+    public static final String usernameKey = "lakesoul.pg.username";
+    public static final String passwordKey = "lakesoul.pg.password";
     private static final String hostKey = "lakesoul.pg.host";
     private static final String portKey = "lakesoul.pg.port";
     private static final String dbNameKey = "lakesoul.pg.dbName";

lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/dao/NamespaceDao.java

@@ -10,6 +10,9 @@
 import com.dmetasoul.lakesoul.meta.entity.Namespace;
 import com.dmetasoul.lakesoul.meta.jnr.NativeMetadataJavaClient;
 import com.dmetasoul.lakesoul.meta.jnr.NativeUtils;
+import com.dmetasoul.lakesoul.meta.rbac.AuthZContext;
+import com.dmetasoul.lakesoul.meta.rbac.AuthZEnforcer;
+import dev.failsafe.internal.util.Lists;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -167,10 +170,11 @@ public void clean() {
     }
 
     public static Namespace namespaceFromResultSet(ResultSet rs) throws SQLException {
+        String comment = rs.getString("comment");
         return Namespace.newBuilder()
                 .setNamespace(rs.getString("namespace"))
                 .setProperties(rs.getString("properties"))
-                .setComment(rs.getString("comment"))
+                .setComment(comment == null ? "" : comment)
                 .setDomain(rs.getString("domain"))
                 .build();
     }
@@ -180,6 +184,7 @@ public static Namespace namespaceFromResultSet(ResultSet rs) throws SQLException
                     .setNamespace(DBConfig.LAKESOUL_DEFAULT_NAMESPACE)
                     .setProperties("{}")
                     .setComment("")
+                    .setDomain(AuthZContext.getInstance().getDomain())
                     .build();
 
 }

lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/NativeMetadataJavaClient.java

@@ -85,6 +85,7 @@ public static NativeMetadataJavaClient getInstance() {
         return instance;
     }
 
+
     public Pointer getTokioPostgresClient() {
         return tokioPostgresClient;
     }
@@ -431,7 +432,7 @@ public static int cleanMeta() {
     }
 
     @Override
-    public void close() throws Exception {
+    public void close() {
         if (tokioRuntime != null) {
             libLakeSoulMetaData.free_tokio_runtime(tokioRuntime);
             tokioRuntime = null;
@@ -445,4 +446,11 @@ public void close() throws Exception {
             preparedStatement = null;
         }
     }
+
+    public static void closeAll() {
+        if (instance != null) {
+            instance.close();
+            instance = null;
+        }
+    }
 }

lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/jnr/NativeUtils.java

@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2023 LakeSoul Contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package com.dmetasoul.lakesoul.meta.jnr;
 
 public class NativeUtils {

lakesoul-flink/src/test/java/com/dmetasoul/lakesoul/meta/LakeSoulRBACTest.java

@@ -4,6 +4,7 @@
 
 package com.dmetasoul.lakesoul.meta;
 
+import com.dmetasoul.lakesoul.meta.jnr.NativeMetadataJavaClient;
 import org.apache.flink.lakesoul.test.LakeSoulFlinkTestBase;
 import org.apache.flink.table.catalog.exceptions.CatalogException;
 import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
@@ -14,7 +15,7 @@
 import java.util.List;
 
 public class LakeSoulRBACTest extends LakeSoulFlinkTestBase {
-    final String ADMIN1  = "admin1";
+    final String ADMIN1 = "admin1";
     final String ADMIN1_PASS = "admin1";
     final String ADMIN2 = "admin2";
     final String ADMIN2_PASS = "admin2";
@@ -23,21 +24,22 @@ public class LakeSoulRBACTest extends LakeSoulFlinkTestBase {
     final String DOMAIN1 = "domain1";
     final String DOMAIN2 = "domain2";
 
-    private void login(String username , String password, String domain)  {
+    private void login(String username, String password, String domain) {
         System.setProperty(DBUtil.usernameKey, username);
         System.setProperty(DBUtil.passwordKey, password);
         System.setProperty(DBUtil.domainKey, domain);
         DBConnector.closeAllConnections();
+        NativeMetadataJavaClient.closeAll();
     }
 
     @Test
-    public void testDifferentDomain(){
+    public void testDifferentDomain() {
         getTableEnv().useCatalog("lakesoul");
         login(ADMIN1, ADMIN1_PASS, DOMAIN1);
 
         // create
         sql("create database if not exists database1");
-        assert(sql("show databases").size() == 2);
+        assert (sql("show databases").size() == 2);
         // drop: coming soon
 //    spark.sql("drop database database1").collect()
 //    val df2 = spark.sql("show databases").toDF()
@@ -53,12 +55,12 @@ public void testDifferentDomain(){
                 + " with ('format' = 'lakesoul', 'path' = '"
                 + getTempDirUri("/lakeSource/table2")
                 + "')");
-        assert( sql("show tables").size() == 2);
+        assert (sql("show tables").size() == 2);
 
         // drop table
         sql("drop table table1");
         sql("drop table table2");
-        assert(sql("show tables").size() == 0);
+        assert (sql("show tables").size() == 0);
 
         // write and read data
         sql("create table if not exists table1 ( id int, foo string, bar string )"
@@ -67,21 +69,21 @@ public void testDifferentDomain(){
                 + "')");
         sql("insert into table1 values(1, 'foo1', 'bar1')");
         sql("insert into table1 values(2, 'foo2', 'bar2')");
-        assert(sql("select * from table1").size() == 2);
+        assert (sql("select * from table1").size() == 2);
 
         // create & drop database
         sql("insert into table1 values(3, 'foo3', 'bar3')");
         login(ADMIN2, ADMIN2_PASS, DOMAIN1);
         try {
             sql("use database1");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e){
-            assert(e instanceof CatalogException);
+        } catch (Exception e) {
+            assert (e instanceof CatalogException);
         }
-        try{
+        try {
             sql("create database if not exists database2");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e) {
+        } catch (Exception e) {
             System.out.println(e.getMessage());
             assert e.getMessage().contains("Could not execute CREATE DATABASE");
         }
@@ -96,14 +98,14 @@ public void testDifferentDomain(){
                     + getTempDirUri("/lakeSource/table3")
                     + "')");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e){
+        } catch (Exception e) {
             System.out.println(e.getMessage());
             assert e.getCause() instanceof DatabaseNotExistException;
         }
         try {
             sql("drop table database1.table1");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e){
+        } catch (Exception e) {
             System.out.println(e.getMessage());
             assert e.getMessage().contains("Table with identifier 'lakesoul.database1.table1' does not exist.");
         }
@@ -112,17 +114,17 @@ public void testDifferentDomain(){
         try {
             sql("insert into database1.table1 values(4, 'foo4', 'bar4')");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e){
+        } catch (Exception e) {
             System.out.println(e.getMessage());
-            assert(e.getMessage().contains("Cannot find table '`lakesoul`.`database1`.`table1`' in any of the catalogs"));
+            assert (e.getMessage().contains("Cannot find table '`lakesoul`.`database1`.`table1`' in any of the catalogs"));
         }
 
         try {
             sql("select * from database1.table1");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e){
+        } catch (Exception e) {
             System.out.println(e.getMessage());
-            assert(e.getMessage().contains("Object 'database1' not found within 'lakesoul'"));
+            assert (e.getMessage().contains("Object 'database1' not found within 'lakesoul'"));
         }
 
         // clear test
@@ -132,7 +134,7 @@ public void testDifferentDomain(){
     }
 
     @Test
-    public void testDifferentRole(){
+    public void testDifferentRole() {
         getTableEnv().useCatalog("lakesoul");
         login(ADMIN1, ADMIN1_PASS, DOMAIN1);
         // create
@@ -142,17 +144,17 @@ public void testDifferentRole(){
         login(USER1, USER1_PASS, DOMAIN1);
         // create table & drop database
         sql("use database1");
-        try{
+        try {
             sql("create database if not exists database3");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e) {
+        } catch (Exception e) {
             System.out.println(e.getMessage());
             assert e.getMessage().contains("Could not execute CREATE DATABASE");
         }
-        try{
+        try {
             sql("drop database database1");
             throw new RuntimeException("test state was unexcepted");
-        }catch (Exception e) {
+        } catch (Exception e) {
             System.out.println(e.getMessage());
             assert e.getMessage().contains("Could not execute DROP DATABASE lakesoul.database1 RESTRICT");
         }
@@ -166,10 +168,10 @@ public void testDifferentRole(){
                 + " with ('format' = 'lakesoul', 'path' = '"
                 + getTempDirUri("/lakeSource/table2")
                 + "')");
-        assert(sql("show tables").size() == 2);
+        assert (sql("show tables").size() == 2);
         sql("drop table table1");
         sql("drop table table2");
-        assert(sql("show tables").size() == 0);
+        assert (sql("show tables").size() == 0);
 
         // CRUD data
         sql("create table if not exists table1 ( id int, foo string, bar string )"
@@ -178,7 +180,7 @@ public void testDifferentRole(){
                 + "')");
         sql("insert into table1 values(1, 'foo1', 'bar1')");
         sql("insert into table1 values(2, 'foo2', 'bar2')");
-        assert(sql("select * from table1").size() == 2);
+        assert (sql("select * from table1").size() == 2);
 
         // clear test
         sql("drop table table1");