[Native-Metadata] Rust implementation of DAO layer

.github/workflows/flink-cdc-test.yml

@@ -61,6 +61,9 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./native-io -> target"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./native-metadata -> target"
       - name: Pull images
         run: |
           docker pull -q bitnami/spark:3.3.1
@@ -69,10 +72,17 @@ jobs:
           use-cross: true
           command: build
           args: '--manifest-path native-io/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: '--manifest-path native-metadata/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
       - name: Build with Maven
         run: |
           mkdir -p native-io/target/release
           cp native-io/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so native-io/target/release
+          mkdir -p native-metadata/target/release
+          cp native-metadata/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so native-metadata/target/release
           MAVEN_OPTS="-Xmx4000m" mvn -q -B package -f pom.xml -Pcross-build -DskipTests
       - name: Get jar names
         run: |

.github/workflows/maven-test.yml

@@ -26,7 +26,35 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-linux-x86_64:
+  build-native-metadata-linux-x86_64:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2023-05-20
+          default: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./native-metadata -> target"
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: '--manifest-path native-metadata/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so
+
+  build-native-io-linux-x86_64:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -56,7 +84,7 @@ jobs:
 
   spark-test-1:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -96,6 +124,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -119,7 +151,7 @@ jobs:
 
   spark-test-2:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -159,6 +191,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -182,7 +218,7 @@ jobs:
 
   spark-test-rbac:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -225,6 +261,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -248,7 +288,7 @@ jobs:
 
   flink-test-1:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -288,6 +328,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -311,7 +355,7 @@ jobs:
 
   flink-test-rbac:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-metadata-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -362,6 +406,10 @@ jobs:
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-metadata/target/release/
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test

.github/workflows/native-build.yml

@@ -23,7 +23,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-linux-x86_64:
+  build-native-io-linux-x86_64:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -51,7 +51,7 @@ jobs:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu
           path: ./native-io/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so
 
-  build-windows-x86_64:
+  build-native-io-windows-x86_64:
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
@@ -78,7 +78,7 @@ jobs:
           name: lakesoul-nativeio-x86_64-pc-windows-msvc
           path: ./native-io/target/release/lakesoul_io_c.dll
 
-  build-macos-x86_64:
+  build-native-io-macos-x86_64:
     runs-on: macos-latest
     steps:
       - name: Install automake
@@ -107,9 +107,101 @@ jobs:
           name: lakesoul-nativeio-x86_64-apple-darwin
           path: ./native-io/target/release/liblakesoul_io_c.dylib
 
+  build-native-metadata-linux-x86_64:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2023-05-20
+          default: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./native-metadata -> target"
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: '--manifest-path native-metadata/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu
+          path: ./native-metadata/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so
+
+  build-native-metadata-windows-x86_64:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2023-05-20
+          default: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./native-metadata -> target"
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: '--manifest-path native-metadata/Cargo.toml --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativemetadata-x86_64-pc-windows-msvc
+          path: ./native-metadata/target/release/lakesoul_metadata_c.dll
+
+  build-native-metadata-macos-x86_64:
+    runs-on: macos-latest
+    steps:
+      - name: Install automake
+        run: brew install automake
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2023-05-20
+          default: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./native-metadata -> target"
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: '--manifest-path native-metadata/Cargo.toml --release --all-features'
+      - uses: actions/upload-artifact@master
+        with:
+          name: lakesoul-nativemetadata-x86_64-apple-darwin
+          path: ./native-metadata/target/release/liblakesoul_metadata_c.dylib
+
   build-maven-package:
     runs-on: ubuntu-latest
-    needs: [ build-linux-x86_64, build-windows-x86_64, build-macos-x86_64 ]
+    needs: [ build-native-io-linux-x86_64, build-native-io-windows-x86_64, build-native-io-macos-x86_64, build-native-metadata-linux-x86_64, build-native-metadata-windows-x86_64, build-native-metadata-macos-x86_64 ]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
@@ -124,6 +216,18 @@ jobs:
         with:
           name: lakesoul-nativeio-x86_64-pc-windows-msvc
           path: ./native-io/target/release/
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu
+          path: ./native-metadata/target/release/
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-apple-darwin
+          path: ./native-metadata/target/release/
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativemetadata-x86_64-pc-windows-msvc
+          path: ./native-metadata/target/release/
       - name: Set up JDK 8
         uses: actions/setup-java@v3
         with:

.github/workflows/rust-clippy.yml

@@ -33,9 +33,15 @@ jobs:
           toolchain: nightly-2023-05-20
           components: clippy
           default: true
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: "./native-io -> target"
           key: "ubuntu-latest-clippy"
-      - name: Run Clippy
+      - name: Run Clippy Native-io
         run: cd native-io && cargo clippy --all-features --package lakesoul-io-c
+      - name: Run Clippy Native-metadata
+        run: cd native-metadata && cargo clippy --all-features --package lakesoul-metadata-c

.gitignore

@@ -17,7 +17,6 @@ dependency-reduced-pom.xml
 /lakesoul-spark/spark-warehouse/
 dependency-reduced-pom.xml
 /lakesoul-spark/spark-warehouse/
-lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/entity/
 /script/benchmark/work-dir/*.jar
 *.DS_Store
 native-io/lakesoul-io-c/lakesoul_c_bindings.h