add mysql backend (Tongsuo-Project#46)

* add mysql backend

* add mysqlclient.lib & change action yml

---------

Co-authored-by: Anyu AY5 Wang <[email protected]>

.github/workflows/rust.yml

@@ -20,10 +20,23 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+    - uses: shogo82148/actions-setup-mysql@v1
+      with:
+        mysql-version: "5.7"
+        root-password: "password"
+        my-cnf: |
+          skip-ssl
+          port=3306
+    - name: install diesel_cli
+      run: cargo install diesel_cli --no-default-features --features mysql
+    - name: init database
+      run: diesel setup --database-url mysql://root:[email protected]:3306/vault
     - name: Build
       run: cargo build --verbose
     - name: Run tests
       run: cargo test --verbose
+
+
   windows-test:
     strategy:
       matrix:
@@ -34,11 +47,31 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append
-    - run: vcpkg install openssl:x64-windows-static-md
+    - name: install openssl
+      run: vcpkg install openssl:x64-windows-static-md
+    - name: Download MySQL Connector/C
+      run: |
+        Invoke-WebRequest -Uri "https://dev.mysql.com/get/Downloads/Connector-C/mysql-connector-c-6.1.11-winx64.msi" -OutFile "mysql-connector.msi"
+    - name: Install MySQL Connector/C
+      run: |
+        Start-Process msiexec.exe -ArgumentList '/i', 'mysql-connector.msi', '/quiet', '/norestart' -NoNewWindow -Wait
+    - name: Set MySQLCLIENT_LIB_DIR
+      run: echo "MYSQLCLIENT_LIB_DIR=C:\Program Files\MySQL\MySQL Connector C 6.1\lib\vs14" | Out-File -FilePath $env:GITHUB_ENV -Append
+    - uses: shogo82148/actions-setup-mysql@v1
+      with:
+        mysql-version: "5.7"
+        root-password: "password"
+        my-cnf: |
+          skip-ssl
+          port=3306
     - name: Setup Rust
       uses: actions-rs/toolchain@v1
       with:
         toolchain: stable
+    - name: install diesel_cli
+      run: cargo install diesel_cli --no-default-features --features mysql
+    - name: init database
+      run: diesel setup --database-url mysql://root:[email protected]:3306/vault
     - name: Build
       run: cargo build --verbose
     - name: Run tests

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+    "rust-analyzer.linkedProjects": [
+        ".\\Cargo.toml",
+        ".\\Cargo.toml",
+        ".\\Cargo.toml",
+        ".\\Cargo.toml",
+        ".\\Cargo.toml",
+        ".\\Cargo.toml"
+    ]
+}

Cargo.toml

@@ -39,6 +39,9 @@ as-any = "0.3.1"
 pem = "3.0"
 chrono = "0.4"
 zeroize = { version = "1.7.0", features= ["zeroize_derive"] }
+diesel = { version = "2.1.4", features = ["mysql", "r2d2"] }
+r2d2 = "0.8.9"
+r2d2-diesel = "1.0.0"
 bcrypt = "0.15"
 
 [target.'cfg(unix)'.dependencies]

diesel.toml

@@ -0,0 +1,9 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"
+custom_type_derives = ["diesel::query_builder::QueryId"]
+
+[migrations_directory]
+dir = "migrations"

doc/backend/database/mysql/mysql.md

@@ -0,0 +1,65 @@
+# MySQL Optimization with Diesel CLI in Rust
+
+This document outlines the process of setting up and using `diesel_cli` with MySQL in Rust, and discusses potential areas for optimization.
+
+## Using Diesel CLI with MySQL in Rust
+
+`diesel_cli` is an ORM (Object-Relational Mapping) framework that enables the generation of structs and DSL (Domain Specific Language) from SQL files. The following steps guide you through setting up and using `diesel_cli` with MySQL in your Rust project.
+
+### Step 1: Environment Setup
+
+Firstly, define the `MYSQLCLIENT_LIB_DIR` environment variable. The process varies depending on your platform:
+
+- **Linux**:
+    ```shell
+    export MYSQLCLIENT_LIB_DIR="your path to mysqlclient.lib"
+    ```
+
+- **Windows**:
+    ```shell
+    setx MYSQLCLIENT_LIB_DIR "your path to mysqlclient.lib"
+    ```
+
+- **GitHub Actions on Windows**:
+    ```shell
+    - run: echo "MYSQLCLIENT_LIB_DIR=C:\Program Files\MySQL\MySQL Connector C 6.1\lib\vs14" | Out-File -FilePath $env:GITHUB_ENV -Append
+    ```
+
+> Note: If you do not only set the `mysqlclient.lib` file, it may result in duplicate library errors during compilation.
+
+### Step 2: Install Diesel CLI
+Install `diesel_cli` using the `cargo` command:
+
+```shell
+cargo install diesel_cli --no-default-features --features mysql
+```
+
+### Step 3: Import Diesel into the Project
+
+Add the following dependencies to your `Cargo.toml`:
+
+```toml
+[dependencies]
+# other dependencies
+diesel = { version = "2.1.4", features = ["mysql", "r2d2"] }
+r2d2 = "0.8.9"
+r2d2-diesel = "1.0.0"
+```
+
+### Step 4: Generate Structs with Diesel CLI
+
+Use `diesel_cli` to set up your database and generate migrations:
+
+```shell
+cd /path/to/project/root
+diesel setup --database-url="mysql://[username:[password]]@[host:[port]]/[database]"
+diesel migration generate your_sql_summary --database-url="mysql://[username:[password]]@[host:[port]]/[database]"
+diesel migration run "mysql://[username:[password]]@[host:[port]]/[database]"
+```
+
+Run the unit test with `mysqlbackend`.
+
+## Potential Optimization Areas
+
+- Establishing a TLS connection to MySQL
+- Connecting to PostgreSQL

doc/backend/database/overview.md

@@ -0,0 +1 @@
+# Using diesel & r2d2 to execute database

migrations/2024-03-07-084239_create_vault_table/down.sql

@@ -0,0 +1 @@
+drop table `vault`;

migrations/2024-03-07-084239_create_vault_table/up.sql

@@ -0,0 +1,6 @@
+-- Create table vault
+CREATE TABLE IF NOT EXISTS `vault` (
+    `vault_key` varbinary(3072) NOT NULL,
+    `vault_value` mediumblob,
+    PRIMARY KEY (`vault_key`)
+);

src/errors.rs

@@ -210,6 +210,26 @@ pub enum RvError {
     ErrRwLockReadPoison,
     #[error("RwLock was poisoned (writing)")]
     ErrRwLockWritePoison,
+
+    /// Database Errors Begin
+    ///
+    #[error("Database type is not support now. Please try postgressql or mysql again.")]
+    ErrDatabaseTypeInvalid,
+    #[error("Database connection pool ocurrs errors when creating， {:?}", .source)]
+    ErrConnectionPoolCreate {
+        #[from]
+        source: r2d2::Error,
+    },
+    #[error("Database connection info is invalid.")]
+    ErrDatabaseConnectionInfoInvalid,
+    #[error("Failed to execute entry with database, {:?}", .source)]
+    ErrDatabaseExecuteEntry {
+        #[from]
+        source: diesel::result::Error,
+    },
+    ///
+    /// Database Errors End
+
     #[error(transparent)]
     ErrOther(#[from] anyhow::Error),
     #[error("Unknown error.")]

src/lib.rs

@@ -1,3 +1,6 @@
+#[macro_use]
+extern crate diesel;
+
 pub mod cli;
 pub mod context;
 pub mod core;
@@ -12,6 +15,7 @@ pub mod router;
 pub mod shamir;
 pub mod storage;
 pub mod utils;
+pub mod schema;
 
 /// Exit ok
 pub const EXIT_CODE_OK: sysexits::ExitCode = sysexits::ExitCode::Ok;

src/schema.rs

@@ -0,0 +1,8 @@
+// @generated automatically by Diesel CLI.
+
+diesel::table! {
+    vault (vault_key) {
+        vault_key -> Varchar,
+        vault_value -> Varbinary,
+    }
+}

src/storage/mod.rs

@@ -6,6 +6,7 @@ pub mod barrier;
 pub mod barrier_aes_gcm;
 pub mod barrier_view;
 pub mod physical;
+pub mod mysql;
 
 pub trait Storage {
     fn list(&self, prefix: &str) -> Result<Vec<String>, RvError>;

src/storage/mysql/mod.rs

@@ -0,0 +1,69 @@
+use std::collections::HashMap;
+
+use diesel::mysql::MysqlConnection;
+use diesel::r2d2::{self, ConnectionManager};
+
+use crate::errors::RvError;
+use serde_json::Value;
+
+type MysqlDbPool = r2d2::Pool<ConnectionManager<MysqlConnection>>;
+
+pub mod mysql_backend;
+
+pub fn new(conf: &HashMap<String, Value>) -> Result<MysqlDbPool, RvError> {
+    let pool = establish_mysql_connection(conf);
+    match pool {
+        Ok(pool)=> Ok(pool),
+        Err(e)=> Err(e),
+    }
+}
+
+/**
+ * The `establish_mysql_connection` function is used to establish a connection to a MySQL database.
+ * The function takes a configuration object as an argument and returns a `Result` containing a `MysqlDbPool` or an `RvError`.
+ */
+fn establish_mysql_connection(conf: &HashMap<String, Value>) -> Result<MysqlDbPool, RvError> {
+    let address = conf.get("address").and_then(|v| v.as_str()).ok_or(RvError::ErrDatabaseConnectionInfoInvalid)?;
+
+    let database = conf.get("database").and_then(|v| v.as_str()).unwrap_or("vault");
+    let username = conf.get("username").and_then(|v| v.as_str()).ok_or(RvError::ErrDatabaseConnectionInfoInvalid)?;
+    let password = conf.get("password").and_then(|v| v.as_str()).ok_or(RvError::ErrDatabaseConnectionInfoInvalid)?;
+
+    // let table = conf.get("table").and_then(|v| v.as_str()).unwrap_or("vault");
+    // let tls_ca_file = conf.get("tls_ca_file").and_then(|v| v.as_str()).unwrap_or("");
+    // let plaintext_credentials_transmission = conf.get("plaintext_credentials_transmission").and_then(|v| v.as_str()).unwrap_or("");
+    // let max_parralel = conf.get("max_parralel").and_then(|v| v.as_i64()).unwrap_or(128) as i32;
+    // let max_idle_connections = conf.get("max_idle_connections").and_then(|v| v.as_i64()).unwrap_or(0) as i32;
+    // let max_connection_lifetime = conf.get("max_connection_lifetime").and_then(|v| v.as_i64()).unwrap_or(0) as i32;
+    //
+    // now this can not support ssl connection yet. Still need to improve it.
+    let database_url = format!("mysql://{}:{}@{}/{}", username, password, address, database);
+
+    let manager = ConnectionManager::<MysqlConnection>::new(database_url);
+    match r2d2::Pool::builder().build(manager) {
+        Ok(pool) => Ok(pool),
+        Err(e) => {
+            log::error!("Error: {:?}", e);
+            Err(RvError::ErrConnectionPoolCreate { source: (e) })
+        },
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashMap;
+
+    use super::*;
+
+    #[test]
+    fn test_establish_mysql_connection() {
+        let mut conf: HashMap<String, Value> = HashMap::new();
+        conf.insert("address".to_string(), Value::String("127.0.0.1:3306".to_string()));
+        conf.insert("username".to_string(), Value::String("root".to_string()));
+        conf.insert("password".to_string(), Value::String("password".to_string()));
+
+        let pool = establish_mysql_connection(&conf);
+
+        assert!(pool.is_ok());
+    }
+}