diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a7534ff4..421d227a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -20,10 +20,23 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: shogo82148/actions-setup-mysql@v1 + with: + mysql-version: "5.7" + root-password: "password" + my-cnf: | + skip-ssl + port=3306 + - name: install diesel_cli + run: cargo install diesel_cli --no-default-features --features mysql + - name: init database + run: diesel setup --database-url mysql://root:password@127.0.0.1:3306/vault - name: Build run: cargo build --verbose - name: Run tests run: cargo test --verbose + + windows-test: strategy: matrix: @@ -34,11 +47,31 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append - - run: vcpkg install openssl:x64-windows-static-md + - name: install openssl + run: vcpkg install openssl:x64-windows-static-md + - name: Download MySQL Connector/C + run: | + Invoke-WebRequest -Uri "https://dev.mysql.com/get/Downloads/Connector-C/mysql-connector-c-6.1.11-winx64.msi" -OutFile "mysql-connector.msi" + - name: Install MySQL Connector/C + run: | + Start-Process msiexec.exe -ArgumentList '/i', 'mysql-connector.msi', '/quiet', '/norestart' -NoNewWindow -Wait + - name: Set MySQLCLIENT_LIB_DIR + run: echo "MYSQLCLIENT_LIB_DIR=C:\Program Files\MySQL\MySQL Connector C 6.1\lib\vs14" | Out-File -FilePath $env:GITHUB_ENV -Append + - uses: shogo82148/actions-setup-mysql@v1 + with: + mysql-version: "5.7" + root-password: "password" + my-cnf: | + skip-ssl + port=3306 - name: Setup Rust uses: actions-rs/toolchain@v1 with: toolchain: stable + - name: install diesel_cli + run: cargo install diesel_cli --no-default-features --features mysql + - name: init database + run: diesel setup --database-url mysql://root:password@127.0.0.1:3306/vault - name: Build run: cargo build --verbose - name: Run tests diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..faf18479 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "rust-analyzer.linkedProjects": [ + ".\\Cargo.toml", + ".\\Cargo.toml", + ".\\Cargo.toml", + ".\\Cargo.toml", + ".\\Cargo.toml", + ".\\Cargo.toml" + ] +} \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 27af6039..8a36d6bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,9 @@ as-any = "0.3.1" pem = "3.0" chrono = "0.4" zeroize = { version = "1.7.0", features= ["zeroize_derive"] } +diesel = { version = "2.1.4", features = ["mysql", "r2d2"] } +r2d2 = "0.8.9" +r2d2-diesel = "1.0.0" bcrypt = "0.15" [target.'cfg(unix)'.dependencies] diff --git a/diesel.toml b/diesel.toml new file mode 100644 index 00000000..c028f4a6 --- /dev/null +++ b/diesel.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema.rs" +custom_type_derives = ["diesel::query_builder::QueryId"] + +[migrations_directory] +dir = "migrations" diff --git a/doc/backend/database/mysql/mysql.md b/doc/backend/database/mysql/mysql.md new file mode 100644 index 00000000..878a86a3 --- /dev/null +++ b/doc/backend/database/mysql/mysql.md @@ -0,0 +1,65 @@ +# MySQL Optimization with Diesel CLI in Rust + +This document outlines the process of setting up and using `diesel_cli` with MySQL in Rust, and discusses potential areas for optimization. + +## Using Diesel CLI with MySQL in Rust + +`diesel_cli` is an ORM (Object-Relational Mapping) framework that enables the generation of structs and DSL (Domain Specific Language) from SQL files. The following steps guide you through setting up and using `diesel_cli` with MySQL in your Rust project. + +### Step 1: Environment Setup + +Firstly, define the `MYSQLCLIENT_LIB_DIR` environment variable. The process varies depending on your platform: + +- **Linux**: + ```shell + export MYSQLCLIENT_LIB_DIR="your path to mysqlclient.lib" + ``` + +- **Windows**: + ```shell + setx MYSQLCLIENT_LIB_DIR "your path to mysqlclient.lib" + ``` + +- **GitHub Actions on Windows**: + ```shell + - run: echo "MYSQLCLIENT_LIB_DIR=C:\Program Files\MySQL\MySQL Connector C 6.1\lib\vs14" | Out-File -FilePath $env:GITHUB_ENV -Append + ``` + +> Note: If you do not only set the `mysqlclient.lib` file, it may result in duplicate library errors during compilation. + +### Step 2: Install Diesel CLI +Install `diesel_cli` using the `cargo` command: + +```shell +cargo install diesel_cli --no-default-features --features mysql +``` + +### Step 3: Import Diesel into the Project + +Add the following dependencies to your `Cargo.toml`: + +```toml +[dependencies] +# other dependencies +diesel = { version = "2.1.4", features = ["mysql", "r2d2"] } +r2d2 = "0.8.9" +r2d2-diesel = "1.0.0" +``` + +### Step 4: Generate Structs with Diesel CLI + +Use `diesel_cli` to set up your database and generate migrations: + +```shell +cd /path/to/project/root +diesel setup --database-url="mysql://[username:[password]]@[host:[port]]/[database]" +diesel migration generate your_sql_summary --database-url="mysql://[username:[password]]@[host:[port]]/[database]" +diesel migration run "mysql://[username:[password]]@[host:[port]]/[database]" +``` + +Run the unit test with `mysqlbackend`. + +## Potential Optimization Areas + +- Establishing a TLS connection to MySQL +- Connecting to PostgreSQL diff --git a/doc/backend/database/overview.md b/doc/backend/database/overview.md new file mode 100644 index 00000000..633f143d --- /dev/null +++ b/doc/backend/database/overview.md @@ -0,0 +1 @@ +# Using diesel & r2d2 to execute database \ No newline at end of file diff --git a/migrations/.keep b/migrations/.keep new file mode 100644 index 00000000..e69de29b diff --git a/migrations/2024-03-07-084239_create_vault_table/down.sql b/migrations/2024-03-07-084239_create_vault_table/down.sql new file mode 100644 index 00000000..3511e3ec --- /dev/null +++ b/migrations/2024-03-07-084239_create_vault_table/down.sql @@ -0,0 +1 @@ +drop table `vault`; diff --git a/migrations/2024-03-07-084239_create_vault_table/up.sql b/migrations/2024-03-07-084239_create_vault_table/up.sql new file mode 100644 index 00000000..6e3952e4 --- /dev/null +++ b/migrations/2024-03-07-084239_create_vault_table/up.sql @@ -0,0 +1,6 @@ +-- Create table vault +CREATE TABLE IF NOT EXISTS `vault` ( + `vault_key` varbinary(3072) NOT NULL, + `vault_value` mediumblob, + PRIMARY KEY (`vault_key`) +); diff --git a/src/errors.rs b/src/errors.rs index 1c45fac6..e9f29a9c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -210,6 +210,26 @@ pub enum RvError { ErrRwLockReadPoison, #[error("RwLock was poisoned (writing)")] ErrRwLockWritePoison, + + /// Database Errors Begin + /// + #[error("Database type is not support now. Please try postgressql or mysql again.")] + ErrDatabaseTypeInvalid, + #[error("Database connection pool ocurrs errors when creating， {:?}", .source)] + ErrConnectionPoolCreate { + #[from] + source: r2d2::Error, + }, + #[error("Database connection info is invalid.")] + ErrDatabaseConnectionInfoInvalid, + #[error("Failed to execute entry with database, {:?}", .source)] + ErrDatabaseExecuteEntry { + #[from] + source: diesel::result::Error, + }, + /// + /// Database Errors End + #[error(transparent)] ErrOther(#[from] anyhow::Error), #[error("Unknown error.")] diff --git a/src/lib.rs b/src/lib.rs index e5d28ed7..d782aa97 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ +#[macro_use] +extern crate diesel; + pub mod cli; pub mod context; pub mod core; @@ -12,6 +15,7 @@ pub mod router; pub mod shamir; pub mod storage; pub mod utils; +pub mod schema; /// Exit ok pub const EXIT_CODE_OK: sysexits::ExitCode = sysexits::ExitCode::Ok; diff --git a/src/schema.rs b/src/schema.rs new file mode 100644 index 00000000..28b112a2 --- /dev/null +++ b/src/schema.rs @@ -0,0 +1,8 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + vault (vault_key) { + vault_key -> Varchar, + vault_value -> Varbinary, + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 4738fecf..df3435b6 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -6,6 +6,7 @@ pub mod barrier; pub mod barrier_aes_gcm; pub mod barrier_view; pub mod physical; +pub mod mysql; pub trait Storage { fn list(&self, prefix: &str) -> Result, RvError>; diff --git a/src/storage/mysql/mod.rs b/src/storage/mysql/mod.rs new file mode 100644 index 00000000..3857ae48 --- /dev/null +++ b/src/storage/mysql/mod.rs @@ -0,0 +1,69 @@ +use std::collections::HashMap; + +use diesel::mysql::MysqlConnection; +use diesel::r2d2::{self, ConnectionManager}; + +use crate::errors::RvError; +use serde_json::Value; + +type MysqlDbPool = r2d2::Pool>; + +pub mod mysql_backend; + +pub fn new(conf: &HashMap) -> Result { + let pool = establish_mysql_connection(conf); + match pool { + Ok(pool)=> Ok(pool), + Err(e)=> Err(e), + } +} + +/** + * The `establish_mysql_connection` function is used to establish a connection to a MySQL database. + * The function takes a configuration object as an argument and returns a `Result` containing a `MysqlDbPool` or an `RvError`. + */ +fn establish_mysql_connection(conf: &HashMap) -> Result { + let address = conf.get("address").and_then(|v| v.as_str()).ok_or(RvError::ErrDatabaseConnectionInfoInvalid)?; + + let database = conf.get("database").and_then(|v| v.as_str()).unwrap_or("vault"); + let username = conf.get("username").and_then(|v| v.as_str()).ok_or(RvError::ErrDatabaseConnectionInfoInvalid)?; + let password = conf.get("password").and_then(|v| v.as_str()).ok_or(RvError::ErrDatabaseConnectionInfoInvalid)?; + + // let table = conf.get("table").and_then(|v| v.as_str()).unwrap_or("vault"); + // let tls_ca_file = conf.get("tls_ca_file").and_then(|v| v.as_str()).unwrap_or(""); + // let plaintext_credentials_transmission = conf.get("plaintext_credentials_transmission").and_then(|v| v.as_str()).unwrap_or(""); + // let max_parralel = conf.get("max_parralel").and_then(|v| v.as_i64()).unwrap_or(128) as i32; + // let max_idle_connections = conf.get("max_idle_connections").and_then(|v| v.as_i64()).unwrap_or(0) as i32; + // let max_connection_lifetime = conf.get("max_connection_lifetime").and_then(|v| v.as_i64()).unwrap_or(0) as i32; + // + // now this can not support ssl connection yet. Still need to improve it. + let database_url = format!("mysql://{}:{}@{}/{}", username, password, address, database); + + let manager = ConnectionManager::::new(database_url); + match r2d2::Pool::builder().build(manager) { + Ok(pool) => Ok(pool), + Err(e) => { + log::error!("Error: {:?}", e); + Err(RvError::ErrConnectionPoolCreate { source: (e) }) + }, + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use super::*; + + #[test] + fn test_establish_mysql_connection() { + let mut conf: HashMap = HashMap::new(); + conf.insert("address".to_string(), Value::String("127.0.0.1:3306".to_string())); + conf.insert("username".to_string(), Value::String("root".to_string())); + conf.insert("password".to_string(), Value::String("password".to_string())); + + let pool = establish_mysql_connection(&conf); + + assert!(pool.is_ok()); + } +} diff --git a/src/storage/mysql/mysql_backend.rs b/src/storage/mysql/mysql_backend.rs new file mode 100644 index 00000000..c5f0100b --- /dev/null +++ b/src/storage/mysql/mysql_backend.rs @@ -0,0 +1,154 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use diesel::prelude::*; +use diesel::{r2d2::ConnectionManager, MysqlConnection}; +use r2d2::Pool; +use serde::Deserialize; +use serde_json::Value; + +use crate::schema::vault; +use crate::schema::vault::dsl::*; +use crate::{ + errors::RvError, + schema::vault::vault_key, + storage::physical::{Backend, BackendEntry}, +}; + +use super::new; + +pub struct MysqlBackend { + pool: Arc>>>, +} + +#[derive(Insertable, Queryable, PartialEq, Debug, Deserialize)] +#[diesel(table_name = vault)] +pub struct MysqlBackendEntry { + pub vault_key: String, + pub vault_value: Vec, +} + +impl Backend for MysqlBackend { + fn list(&self, prefix: &str) -> Result, RvError> { + if prefix.starts_with("/") { + return Err(RvError::ErrPhysicalBackendPrefixInvalid); + } + + let conn: &mut MysqlConnection = &mut self.pool.lock().unwrap().get().unwrap(); + + let results: Result, _> = + vault.filter(vault_key.like(format!("{}%", prefix))).load::(conn); + + match results { + Ok(entries) => { + let mut keys: Vec = Vec::new(); + for entry in entries { + let key = entry.vault_key.clone(); + let key = key.trim_start_matches(prefix); + match key.find('/') { + Some(i) => { + let key = &key[0..i+1]; + if !keys.contains(&key.to_string()) { + keys.push(key.to_string()); + } + } + None => { + keys.push(key.to_string()); + } + } + } + return Ok(keys); + } + Err(e) => return Err(RvError::ErrDatabaseExecuteEntry { source: (e) }), + } + } + + fn get(&self, key: &str) -> Result, RvError> { + if key.starts_with("/") { + return Err(RvError::ErrPhysicalBackendKeyInvalid); + } + + let conn: &mut MysqlConnection = &mut self.pool.lock().unwrap().get().unwrap(); + + let result: Result = vault.filter(vault_key.eq(key)).first::(conn); + + match result { + Ok(entry) => return Ok(Some(BackendEntry { key: entry.vault_key, value: entry.vault_value })), + Err(e) => { + if e == diesel::NotFound { + return Ok(None); + } else { + return Err(RvError::ErrDatabaseExecuteEntry { source: (e) }); + } + } + } + } + + fn put(&self, entry: &BackendEntry) -> Result<(), RvError> { + if entry.key.as_str().starts_with("/") { + return Err(RvError::ErrPhysicalBackendKeyInvalid); + } + + let conn: &mut MysqlConnection = &mut self.pool.lock().unwrap().get().unwrap(); + + let new_entry = MysqlBackendEntry { vault_key: entry.key.clone(), vault_value: entry.value.clone() }; + + match diesel::replace_into(vault).values(&new_entry).execute(conn) { + Ok(_) => return Ok(()), + Err(e) => return Err(RvError::ErrDatabaseExecuteEntry { source: (e) }), + } + } + + fn delete(&self, key: &str) -> Result<(), RvError> { + if key.starts_with("/") { + return Err(RvError::ErrPhysicalBackendKeyInvalid); + } + + let conn: &mut MysqlConnection = &mut self.pool.lock().unwrap().get().unwrap(); + + match diesel::delete(vault.filter(vault_key.eq(key))).execute(conn) { + Ok(_) => return Ok(()), + Err(e) => return Err(RvError::ErrDatabaseExecuteEntry { source: (e) }), + } + } +} + +impl MysqlBackend { + pub fn new(conf: &HashMap) -> Result { + match new(conf) { + Ok(pool) => Ok(MysqlBackend { pool: Arc::new(Mutex::new(pool)) }), + Err(e) => Err(e), + } + } +} + +#[cfg(test)] +mod test { + + use serde_json::Value; + use std::collections::HashMap; + + use crate::storage::physical::test::test_backend; + use crate::storage::physical::test::test_backend_list_prefix; + + use super::MysqlBackend; + + #[test] + fn test_mysql_backend() { + let mut conf: HashMap = HashMap::new(); + conf.insert("address".to_string(), Value::String("127.0.0.1:3306".to_string())); + conf.insert("username".to_string(), Value::String("root".to_string())); + conf.insert("password".to_string(), Value::String("password".to_string())); + + let backend = MysqlBackend::new(&conf); + + assert!(backend.is_ok()); + + let backend = backend.unwrap(); + + test_backend(&backend); + test_backend_list_prefix(&backend); + } +} diff --git a/src/storage/physical/mod.rs b/src/storage/physical/mod.rs index 2d17b0f9..59d6614b 100644 --- a/src/storage/physical/mod.rs +++ b/src/storage/physical/mod.rs @@ -5,6 +5,9 @@ use serde_json::Value; use crate::errors::RvError; +use super::mysql::mysql_backend::MysqlBackend; + + pub mod file; pub mod mock; @@ -27,14 +30,18 @@ pub fn new_backend(t: &str, conf: &HashMap) -> Result { let backend = file::FileBackend::new(conf)?; Ok(Arc::new(backend)) - } + }, + "mysql" => { + let backend = MysqlBackend::new(conf)?; + Ok(Arc::new(backend)) + }, "mock" => Ok(Arc::new(mock::MockBackend::new())), _ => Err(RvError::ErrPhysicalTypeInvalid), } } #[cfg(test)] -mod test { +pub mod test { use std::{collections::HashMap, env, fs}; use go_defer::defer;