diff --git a/judger/Cargo.toml b/judger/Cargo.toml index de497ac..009119d 100644 --- a/judger/Cargo.toml +++ b/judger/Cargo.toml @@ -52,8 +52,3 @@ path ="src/cli/main.rs" [[bin]] name ="judger-server" path ="src/server/main.rs" - -[[bin]] -name ="judger-client" -path ="src/client/main.rs" - diff --git a/judger/README.md b/judger/README.md index bc79642..3bc02a7 100644 --- a/judger/README.md +++ b/judger/README.md @@ -31,7 +31,7 @@ cargo run --bin judger-cli -- batch-judge \ ### How to run -`cargo run --bin judger-server -- --env-path ./judge-server/src/environment/.env.development` +`cargo run --bin judger-server -- --env-path ./judger/src/server/environment/.env.development` ### How to visit OpenAPI diff --git a/judger/src/client/environment/.env.development b/judger/src/client/environment/.env.development deleted file mode 100644 index bdbf771..0000000 --- a/judger/src/client/environment/.env.development +++ /dev/null @@ -1,3 +0,0 @@ -BASE_URL="http://localhost:8080/api/v1/judge" -INTERVAL=10 -RUST_LOG=DEBUG \ No newline at end of file diff --git a/judger/src/client/environment/mod.rs b/judger/src/client/environment/mod.rs deleted file mode 100644 index efc681f..0000000 --- a/judger/src/client/environment/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -use structopt::StructOpt; - -#[derive(StructOpt, Debug, Clone)] -#[structopt(name = "judge-client")] -pub struct JudgeClientOpt { - #[structopt(long)] - pub env_path: Option, - - /// Port to listen to - #[structopt(env = "BASE_URL", default_value = "http://localhost:8080/api/v1/judge")] - pub base_url: String, - - #[structopt(env = "INTERVAL", default_value = "10")] - pub interval: i32, -} - -pub fn load_option() -> JudgeClientOpt { - // First load env_path from Args - let opt = JudgeClientOpt::from_args(); - if let Some(env_path) = opt.env_path { - dotenv::from_path(env_path).ok(); - } else { - dotenv::dotenv().ok(); - } - - // Load opt again with ENV - let opt = JudgeClientOpt::from_args(); - log::debug!("load opt: {:?}", opt); - opt -} - -pub fn setup_logger() { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("debug")).init(); -} diff --git a/judger/src/client/error.rs b/judger/src/client/error.rs deleted file mode 100644 index ef854be..0000000 --- a/judger/src/client/error.rs +++ /dev/null @@ -1,27 +0,0 @@ -use judge_core::error::JudgeCoreError; - -#[derive(Debug, thiserror::Error)] -pub enum ClientError { - #[error("Internal Error: {0}")] - InternalError(anyhow::Error), - #[error("Pick Error: {0}")] - PickFail(anyhow::Error), - #[error("Report Error: {0}")] - ReportFail(anyhow::Error), - #[error("Reqwest Error: {0}")] - ReqwestError(reqwest::Error), - #[error("Judge Core Error")] - JudgeError(JudgeCoreError), -} - -impl From for ClientError { - fn from(err: reqwest::Error) -> ClientError { - ClientError::ReqwestError(err) - } -} - -impl From for ClientError { - fn from(err: JudgeCoreError) -> ClientError { - ClientError::JudgeError(err) - } -} diff --git a/judger/src/lib.rs b/judger/src/lib.rs index 0443af3..1f278a4 100644 --- a/judger/src/lib.rs +++ b/judger/src/lib.rs @@ -1,2 +1 @@ -#[allow(dead_code)] pub mod service; diff --git a/judger/src/client/client.rs b/judger/src/server/client/httpclient.rs similarity index 100% rename from judger/src/client/client.rs rename to judger/src/server/client/httpclient.rs diff --git a/judger/src/client/main.rs b/judger/src/server/client/mod.rs similarity index 84% rename from judger/src/client/main.rs rename to judger/src/server/client/mod.rs index 8e5553f..fe5804b 100644 --- a/judger/src/client/main.rs +++ b/judger/src/server/client/mod.rs @@ -1,9 +1,7 @@ -mod client; -mod environment; -mod error; - -use client::HttpClient; -use error::ClientError; +mod httpclient; +use crate::error::ClientError; +use crate::service::state; +use httpclient::HttpClient; use judge_core::judge; use judge_core::{ compiler::Language, @@ -12,6 +10,7 @@ use judge_core::{ judge::JudgeConfig, package::PackageType, }; +use judger::service::package_manager::package; use serde::{Deserialize, Serialize}; use std::time::Duration; use std::{fs, path::PathBuf}; @@ -47,13 +46,9 @@ struct JudgeTask { redis_stream_id: String, } -#[tokio::main] -async fn main() { - let opt = environment::load_option(); - environment::setup_logger(); - let mut interval = interval(Duration::from_secs(10)); - let base_url = opt.base_url; - let client = client::HttpClient::new(base_url); +pub async fn run_client(base_url: String, interval_sec: u64) { + let mut interval = interval(Duration::from_secs(interval_sec)); + let client = HttpClient::new(base_url); loop { interval.tick().await; @@ -88,7 +83,9 @@ async fn pick_task(client: &HttpClient) -> Result { match response.status() { reqwest::StatusCode::OK => Ok(response.json::().await?.task), - _ => Err(ClientError::PickFail(anyhow::anyhow!("Queue is empty"))), + _ => Err(ClientError::InternalError(anyhow::anyhow!( + "Queue is empty" + ))), } } @@ -117,12 +114,18 @@ async fn report_task( ); Ok(()) } - _ => Err(ClientError::ReportFail(anyhow::anyhow!("Report Failed"))), + _ => Err(ClientError::InternalError(anyhow::anyhow!("Report Failed"))), } } fn run_judge(task: JudgeTask) -> Result, ClientError> { - let problem_package_dir = PathBuf::from("data/dev-problem-package"); + if let Err(sync_err) = package::sync_package(&PathBuf::from("data"), "oj-lab-problem-package") { + return Err(ClientError::PackageError(sync_err)); + }; + if let Err(set_err) = state::set_busy() { + return Err(ClientError::InternalError(set_err)); + } + let problem_package_dir = PathBuf::from(format!("data/{}", package::PACKAGE_SAVE_DIRNAME)); let problem_slug = task.problem_slug; let uuid = uuid::Uuid::new_v4(); let runtime_path = PathBuf::from("/tmp").join(uuid.to_string()); @@ -166,5 +169,6 @@ fn run_judge(task: JudgeTask) -> Result, ClientError> { } log::debug!("Judge finished"); + state::set_idle(); Ok(results) } diff --git a/judger/src/server/environment/.env.development b/judger/src/server/environment/.env.development index 4a0f8bb..658875e 100644 --- a/judger/src/server/environment/.env.development +++ b/judger/src/server/environment/.env.development @@ -1,2 +1,4 @@ PORT=8081 +BASE_URL="http://localhost:8080/api/v1/judge" +INTERVAL=10 RUST_LOG=DEBUG \ No newline at end of file diff --git a/judger/src/server/environment/mod.rs b/judger/src/server/environment/mod.rs index 22a13ea..acdcc4c 100644 --- a/judger/src/server/environment/mod.rs +++ b/judger/src/server/environment/mod.rs @@ -14,6 +14,12 @@ pub struct JudgeServerOpt { #[structopt(long, default_value = "data/dev-problem-package")] pub problem_package_dir: PathBuf, + + #[structopt(env = "BASE_URL", default_value = "http://localhost:8080/api/v1/judge")] + pub base_url: String, + + #[structopt(env = "INTERVAL", default_value = "10")] + pub interval: i32, } pub fn load_option() -> JudgeServerOpt { diff --git a/judger/src/server/error.rs b/judger/src/server/error.rs index 8d6c08a..e9de4bc 100644 --- a/judger/src/server/error.rs +++ b/judger/src/server/error.rs @@ -2,6 +2,7 @@ use actix_web::{HttpResponse, ResponseError}; use judge_core::error::JudgeCoreError; +use judger::service::error::JudgeServiceError; #[derive(Debug, thiserror::Error)] pub enum ServiceError { @@ -19,6 +20,12 @@ pub enum ServiceError { UnauthorizedWithMsg(anyhow::Error, String), } +#[derive(Debug)] +pub enum ClientError { + InternalError(anyhow::Error), + PackageError(JudgeServiceError), +} + #[derive(Serialize)] struct ServiceErrorBody { msg: Option, @@ -67,3 +74,15 @@ impl From for ServiceError { Self::InternalError(anyhow::anyhow!("{:?}", value)) } } + +impl From for ClientError { + fn from(value: reqwest::Error) -> Self { + Self::InternalError(anyhow::anyhow!("{:?}", value)) + } +} + +impl From for ClientError { + fn from(value: JudgeCoreError) -> Self { + Self::InternalError(anyhow::anyhow!("{:?}", value)) + } +} diff --git a/judger/src/server/main.rs b/judger/src/server/main.rs index 2efc354..1588b65 100644 --- a/judger/src/server/main.rs +++ b/judger/src/server/main.rs @@ -1,3 +1,4 @@ +mod client; mod environment; mod error; mod service; @@ -22,6 +23,10 @@ async fn main() -> std::io::Result<()> { // } // }); + tokio::spawn(async move { + client::run_client(opt.base_url, opt.interval as u64).await; + }); + let port = opt.port; HttpServer::new(move || { diff --git a/judger/src/server/service/mod.rs b/judger/src/server/service/mod.rs index e031fb6..ae4878b 100644 --- a/judger/src/server/service/mod.rs +++ b/judger/src/server/service/mod.rs @@ -1,6 +1,6 @@ mod greet; mod judge; -mod state; +pub mod state; use actix_web::web; use utoipa::OpenApi; diff --git a/judger/src/service/error.rs b/judger/src/service/error.rs index 42e7ad6..11925bd 100644 --- a/judger/src/service/error.rs +++ b/judger/src/service/error.rs @@ -6,6 +6,7 @@ use judge_core::error::JudgeCoreError; pub enum JudgeServiceError { JudgeCoreError(JudgeCoreError), IOError(io::Error), + RcloneError(anyhow::Error), AnyhowError(anyhow::Error), } diff --git a/judger/src/service/package_manager/description.rs b/judger/src/service/package_manager/description.rs new file mode 100644 index 0000000..f7f762b --- /dev/null +++ b/judger/src/service/package_manager/description.rs @@ -0,0 +1,100 @@ +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; + +use judge_core::{error::JudgeCoreError, package::PackageType}; +use serde_derive::{Deserialize, Serialize}; + +use crate::service::error::JudgeServiceError; + +pub const PACKAGES_DESCRIPTION_FILE_NAME: &str = "judge-pd.json"; + +#[derive(Debug, Serialize, Deserialize)] +pub struct PackageDescription { + pub name: String, + pub revision: u32, + pub package_type: PackageType, +} + +impl PackageDescription { + pub fn new(name: String, package_type: PackageType) -> Result { + Ok(Self { + name, + revision: 0, + package_type, + }) + } +} + +pub struct StoragedPackageDescriptionMap { + pub folder_path: PathBuf, + pub package_description_map: HashMap, +} + +impl StoragedPackageDescriptionMap { + pub fn init(folder_path: PathBuf) -> Result { + init_package_description_file(&folder_path)?; + let package_description_map = HashMap::new(); + Ok(Self { + folder_path, + package_description_map, + }) + } + + pub fn load(folder_path: PathBuf) -> Result { + let package_description_map = load_package_description_map(&folder_path)?; + Ok(Self { + folder_path, + package_description_map, + }) + } + + pub fn insert( + &mut self, + package_description: PackageDescription, + ) -> Result<(), JudgeCoreError> { + self.package_description_map + .insert(package_description.name.clone(), package_description); + update_package_description_file(&self.folder_path, &self.package_description_map)?; + Ok(()) + } + + pub fn get(&self, package_name: &str) -> Option<&PackageDescription> { + self.package_description_map.get(package_name) + } +} + +fn load_package_description_map( + folder: &Path, +) -> Result, JudgeCoreError> { + let description_file_content = fs::read_to_string(folder.join(PACKAGES_DESCRIPTION_FILE_NAME))?; + let package_description_map: HashMap = + serde_json::from_str(&description_file_content)?; + Ok(package_description_map) +} + +fn init_package_description_file(folder: &Path) -> Result<(), JudgeCoreError> { + let package_description_map = HashMap::::new(); + let package_description_file_content = serde_json::to_string_pretty(&package_description_map)?; + + fs::write( + folder.join(PACKAGES_DESCRIPTION_FILE_NAME), + package_description_file_content, + )?; + Ok(()) +} + +fn update_package_description_file( + folder: &Path, + package_description_map: &HashMap, +) -> Result<(), JudgeCoreError> { + let package_description_file_content = serde_json::to_string_pretty(package_description_map)?; + + fs::write( + folder.join(PACKAGES_DESCRIPTION_FILE_NAME), + package_description_file_content, + )?; + Ok(()) +} diff --git a/judger/src/service/package_manager/discription.rs b/judger/src/service/package_manager/discription.rs deleted file mode 100644 index d1fe137..0000000 --- a/judger/src/service/package_manager/discription.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::{ - collections::HashMap, - fs, - path::{Path, PathBuf}, -}; - -use judge_core::{error::JudgeCoreError, package::PackageType}; -use serde_derive::{Deserialize, Serialize}; - -use crate::service::error::JudgeServiceError; - -pub const PACKAGES_DISCRIPTION_FILE_NAME: &str = "judge-pd.json"; - -#[derive(Debug, Serialize, Deserialize)] -pub struct PackageDiscription { - pub name: String, - pub revision: u32, - pub package_type: PackageType, -} - -impl PackageDiscription { - pub fn new(name: String, package_type: PackageType) -> Result { - Ok(Self { - name, - revision: 0, - package_type, - }) - } -} - -pub struct StoragedPackageDiscriptionMap { - pub folder_path: PathBuf, - pub package_discription_map: HashMap, -} - -impl StoragedPackageDiscriptionMap { - pub fn init(folder_path: PathBuf) -> Result { - init_package_discription_file(&folder_path)?; - let package_discription_map = HashMap::new(); - Ok(Self { - folder_path, - package_discription_map, - }) - } - - pub fn load(folder_path: PathBuf) -> Result { - let package_discription_map = load_package_discription_map(&folder_path)?; - Ok(Self { - folder_path, - package_discription_map, - }) - } - - pub fn insert( - &mut self, - package_discription: PackageDiscription, - ) -> Result<(), JudgeCoreError> { - self.package_discription_map - .insert(package_discription.name.clone(), package_discription); - update_package_discription_file(&self.folder_path, &self.package_discription_map)?; - Ok(()) - } - - pub fn get(&self, package_name: &str) -> Option<&PackageDiscription> { - self.package_discription_map.get(package_name) - } -} - -fn load_package_discription_map( - folder: &Path, -) -> Result, JudgeCoreError> { - let discription_file_content = fs::read_to_string(folder.join(PACKAGES_DISCRIPTION_FILE_NAME))?; - let package_discription_map: HashMap = - serde_json::from_str(&discription_file_content)?; - Ok(package_discription_map) -} - -fn init_package_discription_file(folder: &Path) -> Result<(), JudgeCoreError> { - let package_discription_map = HashMap::::new(); - let package_discription_file_content = serde_json::to_string_pretty(&package_discription_map)?; - - fs::write( - folder.join(PACKAGES_DISCRIPTION_FILE_NAME), - package_discription_file_content, - )?; - Ok(()) -} - -fn update_package_discription_file( - folder: &Path, - package_discription_map: &HashMap, -) -> Result<(), JudgeCoreError> { - let package_discription_file_content = serde_json::to_string_pretty(package_discription_map)?; - - fs::write( - folder.join(PACKAGES_DISCRIPTION_FILE_NAME), - package_discription_file_content, - )?; - Ok(()) -} diff --git a/judger/src/service/package_manager/mod.rs b/judger/src/service/package_manager/mod.rs index 04b0fb5..6440d6a 100644 --- a/judger/src/service/package_manager/mod.rs +++ b/judger/src/service/package_manager/mod.rs @@ -1,4 +1,5 @@ -pub mod discription; +pub mod description; +pub mod package; use std::path::PathBuf; @@ -6,11 +7,11 @@ use judge_core::package::PackageType; use crate::service::error::JudgeServiceError; -use self::discription::StoragedPackageDiscriptionMap; +use self::description::StoragedPackageDescriptionMap; pub struct PackageManager { pub folder_path: PathBuf, - pub package_discription_map: StoragedPackageDiscriptionMap, + pub package_description_map: StoragedPackageDescriptionMap, } impl PackageManager { @@ -26,22 +27,22 @@ impl PackageManager { std::fs::create_dir_all(&folder_path)?; } - let discription_file_path = folder_path.join(discription::PACKAGES_DISCRIPTION_FILE_NAME); - if discription_file_path.exists() && discription_file_path.is_dir() { + let description_file_path = folder_path.join(description::PACKAGES_DESCRIPTION_FILE_NAME); + if description_file_path.exists() && description_file_path.is_dir() { return Err(JudgeServiceError::AnyhowError(anyhow::anyhow!( - "Discription file '{}' appears to be a folder.", + "Description file '{}' appears to be a folder.", folder_path.display() ))); } - let package_discription_map = if !discription_file_path.exists() { - StoragedPackageDiscriptionMap::init(folder_path.clone())? + let package_description_map = if !description_file_path.exists() { + StoragedPackageDescriptionMap::init(folder_path.clone())? } else { - StoragedPackageDiscriptionMap::load(folder_path.clone())? + StoragedPackageDescriptionMap::load(folder_path.clone())? }; Ok(Self { folder_path, - package_discription_map, + package_description_map, }) } @@ -50,8 +51,8 @@ impl PackageManager { package_name: String, package_type: PackageType, ) -> Result<(), JudgeServiceError> { - let package_discription = self.package_discription_map.get(&package_name); - if package_discription.is_some() { + let package_description = self.package_description_map.get(&package_name); + if package_description.is_some() { return Err(JudgeServiceError::AnyhowError(anyhow::anyhow!( "Package '{}' already exists.", package_name @@ -62,9 +63,9 @@ impl PackageManager { .get_package_agent(self.folder_path.join(&package_name))? .validate() { - let package_discription = - discription::PackageDiscription::new(package_name, package_type)?; - self.package_discription_map.insert(package_discription)?; + let package_description = + description::PackageDescription::new(package_name, package_type)?; + self.package_description_map.insert(package_description)?; } else { return Err(JudgeServiceError::AnyhowError(anyhow::anyhow!( "Package '{}' is not valid.", diff --git a/judger/src/service/package_manager/package.rs b/judger/src/service/package_manager/package.rs new file mode 100644 index 0000000..3fed7f1 --- /dev/null +++ b/judger/src/service/package_manager/package.rs @@ -0,0 +1,51 @@ +use crate::service::error::JudgeServiceError; +use anyhow; +use std::path::Path; +use std::process::Command; + +fn check_rclone(config_path: &Path) -> Result<(), JudgeServiceError> { + let mut cmd = Command::new("rclone"); + let status = cmd + .arg("--config") + .arg(config_path) + .arg("ls") + .arg("minio:") + .status() + .expect("Failed to rclone"); + if status.success() { + Ok(()) + } else { + Err(JudgeServiceError::RcloneError(anyhow::anyhow!( + "rclone failed, please check config." + ))) + } +} + +pub const PACKAGE_SAVE_DIRNAME: &str = "rclone-problem-package"; +pub const RCLONE_CONFIG_FILE: &str = "rclone-minio.conf"; + +pub fn sync_package(data_dir: &Path, bucket_name: &str) -> Result<(), JudgeServiceError> { + let config_path = data_dir.join(RCLONE_CONFIG_FILE); + let package_path = data_dir.join(PACKAGE_SAVE_DIRNAME); + let check_res = check_rclone(&config_path); + if check_res.as_ref().is_err() { + return check_res; + } + println!("{:?}", data_dir); + let mut cmd = Command::new("rclone"); + let status = cmd + .arg("--config") + .arg(format!("{}", config_path.to_string_lossy())) + .arg("sync") + .arg(format!("minio:{}", bucket_name)) + .arg(format!("{}", package_path.to_string_lossy())) + .status() + .expect("Failed to rclone"); + if status.success() { + Ok(()) + } else { + Err(JudgeServiceError::RcloneError(anyhow::anyhow!( + "rclone sync failed, please check config." + ))) + } +} diff --git a/judger/tests/package_test.rs b/judger/tests/package_test.rs index 6d01528..3c8c72b 100644 --- a/judger/tests/package_test.rs +++ b/judger/tests/package_test.rs @@ -1,45 +1,45 @@ use std::path::PathBuf; use judge_core::package::PackageType; -use judger::service::package_manager::discription::{ - PackageDiscription, StoragedPackageDiscriptionMap, +use judger::service::package_manager::description::{ + PackageDescription, StoragedPackageDescriptionMap, }; const TEST_TEMP_PATH: &str = "tests/temp"; #[test] -fn test_storaged_package_discription_map() { +fn test_storaged_package_description_map() { let folder = PathBuf::from(TEST_TEMP_PATH); - let mut package_discription_map = StoragedPackageDiscriptionMap::init(folder.clone()).unwrap(); + let mut package_description_map = StoragedPackageDescriptionMap::init(folder.clone()).unwrap(); - let package_discription = PackageDiscription { + let package_description = PackageDescription { name: "test".to_string(), revision: 1, package_type: PackageType::ICPC, }; - package_discription_map.insert(package_discription).unwrap(); + package_description_map.insert(package_description).unwrap(); - let package_discription_map = StoragedPackageDiscriptionMap::load(folder).unwrap(); - assert_eq!(package_discription_map.package_discription_map.len(), 1); + let package_description_map = StoragedPackageDescriptionMap::load(folder).unwrap(); + assert_eq!(package_description_map.package_description_map.len(), 1); assert_eq!( - package_discription_map - .package_discription_map + package_description_map + .package_description_map .get("test") .unwrap() .name, "test" ); assert_eq!( - package_discription_map - .package_discription_map + package_description_map + .package_description_map .get("test") .unwrap() .revision, 1 ); assert_eq!( - package_discription_map - .package_discription_map + package_description_map + .package_description_map .get("test") .unwrap() .package_type,