From cde4466939c93f1da3f1ac1820eb8a7cfbc488ac Mon Sep 17 00:00:00 2001 From: slhmy Date: Sat, 30 Mar 2024 22:03:41 +0800 Subject: [PATCH] Make dev more convient --- .dockerignore | 9 ++---- .github/workflows/docker-publish-judger.yml | 2 +- .vscode/launch.json | 28 +++++++++++++++++++ .vscode/settings.json | 12 ++------ .vscode/tasks.json | 13 +++++++++ docker/judger.dockerfile => Dockerfile | 12 ++++---- README.md | 18 ++++++++++-- docker/README.md | 9 ------ judger/.env.development | 4 --- judger/Cargo.toml | 2 +- judger/envs/development/.env | 2 ++ judger/envs/development/.gitignore | 1 + .../envs/development/rclone.conf | 0 judger/envs/docker/rclone.conf | 9 ++++++ judger/src/agent/platform/mod.rs | 5 +++- judger/src/agent/rclone.rs | 16 ++++++++--- judger/src/main.rs | 9 +++--- judger/src/{env/mod.rs => option.rs} | 25 +++++++++++------ judger/src/worker/mod.rs | 1 + 19 files changed, 120 insertions(+), 57 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json rename docker/judger.dockerfile => Dockerfile (72%) delete mode 100644 docker/README.md delete mode 100644 judger/.env.development create mode 100644 judger/envs/development/.env create mode 100644 judger/envs/development/.gitignore rename data/default-rclone.conf => judger/envs/development/rclone.conf (100%) create mode 100644 judger/envs/docker/rclone.conf rename judger/src/{env/mod.rs => option.rs} (62%) diff --git a/.dockerignore b/.dockerignore index f775f46..5834526 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,4 @@ +.github/ .vscode/ -tmp/ -test-collection/ -target/ -rclone/ -postman/ -docker/ \ No newline at end of file +.devcontainer/ +target/ \ No newline at end of file diff --git a/.github/workflows/docker-publish-judger.yml b/.github/workflows/docker-publish-judger.yml index ba9130d..35bfcb1 100644 --- a/.github/workflows/docker-publish-judger.yml +++ b/.github/workflows/docker-publish-judger.yml @@ -70,7 +70,7 @@ jobs: id: build-and-push uses: docker/build-push-action@v5 with: - file: docker/judger.dockerfile + file: Dockerfile context: . push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..f2d4ec2 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,28 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + // This name overrides rust-analyzer's debugging settings + "name": "run judger", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/target/debug/judger", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/judger/envs/development", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "preLaunchTask": "build debug judger", + "setupCommands": [ + { + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 09a44cd..fd0b8e8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,14 +1,6 @@ { "material-icon-theme.folders.associations": { "judge-core": "core", - "judge-cli": "command", - "judge-service": "helper", - "judge-server": "server", - "postman": "api" - }, - "rust-analyzer.linkedProjects": [ - "./judger/Cargo.toml", - "./judger/Cargo.toml", - "./judger/Cargo.toml" - ] + "judger": "app" + } } diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..8f01a88 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,13 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "build debug judger", + "type": "shell", + "command": "cargo", + "args": ["build", "--bin", "judger"] + } + ] +} diff --git a/docker/judger.dockerfile b/Dockerfile similarity index 72% rename from docker/judger.dockerfile rename to Dockerfile index 5bdc7cd..ade2178 100644 --- a/docker/judger.dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM rust:latest as build -COPY judge-core /usr/src/judge-core -COPY judger /usr/src/judger -WORKDIR /usr/src/judger +COPY judge-core/ /usr/src/judge-core +COPY judger/ /usr/src/judger +WORKDIR /usr/src/judger RUN apt update && apt install -y libseccomp-dev gcc RUN cargo build --bin judger --release @@ -16,10 +16,10 @@ COPY --from=build /usr/src/judger/target/release/judger /usr/local/bin/judger RUN curl https://rclone.org/install.sh | bash RUN mkdir /workspace -WORKDIR /workspace -COPY data/default-rclone.conf /workspace/data/default-rclone.conf -RUN mkdir /workspace/data/problem-package +RUN mkdir /workspace/problem-package +COPY judger/envs/docker/rclone.conf /workspace/rclone.conf +WORKDIR /workspace ENV RUST_LOG=DEBUG EXPOSE 8000 CMD [ "judger-server" ] \ No newline at end of file diff --git a/README.md b/README.md index 38cd07b..270af5b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Judger -![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/OJ-lab/judger/rust-check.yml?logo=github&label=Tests) [![Discord](https://img.shields.io/discord/916955582181822486?label=Discord&color=blue&logo=discord&logoColor=white)](https://discord.gg/vh8NCgdp8J) ![Codespace Supported](https://img.shields.io/badge/Codespace_Supported-000000?style=flat&logo=github) @@ -8,12 +7,27 @@ Judger is supposed to be a simple **sandbox service** which works for online-jud ## System -judger-rs currently use `nix` to make necessary system invoke like `fork()`. +Judger currently use `nix` to make necessary system invoke like `fork()`. So you might need to check whether you are using the supported system from the main-page of [nix](https://github.com/nix-rust/nix). **Briefly speaking, judger-rs is now supposing you are decided to run it on linux.** We'll consider other platform, but in a lower priority. +## Development + +Judger should works fine in VSCode with all recommended extensions installed. + +### Before you start + +You may need to setup your environment before you start. +There is a setup script to help you quickly get ready. + +> 🥰 You won't need to run this script if you are using GitHub Codespaces. + +```sh +./scripts/env_setup.bash +``` + ## Contribute We have a guide in judger's [WIKI](https://github.com/OJ-lab/judger/wiki/Contribution-Guide) diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 41688fa..0000000 --- a/docker/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Docker Build Guide - -⚠️ Do not use VSCode's `Build Image..` usage. - -Run the following command under project root instead. - -```sh -docker build --pull --rm -f "docker/judger-server.dockerfile" -t oj-lab/judger-server:latest . -``` diff --git a/judger/.env.development b/judger/.env.development deleted file mode 100644 index 1f43dad..0000000 --- a/judger/.env.development +++ /dev/null @@ -1,4 +0,0 @@ -PORT=8081 -BASE_URL=http://localhost:8080/api/v1/judge -INTERVAL=10 -RUST_LOG=DEBUG \ No newline at end of file diff --git a/judger/Cargo.toml b/judger/Cargo.toml index acf6ddf..b1e2159 100644 --- a/judger/Cargo.toml +++ b/judger/Cargo.toml @@ -12,7 +12,7 @@ judge-core = { path = "../judge-core" } clap = { version = "4.0", features = ["derive"] } # Client -reqwest = { version = "0.12.1", features = ["json"] } +reqwest = { version = "0.12", features = ["json"] } # Async runtime tokio = { version = "1", features = ["full"] } diff --git a/judger/envs/development/.env b/judger/envs/development/.env new file mode 100644 index 0000000..098bb4f --- /dev/null +++ b/judger/envs/development/.env @@ -0,0 +1,2 @@ +RUST_LOG=DEBUG +FETCH_TASK_INTERVAL=10 diff --git a/judger/envs/development/.gitignore b/judger/envs/development/.gitignore new file mode 100644 index 0000000..a4fd041 --- /dev/null +++ b/judger/envs/development/.gitignore @@ -0,0 +1 @@ +problem-package/ diff --git a/data/default-rclone.conf b/judger/envs/development/rclone.conf similarity index 100% rename from data/default-rclone.conf rename to judger/envs/development/rclone.conf diff --git a/judger/envs/docker/rclone.conf b/judger/envs/docker/rclone.conf new file mode 100644 index 0000000..b35e71b --- /dev/null +++ b/judger/envs/docker/rclone.conf @@ -0,0 +1,9 @@ +[minio] +type = s3 +provider = Minio +env_auth = false +access_key_id = minio-root-user +secret_access_key = minio-root-password +endpoint = http://host.docker.internal:9000 +location_constraint = +acl = private \ No newline at end of file diff --git a/judger/src/agent/platform/mod.rs b/judger/src/agent/platform/mod.rs index 179d061..aedff46 100644 --- a/judger/src/agent/platform/mod.rs +++ b/judger/src/agent/platform/mod.rs @@ -54,7 +54,10 @@ async fn pick_task(client: &HttpClient) -> Result { match response.status() { reqwest::StatusCode::OK => Ok(response.json::().await?.task), - _ => Err(anyhow::anyhow!("Queue is empty")), + _ => { + log::error!("Failed to pick task: {:?}", response); + Err(anyhow::anyhow!("Queue is empty")) + } } } diff --git a/judger/src/agent/rclone.rs b/judger/src/agent/rclone.rs index 6978230..2a922ec 100644 --- a/judger/src/agent/rclone.rs +++ b/judger/src/agent/rclone.rs @@ -12,13 +12,21 @@ impl RcloneClient { } pub fn is_avaliable(&self) -> bool { - let status = Command::new("rclone") + let mut binding = Command::new("rclone"); + let command = binding .arg("--config") .arg(format!("{}", self.config_path.to_string_lossy())) + .arg("--contimeout") + .arg("5s") + // HTTP request is a typical low level operation, which will retry for many times + // https://rclone.org/docs/#low-level-retries-number + // Here we set it to 3 to avoid long waiting time + .arg("--low-level-retries") + .arg("3") .arg("ls") - .arg("minio:") - .status() - .expect("Failed to rclone"); + .arg("minio:"); + log::debug!("Checking rclone with command: {:?}", command); + let status = command.status().expect("Failed to rclone"); status.success() } diff --git a/judger/src/main.rs b/judger/src/main.rs index 6a87066..72434b8 100644 --- a/judger/src/main.rs +++ b/judger/src/main.rs @@ -1,7 +1,7 @@ mod agent; -mod env; mod error; mod handler; +mod option; mod worker; #[macro_use] @@ -12,16 +12,17 @@ use actix_web::{web::Data, App, HttpServer}; use worker::JudgeWorker; #[actix_web::main] // or #[tokio::main] +// The `Run` button provided by rust-analyzer will not work here +// Use `Debug` button instead async fn main() -> std::io::Result<()> { - let opt = env::load_option(); - env::setup_logger(); + let opt = option::load_option(); // TODO: Send heartbeat here to a remote host let worker = match JudgeWorker::new( opt.platform_uri, opt.fetch_task_interval, - opt.rclone_config, + opt.rclone_config_path, opt.problem_package_bucket.clone(), opt.problem_package_dir.clone(), ) { diff --git a/judger/src/env/mod.rs b/judger/src/option.rs similarity index 62% rename from judger/src/env/mod.rs rename to judger/src/option.rs index 99db343..ec2de01 100644 --- a/judger/src/env/mod.rs +++ b/judger/src/option.rs @@ -6,20 +6,21 @@ use structopt::StructOpt; #[structopt(name = "judger")] pub struct JudgeServerOpt { /// For loading Opt from .env file - #[structopt(long)] - pub env_path: Option, + #[structopt(long, default_value = ".env")] + pub env_path: PathBuf, /// Port to listen to - #[structopt(env = "PORT", default_value = "8080")] + #[structopt(env = "PORT", default_value = "8000")] pub port: u16, // TODO: make rclone optional - #[structopt(long, default_value = "data/default-rclone.conf")] - pub rclone_config: PathBuf, + #[structopt(env = "RCLONE_CONFIG_PATH", default_value = "rclone.conf")] + pub rclone_config_path: PathBuf, #[structopt(long, default_value = "oj-lab-problem-package")] pub problem_package_bucket: String, + /// Where to store problem package - #[structopt(long, default_value = "data/problem-package")] + #[structopt(long, default_value = "problem-package")] pub problem_package_dir: PathBuf, #[structopt(env = "PLATFORM_URI", default_value = "http://localhost:8080/")] @@ -29,21 +30,27 @@ pub struct JudgeServerOpt { pub fetch_task_interval: u64, } +/// Try to load env from a .env file, if not found, fallback to ENV pub fn load_option() -> JudgeServerOpt { + println!("PWD: {:?}", std::env::current_dir().unwrap()); // First load env_path from Args let opt = JudgeServerOpt::from_args(); - if let Some(env_path) = opt.env_path { - dotenv::from_path(env_path).ok(); + if opt.env_path.exists() { + println!("loading env from file: {:?}", opt.env_path); + dotenv::from_path(opt.env_path).ok(); } else { + println!("loading env from ENV"); dotenv::dotenv().ok(); } + setup_logger(); + // Load opt again with ENV let opt = JudgeServerOpt::from_args(); log::debug!("load opt: {:?}", opt); opt } -pub fn setup_logger() { +fn setup_logger() { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("debug")).init(); } diff --git a/judger/src/worker/mod.rs b/judger/src/worker/mod.rs index 4f2d6a4..c84cb25 100644 --- a/judger/src/worker/mod.rs +++ b/judger/src/worker/mod.rs @@ -46,6 +46,7 @@ impl JudgeWorker { } pub async fn run(&self) { + log::info!("judge task worker started"); let _ = self .rclone_client .sync_bucket(&self.package_bucket, &self.package_dir)