From a96f4975a862463c332c15762d3294bf81700df6 Mon Sep 17 00:00:00 2001 From: slhmy Date: Mon, 22 Jan 2024 16:43:59 +0800 Subject: [PATCH] Perf environment setup and prepare rclone usage (#137) * perf env_setup script with rclone installed * Add rclone environment setup & perf dev environment --- .devcontainer/devcontainer.json | 2 +- .gitignore | 4 +- data/.gitignore | 2 + data/README.md | 47 +++++++++++++++++++ .../hello-world/.timelimit | 0 .../hello-world/data/secret/0.ans | 0 .../hello-world/data/secret/0.in | 0 .../hello-world/data/secret/1.ans | 0 .../hello-world/data/secret/1.in | 0 .../hello-world/problem.yaml | 0 data/rclone-minio.conf.example | 9 ++++ docker/judger-server.dockerfile | 2 +- judger/src/client/main.rs | 2 +- judger/src/server/environment/mod.rs | 2 +- scripts/env_setup.bash | 47 +++++++++++++++---- 15 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 data/.gitignore create mode 100644 data/README.md rename {dev-problem-package => data/dev-problem-package}/hello-world/.timelimit (100%) rename {dev-problem-package => data/dev-problem-package}/hello-world/data/secret/0.ans (100%) rename {dev-problem-package => data/dev-problem-package}/hello-world/data/secret/0.in (100%) rename {dev-problem-package => data/dev-problem-package}/hello-world/data/secret/1.ans (100%) rename {dev-problem-package => data/dev-problem-package}/hello-world/data/secret/1.in (100%) rename {dev-problem-package => data/dev-problem-package}/hello-world/problem.yaml (100%) create mode 100644 data/rclone-minio.conf.example diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f73d386..ab57b4a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "mcr.microsoft.com/devcontainers/universal:2", + "image": "mcr.microsoft.com/devcontainers/rust:1", "postCreateCommand": "${containerWorkspaceFolder}/scripts/env_setup.bash", "customizations": { "vscode": { diff --git a/.gitignore b/.gitignore index d4d3ffe..0505f0a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ target/ -tmp/ -!tmp/.gitkeep Cargo.lock -scripts/rustup.sh \ No newline at end of file +scripts/thirdparty/ diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..28ceaa7 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,2 @@ +rclone-minio.conf +rclone-problem-package/ diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..1d53c1d --- /dev/null +++ b/data/README.md @@ -0,0 +1,47 @@ +# Judger Data + +In this section we provide two approaches to connect problem packages with judger. + +1. By accessing a local directory, +which is more convenient when you can't not access OJ Lab's platform data collection. +(Ex. you are using github codespaces or some other cloud IDE) +2. By setting up a connection by rclone, +this approach will be closer to the real situation. + +## Configure rclone-minio.conf + +Copy the example file and then edit in your own configuration. + +```sh +cp -i data/rclone-minio.conf.example data/rclone-minio.conf +``` + +You will need to replace `YOUR_ACCESS_KEY` and `YOUR_SECRET_KEY` +with your own access key and secret key +(which can be generated in the minio web interface, +visit http://127.0.0.1:9001 if you are using oj-lab-platform's docker-compose). + +Then you can run the following command to test if the configuration is correct. + +```sh +rclone --config data/rclone-minio.conf ls minio: +``` + +You should notice that the `minio:` is the name of the remote, +remember to replace it with your own remote name if you change it. + +### Use rclone sync + +By design judger will try sync the remote problem package +everytime when it got a judge task, +so that we can ensure the problem package is up-to-date. + +```sh +rclone --config data/rclone-minio.conf sync minio:oj-lab-problem-package data/rclone-problem-package +``` + +## gitignored files + +Some files are gitignored in this directory. +If you follow this README to setup rclone problem package, +this will help keeping your repo tidy. diff --git a/dev-problem-package/hello-world/.timelimit b/data/dev-problem-package/hello-world/.timelimit similarity index 100% rename from dev-problem-package/hello-world/.timelimit rename to data/dev-problem-package/hello-world/.timelimit diff --git a/dev-problem-package/hello-world/data/secret/0.ans b/data/dev-problem-package/hello-world/data/secret/0.ans similarity index 100% rename from dev-problem-package/hello-world/data/secret/0.ans rename to data/dev-problem-package/hello-world/data/secret/0.ans diff --git a/dev-problem-package/hello-world/data/secret/0.in b/data/dev-problem-package/hello-world/data/secret/0.in similarity index 100% rename from dev-problem-package/hello-world/data/secret/0.in rename to data/dev-problem-package/hello-world/data/secret/0.in diff --git a/dev-problem-package/hello-world/data/secret/1.ans b/data/dev-problem-package/hello-world/data/secret/1.ans similarity index 100% rename from dev-problem-package/hello-world/data/secret/1.ans rename to data/dev-problem-package/hello-world/data/secret/1.ans diff --git a/dev-problem-package/hello-world/data/secret/1.in b/data/dev-problem-package/hello-world/data/secret/1.in similarity index 100% rename from dev-problem-package/hello-world/data/secret/1.in rename to data/dev-problem-package/hello-world/data/secret/1.in diff --git a/dev-problem-package/hello-world/problem.yaml b/data/dev-problem-package/hello-world/problem.yaml similarity index 100% rename from dev-problem-package/hello-world/problem.yaml rename to data/dev-problem-package/hello-world/problem.yaml diff --git a/data/rclone-minio.conf.example b/data/rclone-minio.conf.example new file mode 100644 index 0000000..8f688fc --- /dev/null +++ b/data/rclone-minio.conf.example @@ -0,0 +1,9 @@ +[minio] +type = s3 +provider = Minio +env_auth = false +access_key_id = YOUR_ACCESS_KEY +secret_access_key = YOUR_SECRET_KEY +endpoint = http://127.0.0.1:9000 +location_constraint = +acl = private \ No newline at end of file diff --git a/docker/judger-server.dockerfile b/docker/judger-server.dockerfile index 6abb773..9c5de39 100644 --- a/docker/judger-server.dockerfile +++ b/docker/judger-server.dockerfile @@ -14,7 +14,7 @@ RUN apt update && apt install -y libseccomp-dev gcc g++ COPY --from=build /usr/src/judger/target/release/judger-server /usr/local/bin/judger-server RUN mkdir /workspace WORKDIR /workspace -COPY dev-problem-package /workspace/dev-problem-package +COPY data/dev-problem-package /workspace/data/dev-problem-package ENV RUST_LOG=DEBUG EXPOSE 8000 diff --git a/judger/src/client/main.rs b/judger/src/client/main.rs index 19d2fb7..8e5553f 100644 --- a/judger/src/client/main.rs +++ b/judger/src/client/main.rs @@ -122,7 +122,7 @@ async fn report_task( } fn run_judge(task: JudgeTask) -> Result, ClientError> { - let problem_package_dir = PathBuf::from("dev-problem-package"); + let problem_package_dir = PathBuf::from("data/dev-problem-package"); let problem_slug = task.problem_slug; let uuid = uuid::Uuid::new_v4(); let runtime_path = PathBuf::from("/tmp").join(uuid.to_string()); diff --git a/judger/src/server/environment/mod.rs b/judger/src/server/environment/mod.rs index 91b9255..22a13ea 100644 --- a/judger/src/server/environment/mod.rs +++ b/judger/src/server/environment/mod.rs @@ -12,7 +12,7 @@ pub struct JudgeServerOpt { #[structopt(env = "PORT", default_value = "8080")] pub port: u16, - #[structopt(long, default_value = "dev-problem-package")] + #[structopt(long, default_value = "data/dev-problem-package")] pub problem_package_dir: PathBuf, } diff --git a/scripts/env_setup.bash b/scripts/env_setup.bash index 0c739e6..6a15a86 100755 --- a/scripts/env_setup.bash +++ b/scripts/env_setup.bash @@ -1,15 +1,44 @@ #!/bin/bash -UBUNTU_FLAG="ubuntu" +if [ -x "$(command -v apt)" ]; then + # if recently updated jump to installing dependencies + if [ "$(find /var/cache/apt/pkgcache.bin -mmin -60)" ]; then + echo 'Skipping apt update...' + else + echo 'Updating apt...' + sudo apt update + fi + sudo apt install -y libseccomp-dev gcc curl pkg-config libssl-dev cmake +fi -CUR_SYSTEM=$(uname -a | tr '[:upper:]' '[:lower:]') +if [ ! -d "scripts/thirdparty" ]; then + mkdir scripts/thirdparty +fi -if [[ $CUR_SYSTEM =~ $UBUNTU_FLAG ]];then - echo Current system is Ubuntu - sudo apt update - sudo apt install -y libseccomp-dev gcc +echo 'Ensuring rustup is installed...' +if ! [ -x "$(command -v rustup)" ]; then + echo 'rustup not found. Installing rustup...' + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > scripts/thirdparty/rustup.sh + chmod +x scripts/thirdparty/rustup.sh + sudo scripts/thirdparty/rustup.sh -y fi -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> scripts/rustup.sh -chmod +x scripts/rustup.sh -./scripts/rustup.sh -y \ No newline at end of file +echo 'Ensuring rclone is installed...' +if ! [ -x "$(command -v rclone)" ]; then + echo 'rclone not found. Installing rclone...' + curl -sSf https://rclone.org/install.sh > scripts/thirdparty/rclone_install.sh + # Rclone install script can take a fairly long time to download it's install package + # So showing the download progress can be very helpful + echo 'Adjusting rclone install script to show download progress...' + sed -i 's/-OfsS/-OfS/g' scripts/thirdparty/rclone_install.sh + chmod +x scripts/thirdparty/rclone_install.sh + echo 'If it is taking too long to download the rclone install package, try manually install it' + sudo scripts/thirdparty/rclone_install.sh +fi + +echo 'Compiling built-in programs for judge-core testing...' +PWD=$(pwd) +cd judge-core/tests/data/built-in-programs && ./build.sh +cd "$PWD" || exit + +echo 'Environment setup complete.' \ No newline at end of file