Perf environment setup and prepare rclone usage (#137)

* perf env_setup script with rclone installed

* Add rclone environment setup & perf dev environment

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-    "image": "mcr.microsoft.com/devcontainers/universal:2",
+    "image": "mcr.microsoft.com/devcontainers/rust:1",
     "postCreateCommand": "${containerWorkspaceFolder}/scripts/env_setup.bash",
     "customizations": {
         "vscode": {

.gitignore

@@ -1,5 +1,3 @@
 target/
-tmp/
-!tmp/.gitkeep
 Cargo.lock
-scripts/rustup.sh
+scripts/thirdparty/

data/.gitignore

@@ -0,0 +1,2 @@
+rclone-minio.conf
+rclone-problem-package/

data/README.md

@@ -0,0 +1,47 @@
+# Judger Data
+
+In this section we provide two approaches to connect problem packages with judger.
+
+1. By accessing a local directory, 
+which is more convenient when you can't not access OJ Lab's platform data collection.
+(Ex. you are using github codespaces or some other cloud IDE)
+2. By setting up a connection by rclone,
+this approach will be closer to the real situation.
+
+## Configure rclone-minio.conf
+
+Copy the example file and then edit in your own configuration.
+
+```sh
+cp -i data/rclone-minio.conf.example data/rclone-minio.conf
+```
+
+You will need to replace `YOUR_ACCESS_KEY` and `YOUR_SECRET_KEY`
+with your own access key and secret key
+(which can be generated in the minio web interface, 
+visit http://127.0.0.1:9001 if you are using oj-lab-platform's docker-compose).
+
+Then you can run the following command to test if the configuration is correct.
+
+```sh
+rclone --config data/rclone-minio.conf ls minio:
+```
+
+You should notice that the `minio:` is the name of the remote,
+remember to replace it with your own remote name if you change it.
+
+### Use rclone sync
+
+By design judger will try sync the remote problem package
+everytime when it got a judge task,
+so that we can ensure the problem package is up-to-date.
+
+```sh
+rclone --config data/rclone-minio.conf sync minio:oj-lab-problem-package data/rclone-problem-package
+```
+
+## gitignored files
+
+Some files are gitignored in this directory.
+If you follow this README to setup rclone problem package,
+this will help keeping your repo tidy.

data/rclone-minio.conf.example

@@ -0,0 +1,9 @@
+[minio]
+type = s3
+provider = Minio
+env_auth = false
+access_key_id = YOUR_ACCESS_KEY
+secret_access_key = YOUR_SECRET_KEY
+endpoint = http://127.0.0.1:9000
+location_constraint =
+acl = private

docker/judger-server.dockerfile

@@ -14,7 +14,7 @@ RUN apt update && apt install -y libseccomp-dev gcc g++
 COPY --from=build /usr/src/judger/target/release/judger-server /usr/local/bin/judger-server
 RUN mkdir /workspace
 WORKDIR /workspace
-COPY dev-problem-package /workspace/dev-problem-package
+COPY data/dev-problem-package /workspace/data/dev-problem-package
 
 ENV RUST_LOG=DEBUG
 EXPOSE 8000

judger/src/client/main.rs

@@ -122,7 +122,7 @@ async fn report_task(
 }
 
 fn run_judge(task: JudgeTask) -> Result<Vec<JudgeResultInfo>, ClientError> {
-    let problem_package_dir = PathBuf::from("dev-problem-package");
+    let problem_package_dir = PathBuf::from("data/dev-problem-package");
     let problem_slug = task.problem_slug;
     let uuid = uuid::Uuid::new_v4();
     let runtime_path = PathBuf::from("/tmp").join(uuid.to_string());

judger/src/server/environment/mod.rs

@@ -12,7 +12,7 @@ pub struct JudgeServerOpt {
     #[structopt(env = "PORT", default_value = "8080")]
     pub port: u16,
 
-    #[structopt(long, default_value = "dev-problem-package")]
+    #[structopt(long, default_value = "data/dev-problem-package")]
     pub problem_package_dir: PathBuf,
 }
 

scripts/env_setup.bash

@@ -1,15 +1,44 @@
 #!/bin/bash
 
-UBUNTU_FLAG="ubuntu"
+if [ -x "$(command -v apt)" ]; then
+    # if recently updated jump to installing dependencies
+    if [ "$(find /var/cache/apt/pkgcache.bin -mmin -60)" ]; then
+        echo 'Skipping apt update...'
+    else
+        echo 'Updating apt...'
+        sudo apt update
+    fi
+    sudo apt install -y libseccomp-dev gcc curl pkg-config libssl-dev cmake
+fi
 
-CUR_SYSTEM=$(uname -a | tr '[:upper:]' '[:lower:]')
+if [ ! -d "scripts/thirdparty" ]; then
+    mkdir scripts/thirdparty
+fi
 
-if [[ $CUR_SYSTEM =~ $UBUNTU_FLAG ]];then
-    echo Current system is Ubuntu
-    sudo apt update
-    sudo apt install -y libseccomp-dev gcc
+echo 'Ensuring rustup is installed...'
+if ! [ -x "$(command -v rustup)" ]; then
+    echo 'rustup not found. Installing rustup...'
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > scripts/thirdparty/rustup.sh
+    chmod +x scripts/thirdparty/rustup.sh
+    sudo scripts/thirdparty/rustup.sh -y
 fi
 
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> scripts/rustup.sh
-chmod +x scripts/rustup.sh
-./scripts/rustup.sh -y
+echo 'Ensuring rclone is installed...'
+if ! [ -x "$(command -v rclone)" ]; then
+    echo 'rclone not found. Installing rclone...'
+    curl -sSf https://rclone.org/install.sh > scripts/thirdparty/rclone_install.sh
+    # Rclone install script can take a fairly long time to download it's install package
+    # So showing the download progress can be very helpful
+    echo 'Adjusting rclone install script to show download progress...'
+    sed -i 's/-OfsS/-OfS/g' scripts/thirdparty/rclone_install.sh
+    chmod +x scripts/thirdparty/rclone_install.sh
+    echo 'If it is taking too long to download the rclone install package, try manually install it'
+    sudo scripts/thirdparty/rclone_install.sh
+fi
+
+echo 'Compiling built-in programs for judge-core testing...'
+PWD=$(pwd)
+cd judge-core/tests/data/built-in-programs && ./build.sh
+cd "$PWD" || exit
+
+echo 'Environment setup complete.'