Refactor project structure

.gitignore

@@ -11,14 +11,6 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
-# Dependency directories (remove the comment below to include it)
-# vendor/
-
-*.ini
-!package/configs/ini/example.ini
-!package/configs/ini/test.ini
-
-artifacts/
 dist.zip
 
 # Ignore generated files
@@ -27,3 +19,5 @@ dist.zip
 
 bin/
 __debug_bin**
+
+override.toml

.vscode/extensions.json

@@ -4,6 +4,6 @@
     "ms-azuretools.vscode-docker",
     "tamasfe.even-better-toml",
     "pkief.material-icon-theme",
-    "bierner.markdown-preview-github-styles",
+    "davidanson.vscode-markdownlint"
   ]
 }

.vscode/settings.json

@@ -1,13 +1,7 @@
 {
   "material-icon-theme.folders.associations": {
-    "application": "core",
-    "mapper": "database",
-    "migrate_db": "import",
-    "service": "components",
-    "business": "hook",
-    "postman": "api",
-    ".github/workflows": "github",
-    "test-collection": "resource"
+    "problem_packages": "packages",
+    "modules/agent": "client"
   },
   "go.testFlags": [
     "-v",

Dockerfile

@@ -1,23 +1,27 @@
 FROM golang:latest as build
 
-COPY . /oj-lab-platform-build
+WORKDIR /workdir
+
+COPY . /workdir
 
-WORKDIR /oj-lab-platform-build
 RUN apt update && apt install -y make zip curl
 RUN make build
-RUN ./scripts/update-frontend-dist.sh /oj-lab-platform-build/frontend_dist
+RUN make get-front
 
 
 FROM ubuntu:latest
 
-WORKDIR /workdir
+WORKDIR /oj-lab-platform
+
+COPY --from=build /workdir/bin/web_server /usr/local/bin/web_server
+COPY --from=build /workdir/frontend/dist ./frontend_dist
 
-COPY --from=build /oj-lab-platform-build/bin/web_server /usr/local/bin/web_server
-COPY --from=build /oj-lab-platform-build/frontend_dist /workdir/frontend_dist
+COPY config.toml ./config.toml
 
-COPY workdirs/docker/config.toml /workdir/config.toml
+ENV OJ_LAB_SERVICE_ENV='production'
+ENV DATABASE_DSN='user=postgres password=postgres host=host.docker.internal port=5432 dbname=oj_lab sslmode=disable TimeZone=Asia/Shanghai'
+ENV REDIS_HOSTS='["host.docker.internal:6379"]'
+ENV MINIO_ENDPOINT='http://host.docker.internal:9000'
 
-ENV OJ_LAB_SERVICE_ENV=production
-ENV OJ_LAB_WORKDIR=/workdir
 EXPOSE 8080
 CMD [ "web_server" ]

Makefile

@@ -1,33 +1,33 @@
 OS := $(shell uname -s)
 
-DEV_WORKDIR := workdirs/development
-DB_DOCKER_COMPOSE_FILE := $(DEV_WORKDIR)/docker-compose.yml
-JUDGER_DOCKER_COMPOSE_FILE := $(DEV_WORKDIR)/judger/docker-compose.yml
-FRONTEND_DIST_DIR := $(DEV_WORKDIR)/frontend_dist
+FRONTEND_DIST_DIR := frontend/dist
+FRONTEND_DIST_URL := https://github.com/oj-lab/oj-lab-front/releases/download/v0.0.3/dist.zip
+ICPC_PROBLEM_PACKAGES_DIR := problem_packages/icpc
+ICPC_PROBLEM_PACKAGES_URL := https://github.com/oj-lab/problem-packages/releases/download/v0.0.1/icpc_problem.zip
 
 .PHONY: help
 help:
 	@echo "Usage: make [target]"
 	@echo ""
 	@echo "Targets:"
-	@echo "  build     		- Build the application, swagger document will be generated"
-	@echo "  run       		- Run the application"
-	@echo "  clean    		- Clean the build"
-	@echo "  check     		- Run go vet"
-	@echo "  test      		- Run tests, database will be setup"
-	@echo "  gen-swagger 		- Generate swagger document"
-	@echo "  setup-dependencies  	- Setup the dependencies docker image"
-	@echo "  unset-dependencies 	- Unset the dependencies docker image"
-	@echo "  get-front 		- Get the frontend files"
+	@echo "  build     			- Build the application, swagger document will be generated"
+	@echo "  run       			- Run the application"
+	@echo "  clean    			- Clean the build"
+	@echo "  check     			- Run go vet"
+	@echo "  test      			- Run tests, database will be setup"
+	@echo "  gen-swagger 			- Generate swagger document"
+	@echo "  setup-dependencies  		- Setup the dependencies docker image"
+	@echo "  unset-dependencies 		- Unset the dependencies docker image"
+	@echo "  get-front 			- Get the frontend files"
+	@echo "  update-front 			- Update the frontend files"
+	@echo "  get-problem-packages 		- Get the problem packages"
+	@echo "  update-problem-packages 	- Update the problem packages"
 
 .PHONY: build
 build: gen-swagger gen-proto
 	@echo "Building on $(OS)"
 	go mod tidy
-	go build -o bin/init_db cmd/init_db/main.go
-	go build -o bin/web_server cmd/web_server/main.go
-	go build -o bin/schedule cmd/schedule/main.go
-	go build -o bin/problem_loader cmd/problem_loader/main.go
+	go build -o bin/ ./cmd/...
 
 .PHONY: run
 run: build
@@ -56,23 +56,35 @@ gen-proto: install-proto
 
 .PHONY: unset-dependencies
 unset-dependencies:
-	docker compose -f $(JUDGER_DOCKER_COMPOSE_FILE) -p oj-lab-judger stop
-	docker compose -f $(JUDGER_DOCKER_COMPOSE_FILE) -p oj-lab-judger rm -f
-	docker compose -f $(DB_DOCKER_COMPOSE_FILE) -p oj-lab-dbs stop
-	docker compose -f $(DB_DOCKER_COMPOSE_FILE) -p oj-lab-dbs rm -f
+	docker compose stop
+	docker compose rm -f
 
 .PHONY: setup-dependencies
-setup-dependencies: unset-dependencies build
-	docker compose -f $(DB_DOCKER_COMPOSE_FILE) -p oj-lab-dbs up -d
+setup-dependencies: unset-dependencies build get-front get-problem-packages
+	docker compose up -d postgres redis minio clickhouse adminer
 	@echo "Wait 10 seconds for db setup"
 	sleep 10s
-	./bin/init_db
-	./bin/problem_loader
-	docker compose -f $(JUDGER_DOCKER_COMPOSE_FILE) -p oj-lab-judger up -d
+	./bin/init
 
 .PHONY: get-front
 get-front:
-	./scripts/update-frontend-dist.sh $(FRONTEND_DIST_DIR)
+	./scripts/download_and_unzip.sh $(FRONTEND_DIST_DIR) $(FRONTEND_DIST_URL) \
+		OVERRIDE=false
+
+.PHONY: update-front
+update-front:
+	./scripts/download_and_unzip.sh $(FRONTEND_DIST_DIR) $(FRONTEND_DIST_URL) \
+		OVERRIDE=true
+
+.PHONY: get-problem-packages
+get-problem-packages:
+	./scripts/download_and_unzip.sh $(ICPC_PROBLEM_PACKAGES_DIR) $(ICPC_PROBLEM_PACKAGES_URL) \
+		OVERRIDE=false
+
+.PHONY: update-problem-packages
+update-problem-packages:
+	./scripts/download_and_unzip.sh $(ICPC_PROBLEM_PACKAGES_DIR) $(ICPC_PROBLEM_PACKAGES_URL) \
+		OVERRIDE=true
 
 .PHONY: check
 check: gen-proto install-cilint
@@ -101,7 +113,7 @@ install-cilint:
 .PHONY: install-proto
 install-proto:
 	@# Referencing https://grpc.io/docs/protoc-installation/
-	@./scripts/install-protoc.sh
+	@./scripts/install_protoc.sh
 	@# Track https://grpc.io/docs/languages/go/quickstart/ for update
 	go install google.golang.org/protobuf/cmd/[email protected]
 	go install google.golang.org/grpc/cmd/[email protected]

README.md

@@ -11,10 +11,11 @@
 
 OJ Lab Platform depends on several foundational services, including:
 
-- PostgreSQL (or other SQL database in the future)
-- Redis
-- MinIO
-- [Judger](https://github.com/oj-lab/judger)
+- PostgreSQL (or other SQL database in the future) for data storage
+- Redis for caching & session management
+- MinIO (or other S3 like storage) for file storage
+- ClickHouse for analytics (currently not developed)
+- [Judger](https://github.com/oj-lab/judger) for judging
 
 This project provides a Makefile to help you quickly set up dependencies & other optional choices.
 Run `make setup-dependencies` to start these services and load the initial data.
@@ -24,31 +25,18 @@ Run `make setup-dependencies` to start these services and load the initial data.
 Launch the programs with VSCode launch configurations is the most recommended way.
 It will automatically set the environment and run the program in debug mode.
 
-### Optional
+### Run Judger
 
-There is also some optional approach you may want to use.
+There is a `judger` service in the project's `docker-compose.yml`.
+It won't start from the `make setup-dependencies` command by default
+(since it takes time to let MinIO & PostgreSQL start up).
 
-#### Serve frontend
-
-Use `make get-front` to get the frontend dist codes.
-> In development config by default, it points to the postion where the frontend codes are located,
-> so you will automatically get the frontend view when you start the program.
-
-#### Generate Swagger docs
-
-Use `make gen-swagger` to generate swagger docs.
-
-#### Set environment variables
-
-The following environment variables are available to modify the behavior of the program:
-
-- OJ_LAB_SERVICE_ENV: The environment of the service, default to `development`
-- OJ_LAB_WORKDIR: Directly set the path of the workdir, application will automatically locate the workdir if not set
-(it will be set to `workdirs/<service_env>` in this project by default)
+Run `docker-compose up -d judger` to start the judger service.
 
 #### Manage DB data
 
-Along with the `make setup-dependencies`, we provide `adminer` to access PostgreSQL & MinIO with its web interface.
+Along with the `make setup-dependencies`,
+we provide `adminer` to access PostgreSQL & MinIO with its web interface.
 
 > Remember to set the type of the database to `PostgreSQL` when login to adminer.
 

cmd/init_db/main.go → cmd/init/db.go

@@ -6,12 +6,11 @@ import (
 	judge_model "github.com/oj-lab/oj-lab-platform/models/judge"
 	problem_model "github.com/oj-lab/oj-lab-platform/models/problem"
 	user_model "github.com/oj-lab/oj-lab-platform/models/user"
-	casbin_agent "github.com/oj-lab/oj-lab-platform/modules/agent/casbin"
 	gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm"
-	"github.com/oj-lab/oj-lab-platform/modules/log"
+	log_module "github.com/oj-lab/oj-lab-platform/modules/log"
 )
 
-func main() {
+func initDB() {
 	db := gorm_agent.GetDefaultDB()
 	err := db.AutoMigrate(
 		&user_model.User{},
@@ -58,10 +57,5 @@ func main() {
 	if err != nil {
 		panic("failed to create anonymous user")
 	}
-	log.AppLogger().Info("migrate tables ans users success")
-
-	err = casbin_agent.LoadDefaultCasbinPolicies()
-	if err != nil {
-		panic(fmt.Sprintf("failed to load default casbin policies: %v", err))
-	}
+	log_module.AppLogger().Info("migrate tables ans users success")
 }

cmd/init/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	casbin_agent "github.com/oj-lab/oj-lab-platform/modules/agent/casbin"
+)
+
+func main() {
+	ctx := context.Background()
+	initDB()
+	loadProblemPackages(ctx)
+	err := casbin_agent.LoadDefaultCasbinPolicies()
+	if err != nil {
+		panic(fmt.Sprintf("failed to load default casbin policies: %v", err))
+	}
+}

cmd/problem_loader/main.go → cmd/init/problem_package.go

@@ -13,14 +13,12 @@ import (
 	problem_model "github.com/oj-lab/oj-lab-platform/models/problem"
 	gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm"
 	minio_agent "github.com/oj-lab/oj-lab-platform/modules/agent/minio"
-	"github.com/oj-lab/oj-lab-platform/modules/config"
-	"github.com/oj-lab/oj-lab-platform/modules/log"
-	yaml "gopkg.in/yaml.v2"
+	config_module "github.com/oj-lab/oj-lab-platform/modules/config"
+	log_module "github.com/oj-lab/oj-lab-platform/modules/log"
+	"gopkg.in/yaml.v2"
 )
 
-var ctx = context.Background()
-
-func main() {
+func loadProblemPackages(ctx context.Context) {
 	db := gorm_agent.GetDefaultDB()
 	minioClient := minio_agent.GetMinioClient()
 
@@ -31,13 +29,13 @@ func main() {
 	//    parse problem.md as description.
 	// 2. insert object into minio storage.
 	var (
-		packagePath string = path.Join(config.Workdir, "problem_packages")
+		packagePath string = path.Join(config_module.ProjectRoot(), "problem_packages/icpc")
 		title       string
 		slug        string
 	)
 	err := filepath.Walk(packagePath, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
-			log.AppLogger().WithError(err).Error("Walk package path failed")
+			log_module.AppLogger().WithError(err).Error("Walk package path failed")
 			return err
 		}
 		if info == nil {
@@ -47,31 +45,35 @@ func main() {
 			return nil
 		}
 		relativePath := strings.Replace(path, packagePath, "", 1)
-		log.AppLogger().WithField("relativePath", relativePath).Debug("Read file from package")
+		log_module.AppLogger().WithField("relativePath", relativePath).Debug("Read file from package")
 		if filepath.Base(relativePath) == "problem.yaml" {
 			resultMap := make(map[string]interface{})
 			yamlFile, err := os.ReadFile(path)
 			if err != nil {
-				log.AppLogger().WithError(err).Error("Read problem.yaml failed")
+				log_module.AppLogger().WithError(err).Error("Read problem.yaml failed")
 			}
 			err = yaml.Unmarshal(yamlFile, &resultMap)
 			if err != nil {
-				log.AppLogger().WithError(err).Error("Unmarshal problem.yaml failed")
+				log_module.AppLogger().WithError(err).Error("Unmarshal problem.yaml failed")
+			}
+			if resultMap["name"] == nil {
+				log_module.AppLogger().Error("Problem name is nil")
+				return nil
 			}
 			title = resultMap["name"].(string)
 			if title == "" {
-				log.AppLogger().Error("Problem title is empty")
+				log_module.AppLogger().Error("Problem title is empty")
 			}
 			slug = strings.Split(relativePath, "/")[1]
-			log.AppLogger().WithField("title", title).WithField("slug", slug).Debug("Read problem.yaml")
+			log_module.AppLogger().WithField("title", title).WithField("slug", slug).Debug("Read problem.yaml")
 		}
 		if filepath.Base(relativePath) == "problem.md" {
 			content, err := os.ReadFile(path)
 			if err != nil {
-				log.AppLogger().WithError(err).Error("Read problem.md failed")
+				log_module.AppLogger().WithError(err).Error("Read problem.md failed")
 			}
 			description := string(content)
-			log.AppLogger().WithField("description", description).Debug("Read problem.md")
+			log_module.AppLogger().WithField("description", description).Debug("Read problem.md")
 			err = problem_model.CreateProblem(db, problem_model.Problem{
 				Slug:        slug,
 				Title:       title,
@@ -90,13 +92,13 @@ func main() {
 			path,
 			minio.PutObjectOptions{})
 		if err != nil {
-			log.AppLogger().WithError(err).Error("Put object to minio failed")
+			log_module.AppLogger().WithError(err).Error("Put object to minio failed")
 		}
 		return err
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	log.AppLogger().Info("Problem loaded")
+	log_module.AppLogger().Info("Problem loaded")
 }