From 12d66a48a0025b5cf36a274604e46d6a9a3b67cc Mon Sep 17 00:00:00 2001 From: slhmy <1484836413@qq.com> Date: Tue, 11 Jun 2024 09:18:23 +0800 Subject: [PATCH] Refactor code --- .gitignore | 2 +- Makefile | 6 +- cmd/init_db/main.go | 8 +- cmd/problem_loader/main.go | 39 +++--- cmd/web_server/handler/judge.go | 92 +++++++++------ cmd/web_server/handler/judge_task.go | 65 ++++++++++ cmd/web_server/handler/problem.go | 36 +++--- cmd/web_server/handler/submission.go | 97 --------------- cmd/web_server/handler/swaggo.go | 2 +- cmd/web_server/main.go | 2 +- go.mod | 2 +- models/judge/judge.go | 138 ++++++++-------------- models/judge/judge_db.go | 129 ++++++++++++++++++++ models/judge/judge_result.go | 33 ++++++ models/judge/judge_result_db.go | 14 +++ models/judge/judge_task.go | 27 +++++ models/judge/judge_task_stream.go | 78 ++++++++++++ models/judge/submission.go | 69 ----------- models/judge/submission_db.go | 123 ------------------- models/problem/problem.go | 2 +- models/user/user_db.go | 1 - modules/agent/gorm/database.go | 6 +- modules/agent/gorm/logger.go | 24 ++++ modules/agent/judger/client.go | 2 +- modules/agent/judger/judge.go | 2 +- modules/agent/judger/state.go | 2 +- modules/agent/minio/client.go | 21 ++-- modules/agent/minio/local.go | 2 +- modules/agent/redis/client.go | 2 +- modules/auth/redis.go | 8 +- modules/error.go | 7 +- modules/log/log.go | 4 +- services/judge/judge.go | 54 +++++++++ services/judge/judge_stream.go | 79 ------------- services/judge/{task.go => judge_task.go} | 19 ++- services/judge/submission.go | 54 --------- services/problem/problem.go | 10 +- services/problem/problem_info.go | 4 +- services/problem/problem_package.go | 4 +- services/user/user.go | 5 +- tests/models/judge_test.go | 29 +++++ tests/models/problem_test.go | 7 +- tests/models/user_test.go | 7 +- 43 files changed, 665 insertions(+), 652 deletions(-) create mode 100644 cmd/web_server/handler/judge_task.go delete mode 100644 cmd/web_server/handler/submission.go create mode 100644 models/judge/judge_db.go create mode 100644 models/judge/judge_result.go create mode 100644 models/judge/judge_result_db.go create mode 100644 models/judge/judge_task.go create mode 100644 models/judge/judge_task_stream.go delete mode 100644 models/judge/submission.go delete mode 100644 models/judge/submission_db.go create mode 100644 modules/agent/gorm/logger.go create mode 100644 services/judge/judge.go delete mode 100644 services/judge/judge_stream.go rename services/judge/{task.go => judge_task.go} (86%) delete mode 100644 services/judge/submission.go create mode 100644 tests/models/judge_test.go diff --git a/.gitignore b/.gitignore index 4590a00..ce1ee4c 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ dist.zip # Ignore generated files *.pb.go -**/swaggo-gen/docs.go +**/swaggo_gen/docs.go bin/ __debug_bin** diff --git a/Makefile b/Makefile index 532f3bd..c41f1db 100644 --- a/Makefile +++ b/Makefile @@ -36,12 +36,12 @@ run: build .PHONY: clean clean: rm -rf bin - rm -rf cmd/web_server/swaggo-gen + rm -rf cmd/web_server/swaggo_gen .PHONY: gen-swagger gen-swagger: install-swaggo swag fmt -d cmd/web_server - swag init -d cmd/web_server,models -ot go -o cmd/web_server/swaggo-gen + swag init -d cmd/web_server,models -ot go -o cmd/web_server/swaggo_gen # Deprecated # But still needed to pass the build @@ -79,7 +79,7 @@ check: gen-proto install-cilint golangci-lint run .PHONY: test -test: gen-swagger setup-dependencies +test: build gen-swagger setup-dependencies go test -cover -v -count=1 ./... # Dependent targets diff --git a/cmd/init_db/main.go b/cmd/init_db/main.go index 5752003..6924246 100644 --- a/cmd/init_db/main.go +++ b/cmd/init_db/main.go @@ -4,17 +4,17 @@ import ( judge_model "github.com/oj-lab/oj-lab-platform/models/judge" problem_model "github.com/oj-lab/oj-lab-platform/models/problem" user_model "github.com/oj-lab/oj-lab-platform/models/user" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" "github.com/oj-lab/oj-lab-platform/modules/log" ) func main() { - db := gormAgent.GetDefaultDB() + db := gorm_agent.GetDefaultDB() err := db.AutoMigrate( &user_model.User{}, &problem_model.Problem{}, - &judge_model.JudgeTaskSubmission{}, - &judge_model.Judger{}) + &judge_model.Judge{}, + ) if err != nil { panic("failed to migrate database") } diff --git a/cmd/problem_loader/main.go b/cmd/problem_loader/main.go index abec531..f9a5e7f 100644 --- a/cmd/problem_loader/main.go +++ b/cmd/problem_loader/main.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io/fs" - "log" "os" "path" "path/filepath" @@ -12,19 +11,18 @@ import ( "github.com/minio/minio-go/v7" problem_model "github.com/oj-lab/oj-lab-platform/models/problem" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" - minioAgent "github.com/oj-lab/oj-lab-platform/modules/agent/minio" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" + minio_agent "github.com/oj-lab/oj-lab-platform/modules/agent/minio" "github.com/oj-lab/oj-lab-platform/modules/config" + "github.com/oj-lab/oj-lab-platform/modules/log" yaml "gopkg.in/yaml.v2" ) var ctx = context.Background() func main() { - db := gormAgent.GetDefaultDB() - minioClient := minioAgent.GetMinioClient() - - log.Printf("%#v\n", minioClient) // minioClient is now set up + db := gorm_agent.GetDefaultDB() + minioClient := minio_agent.GetMinioClient() // Read package files // Search Problem under packagePath @@ -38,6 +36,10 @@ func main() { slug string ) err := filepath.Walk(packagePath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + log.AppLogger().WithError(err).Error("Walk package path failed") + return err + } if info == nil { return fmt.Errorf("file info is nil") } @@ -45,32 +47,31 @@ func main() { return nil } relativePath := strings.Replace(path, packagePath, "", 1) - println("relativePath: ", relativePath) + log.AppLogger().WithField("relativePath", relativePath).Debug("Read file from package") if filepath.Base(relativePath) == "problem.yaml" { resultMap := make(map[string]interface{}) yamlFile, err := os.ReadFile(path) if err != nil { - log.Println(err) + log.AppLogger().WithError(err).Error("Read problem.yaml failed") } err = yaml.Unmarshal(yamlFile, &resultMap) if err != nil { - log.Printf("Unmarshal: %v\n", err) + log.AppLogger().WithError(err).Error("Unmarshal problem.yaml failed") } title = resultMap["name"].(string) if title == "" { - log.Fatal("name key not exist in problem.yaml") + log.AppLogger().Error("Problem title is empty") } slug = strings.Split(relativePath, "/")[1] - log.Println("title: ", title) - log.Println("slug: ", slug) + log.AppLogger().WithField("title", title).WithField("slug", slug).Debug("Read problem.yaml") } if filepath.Base(relativePath) == "problem.md" { content, err := os.ReadFile(path) if err != nil { - log.Println(err) + log.AppLogger().WithError(err).Error("Read problem.md failed") } description := string(content) - println("description: ", description) + log.AppLogger().WithField("description", description).Debug("Read problem.md") err = problem_model.CreateProblem(db, problem_model.Problem{ Slug: slug, Title: title, @@ -84,12 +85,12 @@ func main() { } } - _, minioErr := minioClient.FPutObject(ctx, minioAgent.GetBucketName(), + _, err = minioClient.FPutObject(ctx, minio_agent.GetBucketName(), relativePath, path, minio.PutObjectOptions{}) - if minioErr != nil { - log.Fatalln(minioErr) + if err != nil { + log.AppLogger().WithError(err).Error("Put object to minio failed") } return err }) @@ -97,5 +98,5 @@ func main() { panic(err) } - log.Println("Read Problem Success!") + log.AppLogger().Info("Problem loaded") } diff --git a/cmd/web_server/handler/judge.go b/cmd/web_server/handler/judge.go index 99c88ee..f795ba5 100644 --- a/cmd/web_server/handler/judge.go +++ b/cmd/web_server/handler/judge.go @@ -1,65 +1,85 @@ package handler import ( + "fmt" + "strconv" + "github.com/gin-gonic/gin" + "github.com/oj-lab/oj-lab-platform/models" + judge_model "github.com/oj-lab/oj-lab-platform/models/judge" + "github.com/oj-lab/oj-lab-platform/modules" judge_service "github.com/oj-lab/oj-lab-platform/services/judge" - "github.com/redis/go-redis/v9" ) -func SetupJudgeRoute(baseRoute *gin.RouterGroup) { +func SetupJudgeRouter(baseRoute *gin.RouterGroup) { g := baseRoute.Group("/judge") { - g.POST("/task/pick", postPickJudgeTask) - g.POST("/task/report", postReportJudgeTaskResult) + g.GET("", getJudgeList) + g.GET("/:uid", getJudge) } } -type PickJudgeTaskBody struct { - Consumer string `json:"consumer"` -} - -func postPickJudgeTask(ginCtx *gin.Context) { - body := PickJudgeTaskBody{} - if err := ginCtx.ShouldBindJSON(&body); err != nil { - _ = ginCtx.Error(err) - return - } - - task, err := judge_service.PickJudgeTask(ginCtx, body.Consumer) - if err == redis.Nil { - ginCtx.Status(204) - return - } +func getJudge(ginCtx *gin.Context) { + uid := ginCtx.Param("uid") + judge, err := judge_service.GetJudge(ginCtx, uid) if err != nil { - _ = ginCtx.Error(err) + modules.NewInternalError(fmt.Sprintf("failed to get judge: %v", err)).AppendToGin(ginCtx) return } - ginCtx.JSON(200, gin.H{ - "task": task, - }) + ginCtx.JSON(200, judge) } -type ReportJudgeTaskResultBody struct { - Consumer string `json:"consumer"` - StreamID string `json:"stream_id"` - VerdictJson string `json:"verdict_json"` +type getJudgeListResponse struct { + Total int64 `json:"total"` + List []*judge_model.Judge `json:"list"` } -func postReportJudgeTaskResult(ginCtx *gin.Context) { - body := ReportJudgeTaskResultBody{} - if err := ginCtx.ShouldBindJSON(&body); err != nil { - _ = ginCtx.Error(err) +// Get Judge List +// +// @Summary Get Judge list +// @Description Get Judge list +// @Tags judge +// @Accept json +// @Param limit query int false "limit" +// @Param offset query int false "offset" +// @Router /judge [get] getJudgeListResponse +func getJudgeList(ginCtx *gin.Context) { + limitQuery, _ := ginCtx.GetQuery("limit") + offsetQuery, _ := ginCtx.GetQuery("offset") + if limitQuery == "" { + limitQuery = "10" + } + if offsetQuery == "" { + offsetQuery = "0" + } + + limit, err := strconv.Atoi(limitQuery) + if err != nil { + modules.NewInvalidParamError("limit", "invalid limit").AppendToGin(ginCtx) return } + offset, err := strconv.Atoi(offsetQuery) + if err != nil { + modules.NewInvalidParamError("offset", "invalid offset").AppendToGin(ginCtx) + return + } + + options := judge_model.GetJudgeOptions{ + Limit: &limit, + Offset: &offset, + OrderByColumns: []models.OrderByColumnOption{{Column: "create_at", Desc: true}}, + } - if err := judge_service.ReportJudgeTaskResult(ginCtx, body.Consumer, body.StreamID, body.VerdictJson); err != nil { - _ = ginCtx.Error(err) + judges, total, err := judge_service.GetJudgeList(ginCtx, options) + if err != nil { + modules.NewInternalError(fmt.Sprintf("failed to get judge list: %v", err)).AppendToGin(ginCtx) return } - ginCtx.JSON(200, gin.H{ - "message": "success", + ginCtx.JSON(200, getJudgeListResponse{ + Total: total, + List: judges, }) } diff --git a/cmd/web_server/handler/judge_task.go b/cmd/web_server/handler/judge_task.go new file mode 100644 index 0000000..272cd99 --- /dev/null +++ b/cmd/web_server/handler/judge_task.go @@ -0,0 +1,65 @@ +package handler + +import ( + "github.com/gin-gonic/gin" + judge_service "github.com/oj-lab/oj-lab-platform/services/judge" + "github.com/redis/go-redis/v9" +) + +func SetupJudgeRoute(baseRoute *gin.RouterGroup) { + g := baseRoute.Group("/judge") + { + g.POST("/task/pick", postPickJudgeTask) + g.POST("/task/report", postReportJudgeTaskResult) + } +} + +type PickJudgeTaskBody struct { + Consumer string `json:"consumer"` +} + +func postPickJudgeTask(ginCtx *gin.Context) { + body := PickJudgeTaskBody{} + if err := ginCtx.ShouldBindJSON(&body); err != nil { + _ = ginCtx.Error(err) + return + } + + task, err := judge_service.PickJudgeTask(ginCtx, body.Consumer) + if err == redis.Nil { + ginCtx.Status(204) + return + } + + if err != nil { + _ = ginCtx.Error(err) + return + } + + ginCtx.JSON(200, gin.H{ + "task": task, + }) +} + +type ReportJudgeTaskResultBody struct { + Consumer string `json:"consumer"` + StreamID string `json:"stream_id"` + VerdictJson string `json:"verdict_json"` +} + +func postReportJudgeTaskResult(ginCtx *gin.Context) { + body := ReportJudgeTaskResultBody{} + if err := ginCtx.ShouldBindJSON(&body); err != nil { + _ = ginCtx.Error(err) + return + } + + if err := judge_service.ReportJudgeTask(ginCtx, body.Consumer, body.StreamID, body.VerdictJson); err != nil { + _ = ginCtx.Error(err) + return + } + + ginCtx.JSON(200, gin.H{ + "message": "success", + }) +} diff --git a/cmd/web_server/handler/problem.go b/cmd/web_server/handler/problem.go index 3a7440c..8c5be3a 100644 --- a/cmd/web_server/handler/problem.go +++ b/cmd/web_server/handler/problem.go @@ -23,7 +23,7 @@ func SetupProblemRoute(baseRoute *gin.RouterGroup) { g.DELETE("/:slug", deleteProblem) g.GET("/:slug/check", checkProblemSlug) g.PUT("/:slug/package", putProblemPackage) - g.POST("/:slug/submission", postSubmission) + g.POST("/:slug/judge", postJudge) } } @@ -164,35 +164,35 @@ func checkProblemSlug(ginCtx *gin.Context) { }) } -// PostSubmissionBody +// PostJudgeBody // -// @Description The body of a submission request, containing the code and the language used for the submission. -// @Property code (string) required "The source code of the submission" minlength(1) -// @Property language (SubmissionLanguage) required "The programming language used for the submission" -type PostSubmissionBody struct { - Code string `json:"code" binding:"required"` - Language judge_model.SubmissionLanguage `json:"language" binding:"required"` +// @Description The body of a judge request, containing the code and the language used for the judge. +// @Property code (string) required "The source code of the judge" minlength(1) +// @Property language (ProgrammingLanguage) required "The programming language used for the judge" +type PostJudgeBody struct { + Code string `json:"code" binding:"required"` + Language judge_model.ProgrammingLanguage `json:"language" binding:"required"` } -// postSubmission +// postJudge // -// @Router /problem/{slug}/submission [post] -// @Summary Post submission -// @Description Post submission +// @Router /problem/{slug}/judge [post] +// @Summary Post judge +// @Description Post judge // @Tags problem // @Accept json -// @Param slug path string true "problem slug" -// @Param judgeRequest body PostSubmissionBody true "judge request" -func postSubmission(ginCtx *gin.Context) { +// @Param slug path string true "problem slug" +// @Param judgeRequest body PostJudgeBody true "judge request" +func postJudge(ginCtx *gin.Context) { slug := ginCtx.Param("slug") - body := PostSubmissionBody{} + body := PostJudgeBody{} if err := ginCtx.ShouldBindJSON(&body); err != nil { _ = ginCtx.Error(err) return } - submission := judge_model.NewSubmission("", slug, body.Code, body.Language) - result, err := judge_service.CreateJudgeTaskSubmission(ginCtx, submission) + judge := judge_model.NewJudge("", slug, body.Code, body.Language) + result, err := judge_service.CreateJudge(ginCtx, judge) if err != nil { modules.NewInternalError(err.Error()).AppendToGin(ginCtx) return diff --git a/cmd/web_server/handler/submission.go b/cmd/web_server/handler/submission.go deleted file mode 100644 index d8f09e5..0000000 --- a/cmd/web_server/handler/submission.go +++ /dev/null @@ -1,97 +0,0 @@ -package handler - -import ( - "fmt" - "strconv" - - "github.com/gin-gonic/gin" - "github.com/oj-lab/oj-lab-platform/models" - judge_model "github.com/oj-lab/oj-lab-platform/models/judge" - "github.com/oj-lab/oj-lab-platform/modules" - judge_service "github.com/oj-lab/oj-lab-platform/services/judge" -) - -func SetupSubmissionRouter(baseRoute *gin.RouterGroup) { - g := baseRoute.Group("/submission") - { - g.GET("", getSubmissionList) - g.GET("/:uid", getSubmission) - } -} - -func getSubmission(ginCtx *gin.Context) { - uid := ginCtx.Param("uid") - - submission, err := judge_service.GetJudgeTaskSubmission(ginCtx, uid) - if err != nil { - modules.NewInternalError(fmt.Sprintf("failed to get submission: %v", err)).AppendToGin(ginCtx) - return - } - - ginCtx.JSON(200, gin.H{ - "UID": submission.UID, - "redisStreamID": submission.RedisStreamID, - "userAccount": submission.UserAccount, - "user": submission.User, // include User metadata, If is needed - "problemSlug": submission.ProblemSlug, - "problem": submission.Problem, // include Problem metadata, If is needed - "code": submission.Code, - "language": submission.Language, - "status": submission.Status, - "verdictJson": submission.VerdictJson, - "mainResult": submission.MainResult, - }) -} - -type getSubmissionListResponse struct { - Total int64 `json:"total"` - List []*judge_model.JudgeTaskSubmission `json:"list"` -} - -// Get Submission List -// -// @Summary Get submission list -// @Description Get submission list -// @Tags submission -// @Accept json -// @Param limit query int false "limit" -// @Param offset query int false "offset" -// @Router /submission [get] getSubmissionListResponse -func getSubmissionList(ginCtx *gin.Context) { - limitQuery, _ := ginCtx.GetQuery("limit") - offsetQuery, _ := ginCtx.GetQuery("offset") - if limitQuery == "" { - limitQuery = "10" - } - if offsetQuery == "" { - offsetQuery = "0" - } - - limit, err := strconv.Atoi(limitQuery) - if err != nil { - modules.NewInvalidParamError("limit", "invalid limit").AppendToGin(ginCtx) - return - } - offset, err := strconv.Atoi(offsetQuery) - if err != nil { - modules.NewInvalidParamError("offset", "invalid offset").AppendToGin(ginCtx) - return - } - - options := judge_model.GetSubmissionOptions{ - Limit: &limit, - Offset: &offset, - OrderByColumns: []models.OrderByColumnOption{{Column: "create_at", Desc: true}}, - } - - submissions, total, err := judge_service.GetJudgeTaskSubmissionList(ginCtx, options) - if err != nil { - modules.NewInternalError(fmt.Sprintf("failed to get submission list: %v", err)).AppendToGin(ginCtx) - return - } - - ginCtx.JSON(200, getSubmissionListResponse{ - Total: total, - List: submissions, - }) -} diff --git a/cmd/web_server/handler/swaggo.go b/cmd/web_server/handler/swaggo.go index cca708d..5016137 100644 --- a/cmd/web_server/handler/swaggo.go +++ b/cmd/web_server/handler/swaggo.go @@ -2,7 +2,7 @@ package handler import ( "github.com/gin-gonic/gin" - swaggo_gen "github.com/oj-lab/oj-lab-platform/cmd/web_server/swaggo-gen" + swaggo_gen "github.com/oj-lab/oj-lab-platform/cmd/web_server/swaggo_gen" "github.com/spf13/viper" swagger_files "github.com/swaggo/files" gin_swagger "github.com/swaggo/gin-swagger" diff --git a/cmd/web_server/main.go b/cmd/web_server/main.go index f665668..9a07698 100644 --- a/cmd/web_server/main.go +++ b/cmd/web_server/main.go @@ -73,7 +73,7 @@ func main() { handler.SetupUserRouter(apiRouter) handler.SetupProblemRoute(apiRouter) handler.SetupEventRouter(apiRouter) - handler.SetupSubmissionRouter(apiRouter) + handler.SetupJudgeRouter(apiRouter) handler.SetupJudgeRoute(apiRouter) err := r.Run(servicePort) diff --git a/go.mod b/go.mod index fcf7c18..6b91f73 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( gorm.io/gorm v1.25.10 ) -require github.com/swaggo/swag v1.16.3 // indirect +require github.com/swaggo/swag v1.16.3 require ( github.com/bytedance/sonic/loader v0.1.1 // indirect diff --git a/models/judge/judge.go b/models/judge/judge.go index 68858e1..ce6281c 100644 --- a/models/judge/judge.go +++ b/models/judge/judge.go @@ -1,114 +1,70 @@ -package judge +package judge_model import ( - "strings" - + "github.com/google/uuid" "github.com/oj-lab/oj-lab-platform/models" + problem_model "github.com/oj-lab/oj-lab-platform/models/problem" + user_model "github.com/oj-lab/oj-lab-platform/models/user" ) -// Should contains a priority definition -// Ex. CompileError > RuntimeError > TimeLimitExceeded > MemoryLimitExceeded > SystemError > WrongAnswer > Accepted -type JudgeVerdict string +type JudgeTaskStatus string const ( - JudgeVerdictCompileError JudgeVerdict = "CompileError" // Only for main verdict - JudgeVerdictRuntimeError JudgeVerdict = "RuntimeError" - JudgeVerdictTimeLimitExceeded JudgeVerdict = "TimeLimitExceeded" - JudgeVerdictMemoryLimitExceeded JudgeVerdict = "MemoryLimitExceeded" - JudgeVerdictSystemError JudgeVerdict = "SystemError" // Some runtime unknown error ? - JudgeVerdictWrongAnswer JudgeVerdict = "WrongAnswer" - JudgeVerdictAccepted JudgeVerdict = "Accepted" - JudgeVerdictCancelled JudgeVerdict = "cancelled" // Judge will be cancelled if some point results in Runtime error, Time limit exceeded, Memory limit exceeded + JudgeTaskStatusPending JudgeTaskStatus = "pending" + JudgeTaskStatusWaiting JudgeTaskStatus = "waiting" + JudgeTaskStatusRunning JudgeTaskStatus = "running" + JudgeTaskStatusFinished JudgeTaskStatus = "finished" ) -type JudgeResult struct { - MainVerdict JudgeVerdict `json:"verdict"` // A merge of all TestPoints' verdict, according to the pirority - Detail string `json:"detail"` // A brief description of the result - TestPointCount uint64 `json:"testPointCount"` // Won't be stored in database - TestPointMap map[string]TestPoint `json:"testPoints"` // Won't be stored in database - TestPointsJson string `json:"-"` // Used to store TestPoints in database - AverageTimeMs uint64 `json:"averageTimeMs"` // Won't be stored in database - MaxTimeMs uint64 `json:"maxTimeMs"` // Won't be stored in database - AverageMemory uint64 `json:"averageMemory"` // Won't be stored in database - MaxMemory uint64 `json:"maxMemory"` // Won't be stored in database -} - -type TestPoint struct { - Index string `json:"index"` // The name of *.in/ans file - Verdict JudgeVerdict `json:"verdict"` - Diff *ResultDiff `json:"diff"` // Required if verdict is wrong_answer - TimeUsageMs uint64 `json:"timeUsageMs"` - MemoryUsageByte uint64 `json:"memoryUsageByte"` -} +type ProgrammingLanguage string -type ResultDiff struct { - Expected string `json:"expected"` - Received string `json:"received"` +func (sl ProgrammingLanguage) String() string { + return string(sl) } -type JudgerState string - const ( - JudgerStateIdle JudgerState = "idle" - JudgerStateBusy JudgerState = "busy" - JudgerStateOffline JudgerState = "offline" + ProgrammingLanguageCpp ProgrammingLanguage = "Cpp" + ProgrammingLanguageRust ProgrammingLanguage = "Rust" + ProgrammingLanguagePython ProgrammingLanguage = "Python" ) -type Judger struct { +// Using relationship according to https://gorm.io/docs/belongs_to.html +type Judge struct { models.MetaFields - Host string `gorm:"primaryKey" json:"host"` - State JudgerState `gorm:"default:offline" json:"status"` -} - -type JudgeTask struct { - SubmissionUID string `json:"submissionUID"` - ProblemSlug string `json:"problemSlug"` - Code string `json:"code"` - Language string `json:"language"` - RedisStreamID *string `json:"redisStreamID"` -} - -func (jt *JudgeTask) ToStringMap() map[string]interface{} { - return map[string]interface{}{ - "submission_uid": jt.SubmissionUID, - "problem_slug": jt.ProblemSlug, - "code": jt.Code, - "language": jt.Language, - } -} - -func JudgeTaskFromMap(m map[string]interface{}) *JudgeTask { - return &JudgeTask{ - SubmissionUID: m["submission_uid"].(string), - ProblemSlug: m["problem_slug"].(string), - Code: m["code"].(string), - Language: m["language"].(string), - } + UID uuid.UUID `json:"UID" gorm:"primaryKey"` + RedisStreamID string `json:"redisStreamID"` + UserAccount string `json:"userAccount" gorm:"not null"` + User user_model.User `json:"-"` + ProblemSlug string `json:"problemSlug" gorm:"not null"` + Problem problem_model.Problem `json:"-"` + Code string `json:"code" gorm:"not null"` + Language ProgrammingLanguage `json:"language" gorm:"not null"` + Status JudgeTaskStatus `json:"status" gorm:"default:pending"` + JudgeResultCount uint `json:"judgeResultCount"` + JudgeResults []JudgeResult `json:"judgeResults" gorm:"foreignKey:JudgeUID"` + MainResult JudgeVerdict `json:"mainResult"` } -func (js JudgerState) CanUpdate(nextStatus JudgerState) bool { - switch js { - case JudgerStateOffline: - return nextStatus == JudgerStateIdle - case JudgerStateIdle: - return nextStatus == JudgerStateBusy || nextStatus == JudgerStateOffline - case JudgerStateBusy: - return nextStatus == JudgerStateIdle || nextStatus == JudgerStateOffline - default: - return false +func NewJudge( + userAccount string, + problemSlug string, + code string, + language ProgrammingLanguage, +) Judge { + return Judge{ + UserAccount: userAccount, + ProblemSlug: problemSlug, + Code: code, + Language: language, + Status: JudgeTaskStatusPending, } } -func StringToJudgerState(state string) JudgerState { - state = strings.ToLower(state) - switch state { - case "idle": - return JudgerStateIdle - case "busy": - return JudgerStateBusy - case "offline": - return JudgerStateOffline - default: - return JudgerStateOffline +func (s *Judge) ToJudgeTask() JudgeTask { + return JudgeTask{ + JudgeUID: s.UID.String(), + ProblemSlug: s.ProblemSlug, + Code: s.Code, + Language: s.Language.String(), } } diff --git a/models/judge/judge_db.go b/models/judge/judge_db.go new file mode 100644 index 0000000..feb8bfe --- /dev/null +++ b/models/judge/judge_db.go @@ -0,0 +1,129 @@ +package judge_model + +import ( + "fmt" + + "github.com/google/uuid" + "github.com/oj-lab/oj-lab-platform/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func CreateJudge(tx *gorm.DB, judge Judge) (*Judge, error) { + judge.UID = uuid.New() + judge.MetaFields = models.NewMetaFields() + if judge.UserAccount == "" { + judge.UserAccount = "anonymous" + } + + return &judge, tx.Create(&judge).Error +} + +func GetJudge(tx *gorm.DB, uid string) (*Judge, error) { + db_judge := Judge{} + err := tx.Model(&Judge{}). + Preload("User").Preload("Problem").Where("UID = ?", uid).First(&db_judge).Error + if err != nil { + return nil, err + } + + return &db_judge, nil +} + +type GetJudgeOptions struct { + Selection []string + Statuses []JudgeTaskStatus + UserAccount *string + ProblemSlug *string + Offset *int + Limit *int + OrderByColumns []models.OrderByColumnOption +} + +func buildGetJudgeTXByOptions( + tx *gorm.DB, options GetJudgeOptions, isCount bool, +) *gorm.DB { + tx = tx.Model(&Judge{}). + Preload(clause.Associations) + // See more in: https://gorm.io/docs/preload.html + // Preload("User.Roles").Preload("Problem.Tags").Preload(clause.Associations) + if len(options.Selection) > 0 { + tx = tx.Select(options.Selection) + } + if options.UserAccount != nil { + tx = tx.Where("user_account = ?", *options.UserAccount) + } + if options.ProblemSlug != nil { + tx = tx.Where("problem_slug = ?", *options.ProblemSlug) + } + if len(options.Statuses) > 0 { + tx = tx.Where("status IN ?", options.Statuses) + } + + if !isCount { + if options.Offset != nil { + tx = tx.Offset(*options.Offset) + } + if options.Limit != nil { + tx = tx.Limit(*options.Limit) + } + for _, option := range options.OrderByColumns { + tx = tx.Order(clause.OrderByColumn{ + Column: clause.Column{Name: option.Column}, + Desc: option.Desc, + }) + } + } + + return tx +} + +func GetJudgeListByOptions( + tx *gorm.DB, options GetJudgeOptions, +) ([]*Judge, int64, error) { + tx = buildGetJudgeTXByOptions(tx, options, false) + var judges []*Judge + err := tx.Find(&judges).Error + if err != nil { + return nil, 0, err + } + + tx = buildGetJudgeTXByOptions(tx, options, true) + var count int64 + err = tx.Count(&count).Error + if err != nil { + return nil, 0, err + } + + return judges, count, nil +} + +func UpdateJudge(tx *gorm.DB, judge Judge) error { + updatingJudge := Judge{} + if judge.UID != uuid.Nil { + err := tx.Where("uid = ?", judge.UID).First(&updatingJudge).Error + if err != nil { + return err + } + } else if judge.RedisStreamID != "" { + err := tx.Where("redis_stream_id = ?", judge.RedisStreamID). + First(&updatingJudge).Error + if err != nil { + return err + } + } else { + return fmt.Errorf("judge uid and redis stream id are both empty") + } + + if judge.Status != "" { + updatingJudge.Status = judge.Status + } + if judge.RedisStreamID != "" { + updatingJudge.RedisStreamID = judge.RedisStreamID + } + if judge.MainResult != "" { + updatingJudge.MainResult = judge.MainResult + } + + return tx.Model(&updatingJudge).Updates(updatingJudge).Error +} diff --git a/models/judge/judge_result.go b/models/judge/judge_result.go new file mode 100644 index 0000000..a07ffcf --- /dev/null +++ b/models/judge/judge_result.go @@ -0,0 +1,33 @@ +package judge_model + +import ( + "github.com/google/uuid" + "github.com/oj-lab/oj-lab-platform/models" +) + +// Should contains a priority definition +// Ex. CompileError > RuntimeError > TimeLimitExceeded > MemoryLimitExceeded > SystemError > WrongAnswer > Accepted +type JudgeVerdict string + +const ( + JudgeVerdictCompileError JudgeVerdict = "CompileError" // Only for main verdict + JudgeVerdictRuntimeError JudgeVerdict = "RuntimeError" + JudgeVerdictTimeLimitExceeded JudgeVerdict = "TimeLimitExceeded" + JudgeVerdictMemoryLimitExceeded JudgeVerdict = "MemoryLimitExceeded" + JudgeVerdictSystemError JudgeVerdict = "SystemError" // Some runtime unknown error ? + JudgeVerdictWrongAnswer JudgeVerdict = "WrongAnswer" + JudgeVerdictAccepted JudgeVerdict = "Accepted" + JudgeVerdictCancelled JudgeVerdict = "cancelled" // Judge will be cancelled if some point results in Runtime error, Time limit exceeded, Memory limit exceeded +) + +type JudgeResult struct { + models.MetaFields + UID uuid.UUID `json:"UID" gorm:"primaryKey"` + JudgeUID uuid.UUID `json:"judgeUID"` + Verdict JudgeVerdict `json:"verdict"` + TimeUsageMs uint64 `json:"timeUsageMs"` + MemoryUsageByte uint64 `json:"memoryUsageByte"` + Output string `json:"output"` + ExpectedOutput string `json:"expectedOutput"` + SystemOutput string `json:"systemOutput"` +} diff --git a/models/judge/judge_result_db.go b/models/judge/judge_result_db.go new file mode 100644 index 0000000..14de8ef --- /dev/null +++ b/models/judge/judge_result_db.go @@ -0,0 +1,14 @@ +package judge_model + +import ( + "github.com/google/uuid" + "github.com/oj-lab/oj-lab-platform/models" + "gorm.io/gorm" +) + +func CreateJudgeResult(tx *gorm.DB, result JudgeResult) (*JudgeResult, error) { + result.UID = uuid.New() + result.MetaFields = models.NewMetaFields() + + return &result, tx.Create(&result).Error +} diff --git a/models/judge/judge_task.go b/models/judge/judge_task.go new file mode 100644 index 0000000..4fd667a --- /dev/null +++ b/models/judge/judge_task.go @@ -0,0 +1,27 @@ +package judge_model + +type JudgeTask struct { + JudgeUID string `json:"JudgeUID"` + ProblemSlug string `json:"problemSlug"` + Code string `json:"code"` + Language string `json:"language"` + RedisStreamID *string `json:"redisStreamID"` +} + +func (jt *JudgeTask) ToStringMap() map[string]interface{} { + return map[string]interface{}{ + "judge_uid": jt.JudgeUID, + "problem_slug": jt.ProblemSlug, + "code": jt.Code, + "language": jt.Language, + } +} + +func JudgeTaskFromMap(m map[string]interface{}) *JudgeTask { + return &JudgeTask{ + JudgeUID: m["judge_uid"].(string), + ProblemSlug: m["problem_slug"].(string), + Code: m["code"].(string), + Language: m["language"].(string), + } +} diff --git a/models/judge/judge_task_stream.go b/models/judge/judge_task_stream.go new file mode 100644 index 0000000..ed125dc --- /dev/null +++ b/models/judge/judge_task_stream.go @@ -0,0 +1,78 @@ +package judge_model + +import ( + "context" + + redis_agent "github.com/oj-lab/oj-lab-platform/modules/agent/redis" + "github.com/redis/go-redis/v9" +) + +const ( + streamName = "oj_lab_judge_stream" + consumerGroupName = "oj_lab_judge_stream_consumer_group" + defaultConsumerName = "oj_lab_judge_stream_consumer_default" +) + +func init() { + redis_agent := redis_agent.GetDefaultRedisClient() + _, err := redis_agent.XGroupCreateMkStream( + context.Background(), streamName, consumerGroupName, "0").Result() + if err != nil && + err != redis.Nil && + err.Error() != "BUSYGROUP Consumer Group name already exists" { + panic(err) + } +} + +func AddTaskToStream(ctx context.Context, task *JudgeTask) (*string, error) { + redis_agent := redis_agent.GetDefaultRedisClient() + id, err := redis_agent.XAdd(ctx, &redis.XAddArgs{ + Stream: streamName, + Values: task.ToStringMap(), + }).Result() + if err != nil { + return nil, err + } + + return &id, err +} + +func GetTaskFromStream(ctx context.Context, consumer string) (*JudgeTask, error) { + redis_agent := redis_agent.GetDefaultRedisClient() + if consumer == "" { + consumer = defaultConsumerName + } + result, err := redis_agent.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: consumerGroupName, + Consumer: consumer, + Streams: []string{streamName, ">"}, + Count: 1, + Block: -1, + }).Result() + + if err != nil { + return nil, err + } + if len(result) == 0 { + return nil, nil + } + + task := JudgeTask{} + for _, message := range result[0].Messages { + task = *JudgeTaskFromMap(message.Values) + task.RedisStreamID = &message.ID + } + + return &task, nil +} + +func AckTaskFromStream(ctx context.Context, streamID string) error { + redis_agent := redis_agent.GetDefaultRedisClient() + + _, err := redis_agent.XAck(ctx, streamName, consumerGroupName, streamID).Result() + if err != nil { + return err + } + + return nil +} diff --git a/models/judge/submission.go b/models/judge/submission.go deleted file mode 100644 index d8b2463..0000000 --- a/models/judge/submission.go +++ /dev/null @@ -1,69 +0,0 @@ -package judge - -import ( - "github.com/google/uuid" - "github.com/oj-lab/oj-lab-platform/models" - problem_model "github.com/oj-lab/oj-lab-platform/models/problem" - user_model "github.com/oj-lab/oj-lab-platform/models/user" -) - -type SubmissionStatus string - -const ( - SubmissionStatusPending SubmissionStatus = "pending" - SubmissionStatusWaiting SubmissionStatus = "waiting" - SubmissionStatusRunning SubmissionStatus = "running" - SubmissionStatusFinished SubmissionStatus = "finished" -) - -type SubmissionLanguage string - -func (sl SubmissionLanguage) String() string { - return string(sl) -} - -const ( - SubmissionLanguageCpp SubmissionLanguage = "Cpp" - SubmissionLanguageRust SubmissionLanguage = "Rust" - SubmissionLanguagePython SubmissionLanguage = "Python" -) - -// Using relationship according to https://gorm.io/docs/belongs_to.html -type JudgeTaskSubmission struct { - models.MetaFields - UID uuid.UUID `gorm:"primaryKey" json:"UID"` - RedisStreamID string `json:"redisStreamID"` - UserAccount string `gorm:"not null" json:"userAccount"` - User user_model.User `json:"user"` - ProblemSlug string `gorm:"not null" json:"problemSlug"` - Problem problem_model.Problem `json:"problem"` - Code string `gorm:"not null" json:"code"` - Language SubmissionLanguage `gorm:"not null" json:"language"` - Status SubmissionStatus `gorm:"default:pending" json:"status"` - VerdictJson string `json:"verdictJson"` - MainResult JudgeVerdict `json:"mainResult"` -} - -func NewSubmission( - userAccount string, - problemSlug string, - code string, - language SubmissionLanguage, -) JudgeTaskSubmission { - return JudgeTaskSubmission{ - UserAccount: userAccount, - ProblemSlug: problemSlug, - Code: code, - Language: language, - Status: SubmissionStatusPending, - } -} - -func (s *JudgeTaskSubmission) ToJudgeTask() JudgeTask { - return JudgeTask{ - SubmissionUID: s.UID.String(), - ProblemSlug: s.ProblemSlug, - Code: s.Code, - Language: s.Language.String(), - } -} diff --git a/models/judge/submission_db.go b/models/judge/submission_db.go deleted file mode 100644 index cbb377c..0000000 --- a/models/judge/submission_db.go +++ /dev/null @@ -1,123 +0,0 @@ -package judge - -import ( - "fmt" - - "github.com/google/uuid" - "github.com/oj-lab/oj-lab-platform/models" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -func CreateSubmission(tx *gorm.DB, submission JudgeTaskSubmission) (*JudgeTaskSubmission, error) { - submission.UID = uuid.New() - submission.MetaFields = models.NewMetaFields() - if submission.UserAccount == "" { - submission.UserAccount = "anonymous" - } - - return &submission, tx.Create(&submission).Error -} - -func GetSubmission(tx *gorm.DB, uid string) (*JudgeTaskSubmission, error) { - db_submission := JudgeTaskSubmission{} - err := tx.Model(&JudgeTaskSubmission{}).Preload("User").Preload("Problem").Where("UID = ?", uid).First(&db_submission).Error - if err != nil { - return nil, err - } - - return &db_submission, nil -} - -type GetSubmissionOptions struct { - Selection []string - Statuses []SubmissionStatus - UserAccount *string - ProblemSlug *string - Offset *int - Limit *int - OrderByColumns []models.OrderByColumnOption -} - -func BuildGetSubmissionTXByOptions(tx *gorm.DB, options GetSubmissionOptions, isCount bool) *gorm.DB { - tx = tx.Model(&JudgeTaskSubmission{}). - Preload(clause.Associations) - // See more in: https://gorm.io/docs/preload.html - // Preload("User.Roles").Preload("Problem.Tags").Preload(clause.Associations) - if len(options.Selection) > 0 { - tx = tx.Select(options.Selection) - } - if options.UserAccount != nil { - tx = tx.Where("user_account = ?", *options.UserAccount) - } - if options.ProblemSlug != nil { - tx = tx.Where("problem_slug = ?", *options.ProblemSlug) - } - if len(options.Statuses) > 0 { - tx = tx.Where("status IN ?", options.Statuses) - } - if options.Offset != nil { - tx = tx.Offset(*options.Offset) - } - if options.Limit != nil { - tx = tx.Limit(*options.Limit) - } - for _, option := range options.OrderByColumns { - tx = tx.Order(clause.OrderByColumn{ - Column: clause.Column{Name: option.Column}, - Desc: option.Desc, - }) - } - - return tx -} - -func GetSubmissionListByOptions(tx *gorm.DB, options GetSubmissionOptions) ([]*JudgeTaskSubmission, int64, error) { - tx = BuildGetSubmissionTXByOptions(tx, options, false) - var submissions []*JudgeTaskSubmission - err := tx.Find(&submissions).Error - if err != nil { - return nil, 0, err - } - - tx = BuildGetSubmissionTXByOptions(tx, options, true) - var count int64 - err = tx.Count(&count).Error - if err != nil { - return nil, 0, err - } - - return submissions, count, nil -} - -func UpdateSubmission(tx *gorm.DB, submission JudgeTaskSubmission) error { - updatingSubmission := JudgeTaskSubmission{} - if submission.UID != uuid.Nil { - err := tx.Where("uid = ?", submission.UID).First(&updatingSubmission).Error - if err != nil { - return err - } - } else if submission.RedisStreamID != "" { - err := tx.Where("redis_stream_id = ?", submission.RedisStreamID).First(&updatingSubmission).Error - if err != nil { - return err - } - } else { - return fmt.Errorf("submission uid and redis stream id are both empty") - } - - if submission.Status != "" { - updatingSubmission.Status = submission.Status - } - if submission.VerdictJson != "" { - updatingSubmission.VerdictJson = submission.VerdictJson - } - if submission.RedisStreamID != "" { - updatingSubmission.RedisStreamID = submission.RedisStreamID - } - if submission.MainResult != "" { - updatingSubmission.MainResult = submission.MainResult - } - - return tx.Model(&updatingSubmission).Updates(updatingSubmission).Error -} diff --git a/models/problem/problem.go b/models/problem/problem.go index 014258d..fd914a9 100644 --- a/models/problem/problem.go +++ b/models/problem/problem.go @@ -6,7 +6,7 @@ type Problem struct { models.MetaFields Slug string `gorm:"primaryKey" json:"slug"` Title string `gorm:"not null" json:"title"` - Description *string `gorm:"not null" json:"description,omitempty"` + Description *string `json:"description,omitempty"` Tags []*AlgorithmTag `gorm:"many2many:problem_algorithm_tags;" json:"tags"` } diff --git a/models/user/user_db.go b/models/user/user_db.go index 7cf65d3..e8cd76e 100644 --- a/models/user/user_db.go +++ b/models/user/user_db.go @@ -36,7 +36,6 @@ func GetUser(tx *gorm.DB, account string) (*User, error) { } func GetPublicUser(tx *gorm.DB, account string) (*User, error) { - db_user := User{} err := tx.Model(&User{}).Preload("Roles").Select(PublicUserSelection).Where("account = ?", account).First(&db_user).Error if err != nil { diff --git a/modules/agent/gorm/database.go b/modules/agent/gorm/database.go index 95705b1..95099b2 100644 --- a/modules/agent/gorm/database.go +++ b/modules/agent/gorm/database.go @@ -1,4 +1,4 @@ -package gormAgent +package gorm_agent import ( "github.com/oj-lab/oj-lab-platform/modules/config" @@ -25,7 +25,9 @@ func GetDefaultDB() *gorm.DB { db, err = gorm.Open(postgres.New(postgres.Config{ DSN: dsn, PreferSimpleProtocol: true, // disables implicit prepared statement usage - }), &gorm.Config{}) + }), &gorm.Config{ + Logger: getLogger(), + }) if err != nil { panic("failed to connect database") } diff --git a/modules/agent/gorm/logger.go b/modules/agent/gorm/logger.go new file mode 100644 index 0000000..4b9e420 --- /dev/null +++ b/modules/agent/gorm/logger.go @@ -0,0 +1,24 @@ +package gorm_agent + +import ( + "log" + "os" + "time" + + "gorm.io/gorm/logger" +) + +func getLogger() logger.Interface { + logger := logger.New( + log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer + logger.Config{ + SlowThreshold: time.Second, // Slow SQL threshold + LogLevel: logger.Silent, // Log level + IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger + ParameterizedQueries: true, // Don't include params in the SQL log + Colorful: false, // Disable color + }, + ) + + return logger +} diff --git a/modules/agent/judger/client.go b/modules/agent/judger/client.go index 52a1207..6a36c54 100644 --- a/modules/agent/judger/client.go +++ b/modules/agent/judger/client.go @@ -1,4 +1,4 @@ -package judgerAgent +package judger_agent type JudgerClient struct { Host string diff --git a/modules/agent/judger/judge.go b/modules/agent/judger/judge.go index b4ce817..b33c7dd 100644 --- a/modules/agent/judger/judge.go +++ b/modules/agent/judger/judge.go @@ -1,4 +1,4 @@ -package judgerAgent +package judger_agent import ( "bytes" diff --git a/modules/agent/judger/state.go b/modules/agent/judger/state.go index 26c5cab..6f117af 100644 --- a/modules/agent/judger/state.go +++ b/modules/agent/judger/state.go @@ -1,4 +1,4 @@ -package judgerAgent +package judger_agent import ( "io" diff --git a/modules/agent/minio/client.go b/modules/agent/minio/client.go index b32adb6..0a99e56 100644 --- a/modules/agent/minio/client.go +++ b/modules/agent/minio/client.go @@ -1,12 +1,12 @@ -package minioAgent +package minio_agent import ( "context" - "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/oj-lab/oj-lab-platform/modules/config" + "github.com/oj-lab/oj-lab-platform/modules/log" ) const ( @@ -54,17 +54,18 @@ func GetMinioClient() *minio.Client { } ctx := context.Background() + exists, err := minioClient.BucketExists(ctx, bucketName) + if err == nil && exists { + log.AppLogger().WithField("bucket", bucketName).Info("Bucket already exists") + return minioClient + } + err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) if err != nil { - // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := minioClient.BucketExists(ctx, bucketName) - if errBucketExists == nil && exists { - log.Printf("We already own %s\n", bucketName) - } else { - log.Fatalln(err) - } + log.AppLogger().WithError(err). + WithField("bucket", bucketName).Error("Failed to create bucket") } else { - log.Printf("Successfully created %s\n", bucketName) + log.AppLogger().WithField("bucket", bucketName).Info("Successfully created bucket") } } diff --git a/modules/agent/minio/local.go b/modules/agent/minio/local.go index 5e9285b..b53db52 100644 --- a/modules/agent/minio/local.go +++ b/modules/agent/minio/local.go @@ -1,4 +1,4 @@ -package minioAgent +package minio_agent import ( "context" diff --git a/modules/agent/redis/client.go b/modules/agent/redis/client.go index e388f48..9f3aa14 100644 --- a/modules/agent/redis/client.go +++ b/modules/agent/redis/client.go @@ -1,4 +1,4 @@ -package redisAgent +package redis_agent import ( "github.com/oj-lab/oj-lab-platform/modules/config" diff --git a/modules/auth/redis.go b/modules/auth/redis.go index ede64ba..a901d00 100644 --- a/modules/auth/redis.go +++ b/modules/auth/redis.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - redisAgent "github.com/oj-lab/oj-lab-platform/modules/agent/redis" + redis_agent "github.com/oj-lab/oj-lab-platform/modules/agent/redis" "github.com/oj-lab/oj-lab-platform/modules/log" "github.com/redis/go-redis/v9" ) @@ -18,7 +18,7 @@ func getLoginSessionRedisKey(key LoginSessionKey) string { } func SetLoginSession(ctx context.Context, key LoginSessionKey, data LoginSessionData) error { - redisClient := redisAgent.GetDefaultRedisClient() + redisClient := redis_agent.GetDefaultRedisClient() value, err := data.GetJsonString() if err != nil { @@ -34,7 +34,7 @@ func SetLoginSession(ctx context.Context, key LoginSessionKey, data LoginSession } func GetLoginSession(ctx context.Context, key LoginSessionKey) (*LoginSession, error) { - redisClient := redisAgent.GetDefaultRedisClient() + redisClient := redis_agent.GetDefaultRedisClient() val, err := redisClient.Get(ctx, getLoginSessionRedisKey(key)).Result() if err != nil { @@ -52,7 +52,7 @@ func GetLoginSession(ctx context.Context, key LoginSessionKey) (*LoginSession, e } func UpdateLoginSessionByAccount(ctx context.Context, account string, data LoginSessionData) error { - redisClient := redisAgent.GetDefaultRedisClient() + redisClient := redis_agent.GetDefaultRedisClient() redisKeys, err := redisClient.Keys(ctx, fmt.Sprintf(loginSessionKeyFormat, account, "*")).Result() if err != nil { diff --git a/modules/error.go b/modules/error.go index 884a003..7b809b0 100644 --- a/modules/error.go +++ b/modules/error.go @@ -2,6 +2,7 @@ package modules import ( "fmt" + "net/http" "runtime" "github.com/gin-gonic/gin" @@ -39,14 +40,14 @@ func IsServiceError(err interface{}) bool { func NewInternalError(msg string) *SeviceError { return &SeviceError{ - Code: 500, + Code: http.StatusInternalServerError, Msg: msg, } } func NewUnauthorizedError(msg string) *SeviceError { return &SeviceError{ - Code: 401, + Code: http.StatusUnauthorized, Msg: msg, } } @@ -58,7 +59,7 @@ func NewInvalidParamError(param string, hints ...string) *SeviceError { } return &SeviceError{ - Code: 400, + Code: http.StatusBadRequest, Msg: msg, } } diff --git a/modules/log/log.go b/modules/log/log.go index 18bac59..b84183c 100644 --- a/modules/log/log.go +++ b/modules/log/log.go @@ -12,7 +12,7 @@ const logLevelProp = "log.level" func AppLogger() *logrus.Entry { return logrus.WithFields(logrus.Fields{ - "CALLER": func() string { + "caller": func() string { pc := make([]uintptr, 1) runtime.Callers(3, pc) f := runtime.FuncForPC(pc[0]) @@ -31,6 +31,8 @@ func setupLog() { println("log level:", lvl) logrus.SetLevel(logLevel) } + // TODO: control log format in config + // logrus.SetFormatter(&logrus.JSONFormatter{}) } func init() { diff --git a/services/judge/judge.go b/services/judge/judge.go new file mode 100644 index 0000000..200bf10 --- /dev/null +++ b/services/judge/judge.go @@ -0,0 +1,54 @@ +package judge + +import ( + "context" + + judge_model "github.com/oj-lab/oj-lab-platform/models/judge" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" +) + +func GetJudge(ctx context.Context, uid string) (*judge_model.Judge, error) { + db := gorm_agent.GetDefaultDB() + judge, err := judge_model.GetJudge(db, uid) + if err != nil { + return nil, err + } + + return judge, nil +} + +func GetJudgeList( + ctx context.Context, options judge_model.GetJudgeOptions, +) ([]*judge_model.Judge, int64, error) { + db := gorm_agent.GetDefaultDB() + judges, total, err := judge_model.GetJudgeListByOptions(db, options) + if err != nil { + return nil, 0, err + } + + return judges, total, nil +} + +func CreateJudge( + ctx context.Context, judge judge_model.Judge, +) (*judge_model.Judge, error) { + db := gorm_agent.GetDefaultDB() + newJudge, err := judge_model.CreateJudge(db, judge) + if err != nil { + return nil, err + } + + task := newJudge.ToJudgeTask() + streamId, err := judge_model.AddTaskToStream(ctx, &task) + if err != nil { + return nil, err + } + + newJudge.RedisStreamID = *streamId + err = judge_model.UpdateJudge(db, *newJudge) + if err != nil { + return nil, err + } + + return newJudge, nil +} diff --git a/services/judge/judge_stream.go b/services/judge/judge_stream.go deleted file mode 100644 index ac824fe..0000000 --- a/services/judge/judge_stream.go +++ /dev/null @@ -1,79 +0,0 @@ -package judge - -import ( - "context" - - judge_model "github.com/oj-lab/oj-lab-platform/models/judge" - redis_agent "github.com/oj-lab/oj-lab-platform/modules/agent/redis" - "github.com/redis/go-redis/v9" -) - -const ( - streamName = "oj_lab_judge_stream" - consumerGroupName = "oj_lab_judge_stream_consumer_group" - defaultConsumerName = "oj_lab_judge_stream_consumer_default" -) - -func init() { - redisAgent := redis_agent.GetDefaultRedisClient() - _, err := redisAgent.XGroupCreateMkStream(context.Background(), streamName, consumerGroupName, "0").Result() - if err != nil && err != redis.Nil && err.Error() != "BUSYGROUP Consumer Group name already exists" { - panic(err) - } -} - -func addTaskToStream(ctx context.Context, task *judge_model.JudgeTask) (*string, error) { - redisAgent := redis_agent.GetDefaultRedisClient() - id, err := redisAgent.XAdd(ctx, &redis.XAddArgs{ - Stream: streamName, - Values: task.ToStringMap(), - }).Result() - if err != nil { - return nil, err - } - - return &id, err -} - -func getTaskFromStream(ctx context.Context, consumer string) (*judge_model.JudgeTask, error) { - redisAgent := redis_agent.GetDefaultRedisClient() - if consumer == "" { - consumer = defaultConsumerName - } - result, err := redisAgent.XReadGroup(ctx, &redis.XReadGroupArgs{ - Group: consumerGroupName, - Consumer: consumer, - Streams: []string{streamName, ">"}, - Count: 1, - Block: -1, - }).Result() - - if err != nil { - return nil, err - } - if len(result) == 0 { - return nil, nil - } - - task := judge_model.JudgeTask{} - for _, message := range result[0].Messages { - task = *judge_model.JudgeTaskFromMap(message.Values) - task.RedisStreamID = &message.ID - } - - return &task, nil -} - -func ackTaskFromStream(ctx context.Context, consumer string, streamID string) error { - redisAgent := redis_agent.GetDefaultRedisClient() - // TODO: Some ineffectual assignment here, need to find out why - // if consumer == "" { - // consumer = defaultConsumerName - // } - _, err := redisAgent.XAck(ctx, streamName, consumerGroupName, streamID).Result() - if err != nil { - return err - } - - return nil -} diff --git a/services/judge/task.go b/services/judge/judge_task.go similarity index 86% rename from services/judge/task.go rename to services/judge/judge_task.go index 3c4e3cd..5115351 100644 --- a/services/judge/task.go +++ b/services/judge/judge_task.go @@ -12,24 +12,24 @@ import ( ) func PickJudgeTask(ctx context.Context, consumer string) (*judge_model.JudgeTask, error) { - task, err := getTaskFromStream(ctx, consumer) + task, err := judge_model.GetTaskFromStream(ctx, consumer) if err != nil { return nil, fmt.Errorf("failed to get task from stream: %w", err) } db := gorm_agent.GetDefaultDB() - err = judge_model.UpdateSubmission(db, judge_model.JudgeTaskSubmission{ - UID: uuid.MustParse(task.SubmissionUID), - Status: judge_model.SubmissionStatusRunning, + err = judge_model.UpdateJudge(db, judge_model.Judge{ + UID: uuid.MustParse(task.JudgeUID), + Status: judge_model.JudgeTaskStatusRunning, }) if err != nil { - return nil, fmt.Errorf("failed to update submission status: %w", err) + return nil, fmt.Errorf("failed to update judge status: %w", err) } return task, nil } -func ReportJudgeTaskResult( +func ReportJudgeTask( ctx context.Context, consumer string, streamID string, verdictJson string, ) error { @@ -39,10 +39,9 @@ func ReportJudgeTaskResult( if err != nil { return err } - err = judge_model.UpdateSubmission(db, judge_model.JudgeTaskSubmission{ + err = judge_model.UpdateJudge(db, judge_model.Judge{ RedisStreamID: streamID, - Status: judge_model.SubmissionStatusFinished, - VerdictJson: verdictJson, + Status: judge_model.JudgeTaskStatusFinished, MainResult: mainVerdict, }) @@ -50,7 +49,7 @@ func ReportJudgeTaskResult( return err } - err = ackTaskFromStream(ctx, consumer, streamID) + err = judge_model.AckTaskFromStream(ctx, streamID) if err != nil { return err } diff --git a/services/judge/submission.go b/services/judge/submission.go deleted file mode 100644 index 5dccf93..0000000 --- a/services/judge/submission.go +++ /dev/null @@ -1,54 +0,0 @@ -package judge - -import ( - "context" - - judge_model "github.com/oj-lab/oj-lab-platform/models/judge" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" -) - -func GetJudgeTaskSubmission(ctx context.Context, uid string) (*judge_model.JudgeTaskSubmission, error) { - db := gormAgent.GetDefaultDB() - submission, err := judge_model.GetSubmission(db, uid) - if err != nil { - return nil, err - } - - return submission, nil -} - -func GetJudgeTaskSubmissionList( - ctx context.Context, options judge_model.GetSubmissionOptions, -) ([]*judge_model.JudgeTaskSubmission, int64, error) { - db := gormAgent.GetDefaultDB() - submissions, total, err := judge_model.GetSubmissionListByOptions(db, options) - if err != nil { - return nil, 0, err - } - - return submissions, total, nil -} - -func CreateJudgeTaskSubmission( - ctx context.Context, submission judge_model.JudgeTaskSubmission, -) (*judge_model.JudgeTaskSubmission, error) { - db := gormAgent.GetDefaultDB() - newSubmission, err := judge_model.CreateSubmission(db, submission) - if err != nil { - return nil, err - } - - task := newSubmission.ToJudgeTask() - streamId, err := addTaskToStream(ctx, &task) - if err != nil { - return nil, err - } - - newSubmission.RedisStreamID = *streamId - err = judge_model.UpdateSubmission(db, *newSubmission) - if err != nil { - return nil, err - } - - return newSubmission, nil -} diff --git a/services/problem/problem.go b/services/problem/problem.go index f4d023a..449d441 100644 --- a/services/problem/problem.go +++ b/services/problem/problem.go @@ -4,13 +4,13 @@ import ( "context" problem_model "github.com/oj-lab/oj-lab-platform/models/problem" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" "gorm.io/gorm" ) func GetProblem(ctx context.Context, slug string) (*problem_model.Problem, error) { - db := gormAgent.GetDefaultDB() + db := gorm_agent.GetDefaultDB() problem, err := problem_model.GetProblem(db, slug) if err != nil { return nil, err @@ -19,7 +19,7 @@ func GetProblem(ctx context.Context, slug string) (*problem_model.Problem, error } func PutProblem(ctx context.Context, problem problem_model.Problem) error { - db := gormAgent.GetDefaultDB() + db := gorm_agent.GetDefaultDB() err := problem_model.CreateProblem(db, problem) if err != nil { return err @@ -28,7 +28,7 @@ func PutProblem(ctx context.Context, problem problem_model.Problem) error { } func DeleteProblem(ctx context.Context, slug string) error { - db := gormAgent.GetDefaultDB() + db := gorm_agent.GetDefaultDB() err := problem_model.DeleteProblem(db, slug) if err != nil { return err @@ -37,7 +37,7 @@ func DeleteProblem(ctx context.Context, slug string) error { } func CheckProblemSlug(ctx context.Context, slug string) (bool, error) { - db := gormAgent.GetDefaultDB() + db := gorm_agent.GetDefaultDB() problem, err := problem_model.GetProblem(db, slug) if err != nil { if err == gorm.ErrRecordNotFound { diff --git a/services/problem/problem_info.go b/services/problem/problem_info.go index 43b4700..6113c65 100644 --- a/services/problem/problem_info.go +++ b/services/problem/problem_info.go @@ -4,11 +4,11 @@ import ( "context" problem_model "github.com/oj-lab/oj-lab-platform/models/problem" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" ) func getProblemInfoList(ctx context.Context) ([]problem_model.ProblemInfo, int64, error) { - db := gormAgent.GetDefaultDB() + db := gorm_agent.GetDefaultDB() getOptions := problem_model.GetProblemOptions{ Selection: problem_model.ProblemInfoSelection, } diff --git a/services/problem/problem_package.go b/services/problem/problem_package.go index ee59292..8eeaa86 100644 --- a/services/problem/problem_package.go +++ b/services/problem/problem_package.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - minioAgent "github.com/oj-lab/oj-lab-platform/modules/agent/minio" + minio_agent "github.com/oj-lab/oj-lab-platform/modules/agent/minio" ) func unzipProblemPackage(ctx context.Context, zipFile, targetDir string) error { @@ -53,7 +53,7 @@ func unzipProblemPackage(ctx context.Context, zipFile, targetDir string) error { } func putProblemPackage(ctx context.Context, slug string, pkgDir string) error { - err := minioAgent.PutLocalObjects(ctx, slug, pkgDir) + err := minio_agent.PutLocalObjects(ctx, slug, pkgDir) if err != nil { return err } diff --git a/services/user/user.go b/services/user/user.go index e28b42f..c4a7808 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -44,7 +44,10 @@ func CheckUserExist(ctx context.Context, account string) (bool, error) { } if count > 1 { - log.AppLogger().Warnf("user %s has %d records", account, count) + log.AppLogger(). + WithField("account", account). + WithField("count", count). + Warn("user account is not unique") } return count > 0, nil diff --git a/tests/models/judge_test.go b/tests/models/judge_test.go new file mode 100644 index 0000000..14622aa --- /dev/null +++ b/tests/models/judge_test.go @@ -0,0 +1,29 @@ +package models_test + +import ( + "testing" + + judge_model "github.com/oj-lab/oj-lab-platform/models/judge" + problem_model "github.com/oj-lab/oj-lab-platform/models/problem" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" +) + +func TestJudgeDB(t *testing.T) { + db := gorm_agent.GetDefaultDB() + problem := &problem_model.Problem{ + Slug: "test-judge-db-problem", + } + var err error + err = problem_model.CreateProblem(db, *problem) + if err != nil { + t.Error(err) + } + judge := &judge_model.Judge{ + Language: judge_model.ProgrammingLanguageCpp, + ProblemSlug: problem.Slug, + } + _, err = judge_model.CreateJudge(db, *judge) + if err != nil { + t.Error(err) + } +} diff --git a/tests/models/problem_test.go b/tests/models/problem_test.go index f2f4ded..6908b17 100644 --- a/tests/models/problem_test.go +++ b/tests/models/problem_test.go @@ -6,11 +6,11 @@ import ( "testing" problem_model "github.com/oj-lab/oj-lab-platform/models/problem" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" ) -func TestProblemMapper(t *testing.T) { - db := gormAgent.GetDefaultDB() +func TestProblemDB(t *testing.T) { + db := gorm_agent.GetDefaultDB() description := "Given two integer A and B, please output the answer of A+B." problem := problem_model.Problem{ Slug: "a-plus-b-problem", @@ -60,5 +60,4 @@ func TestProblemMapper(t *testing.T) { if err != nil { t.Error(err) } - } diff --git a/tests/models/user_test.go b/tests/models/user_test.go index 4109454..294efee 100644 --- a/tests/models/user_test.go +++ b/tests/models/user_test.go @@ -6,11 +6,11 @@ import ( "testing" user_model "github.com/oj-lab/oj-lab-platform/models/user" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" + gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" ) -func TestUserMapper(t *testing.T) { - db := gormAgent.GetDefaultDB() +func TestUserDB(t *testing.T) { + db := gorm_agent.GetDefaultDB() user := user_model.User{ Account: "test", Password: func() *string { s := "test"; return &s }(), @@ -45,5 +45,4 @@ func TestUserMapper(t *testing.T) { if err != nil { t.Error(err) } - }