From e7abb8d9bb5c520c1c42900bd8cb3dfe686cd8c2 Mon Sep 17 00:00:00 2001 From: slhmy <1484836413@qq.com> Date: Tue, 11 Jun 2024 09:18:23 +0800 Subject: [PATCH] Refactor code --- Makefile | 2 +- cmd/init_db/main.go | 4 +- cmd/problem_loader/main.go | 31 ++-- cmd/web_server/handler/judge.go | 2 +- cmd/web_server/handler/problem.go | 10 +- cmd/web_server/handler/submission.go | 11 +- go.mod | 2 +- models/judge/judge.go | 138 ++++++------------ models/judge/judge_db.go | 129 ++++++++++++++++ models/judge/judge_result.go | 33 +++++ models/judge/judge_result_db.go | 14 ++ models/judge/judge_task.go | 27 ++++ .../judge/judge_task_stream.go | 25 ++-- models/judge/submission.go | 69 --------- models/judge/submission_db.go | 123 ---------------- models/problem/problem.go | 2 +- models/user/user_db.go | 1 - modules/agent/gorm/database.go | 6 +- modules/agent/gorm/logger.go | 24 +++ modules/agent/judger/client.go | 2 +- modules/agent/judger/judge.go | 2 +- modules/agent/judger/state.go | 2 +- modules/agent/minio/client.go | 21 +-- modules/agent/minio/local.go | 2 +- modules/agent/redis/client.go | 2 +- modules/log/log.go | 4 +- services/judge/judge.go | 54 +++++++ services/judge/{task.go => judge_task.go} | 17 +-- services/judge/submission.go | 54 ------- services/user/user.go | 5 +- tests/models/judge_test.go | 29 ++++ tests/models/problem_test.go | 3 +- tests/models/user_test.go | 3 +- 33 files changed, 438 insertions(+), 415 deletions(-) create mode 100644 models/judge/judge_db.go create mode 100644 models/judge/judge_result.go create mode 100644 models/judge/judge_result_db.go create mode 100644 models/judge/judge_task.go rename services/judge/judge_stream.go => models/judge/judge_task_stream.go (62%) delete mode 100644 models/judge/submission.go delete mode 100644 models/judge/submission_db.go create mode 100644 modules/agent/gorm/logger.go create mode 100644 services/judge/judge.go rename services/judge/{task.go => judge_task.go} (88%) delete mode 100644 services/judge/submission.go create mode 100644 tests/models/judge_test.go diff --git a/Makefile b/Makefile index 532f3bd..e7e6195 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ check: gen-proto install-cilint golangci-lint run .PHONY: test -test: gen-swagger setup-dependencies +test: build gen-swagger setup-dependencies go test -cover -v -count=1 ./... # Dependent targets diff --git a/cmd/init_db/main.go b/cmd/init_db/main.go index 5752003..68f8a88 100644 --- a/cmd/init_db/main.go +++ b/cmd/init_db/main.go @@ -13,8 +13,8 @@ func main() { err := db.AutoMigrate( &user_model.User{}, &problem_model.Problem{}, - &judge_model.JudgeTaskSubmission{}, - &judge_model.Judger{}) + &judge_model.Judge{}, + ) if err != nil { panic("failed to migrate database") } diff --git a/cmd/problem_loader/main.go b/cmd/problem_loader/main.go index abec531..fd18b7c 100644 --- a/cmd/problem_loader/main.go +++ b/cmd/problem_loader/main.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io/fs" - "log" "os" "path" "path/filepath" @@ -15,6 +14,7 @@ import ( gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" minioAgent "github.com/oj-lab/oj-lab-platform/modules/agent/minio" "github.com/oj-lab/oj-lab-platform/modules/config" + "github.com/oj-lab/oj-lab-platform/modules/log" yaml "gopkg.in/yaml.v2" ) @@ -24,8 +24,6 @@ func main() { db := gormAgent.GetDefaultDB() minioClient := minioAgent.GetMinioClient() - log.Printf("%#v\n", minioClient) // minioClient is now set up - // Read package files // Search Problem under packagePath // 1. parse problem path as `slug`, @@ -38,6 +36,10 @@ func main() { slug string ) err := filepath.Walk(packagePath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + log.AppLogger().WithError(err).Error("Walk package path failed") + return err + } if info == nil { return fmt.Errorf("file info is nil") } @@ -45,32 +47,31 @@ func main() { return nil } relativePath := strings.Replace(path, packagePath, "", 1) - println("relativePath: ", relativePath) + log.AppLogger().WithField("relativePath", relativePath).Debug("Read file from package") if filepath.Base(relativePath) == "problem.yaml" { resultMap := make(map[string]interface{}) yamlFile, err := os.ReadFile(path) if err != nil { - log.Println(err) + log.AppLogger().WithError(err).Error("Read problem.yaml failed") } err = yaml.Unmarshal(yamlFile, &resultMap) if err != nil { - log.Printf("Unmarshal: %v\n", err) + log.AppLogger().WithError(err).Error("Unmarshal problem.yaml failed") } title = resultMap["name"].(string) if title == "" { - log.Fatal("name key not exist in problem.yaml") + log.AppLogger().Error("Problem title is empty") } slug = strings.Split(relativePath, "/")[1] - log.Println("title: ", title) - log.Println("slug: ", slug) + log.AppLogger().WithField("title", title).WithField("slug", slug).Debug("Read problem.yaml") } if filepath.Base(relativePath) == "problem.md" { content, err := os.ReadFile(path) if err != nil { - log.Println(err) + log.AppLogger().WithError(err).Error("Read problem.md failed") } description := string(content) - println("description: ", description) + log.AppLogger().WithField("description", description).Debug("Read problem.md") err = problem_model.CreateProblem(db, problem_model.Problem{ Slug: slug, Title: title, @@ -84,12 +85,12 @@ func main() { } } - _, minioErr := minioClient.FPutObject(ctx, minioAgent.GetBucketName(), + _, err = minioClient.FPutObject(ctx, minioAgent.GetBucketName(), relativePath, path, minio.PutObjectOptions{}) - if minioErr != nil { - log.Fatalln(minioErr) + if err != nil { + log.AppLogger().WithError(err).Error("Put object to minio failed") } return err }) @@ -97,5 +98,5 @@ func main() { panic(err) } - log.Println("Read Problem Success!") + log.AppLogger().Info("Problem loaded") } diff --git a/cmd/web_server/handler/judge.go b/cmd/web_server/handler/judge.go index 99c88ee..272cd99 100644 --- a/cmd/web_server/handler/judge.go +++ b/cmd/web_server/handler/judge.go @@ -54,7 +54,7 @@ func postReportJudgeTaskResult(ginCtx *gin.Context) { return } - if err := judge_service.ReportJudgeTaskResult(ginCtx, body.Consumer, body.StreamID, body.VerdictJson); err != nil { + if err := judge_service.ReportJudgeTask(ginCtx, body.Consumer, body.StreamID, body.VerdictJson); err != nil { _ = ginCtx.Error(err) return } diff --git a/cmd/web_server/handler/problem.go b/cmd/web_server/handler/problem.go index 3a7440c..ccecb93 100644 --- a/cmd/web_server/handler/problem.go +++ b/cmd/web_server/handler/problem.go @@ -168,10 +168,10 @@ func checkProblemSlug(ginCtx *gin.Context) { // // @Description The body of a submission request, containing the code and the language used for the submission. // @Property code (string) required "The source code of the submission" minlength(1) -// @Property language (SubmissionLanguage) required "The programming language used for the submission" +// @Property language (ProgrammingLanguage) required "The programming language used for the submission" type PostSubmissionBody struct { - Code string `json:"code" binding:"required"` - Language judge_model.SubmissionLanguage `json:"language" binding:"required"` + Code string `json:"code" binding:"required"` + Language judge_model.ProgrammingLanguage `json:"language" binding:"required"` } // postSubmission @@ -191,8 +191,8 @@ func postSubmission(ginCtx *gin.Context) { return } - submission := judge_model.NewSubmission("", slug, body.Code, body.Language) - result, err := judge_service.CreateJudgeTaskSubmission(ginCtx, submission) + submission := judge_model.NewJudge("", slug, body.Code, body.Language) + result, err := judge_service.CreateJudge(ginCtx, submission) if err != nil { modules.NewInternalError(err.Error()).AppendToGin(ginCtx) return diff --git a/cmd/web_server/handler/submission.go b/cmd/web_server/handler/submission.go index d8f09e5..bd3db01 100644 --- a/cmd/web_server/handler/submission.go +++ b/cmd/web_server/handler/submission.go @@ -22,7 +22,7 @@ func SetupSubmissionRouter(baseRoute *gin.RouterGroup) { func getSubmission(ginCtx *gin.Context) { uid := ginCtx.Param("uid") - submission, err := judge_service.GetJudgeTaskSubmission(ginCtx, uid) + submission, err := judge_service.GetJudge(ginCtx, uid) if err != nil { modules.NewInternalError(fmt.Sprintf("failed to get submission: %v", err)).AppendToGin(ginCtx) return @@ -38,14 +38,13 @@ func getSubmission(ginCtx *gin.Context) { "code": submission.Code, "language": submission.Language, "status": submission.Status, - "verdictJson": submission.VerdictJson, "mainResult": submission.MainResult, }) } type getSubmissionListResponse struct { - Total int64 `json:"total"` - List []*judge_model.JudgeTaskSubmission `json:"list"` + Total int64 `json:"total"` + List []*judge_model.Judge `json:"list"` } // Get Submission List @@ -78,13 +77,13 @@ func getSubmissionList(ginCtx *gin.Context) { return } - options := judge_model.GetSubmissionOptions{ + options := judge_model.GetJudgeOptions{ Limit: &limit, Offset: &offset, OrderByColumns: []models.OrderByColumnOption{{Column: "create_at", Desc: true}}, } - submissions, total, err := judge_service.GetJudgeTaskSubmissionList(ginCtx, options) + submissions, total, err := judge_service.GetJudgeList(ginCtx, options) if err != nil { modules.NewInternalError(fmt.Sprintf("failed to get submission list: %v", err)).AppendToGin(ginCtx) return diff --git a/go.mod b/go.mod index fcf7c18..6b91f73 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( gorm.io/gorm v1.25.10 ) -require github.com/swaggo/swag v1.16.3 // indirect +require github.com/swaggo/swag v1.16.3 require ( github.com/bytedance/sonic/loader v0.1.1 // indirect diff --git a/models/judge/judge.go b/models/judge/judge.go index 68858e1..ceec129 100644 --- a/models/judge/judge.go +++ b/models/judge/judge.go @@ -1,114 +1,70 @@ -package judge +package judge_model import ( - "strings" - + "github.com/google/uuid" "github.com/oj-lab/oj-lab-platform/models" + problem_model "github.com/oj-lab/oj-lab-platform/models/problem" + user_model "github.com/oj-lab/oj-lab-platform/models/user" ) -// Should contains a priority definition -// Ex. CompileError > RuntimeError > TimeLimitExceeded > MemoryLimitExceeded > SystemError > WrongAnswer > Accepted -type JudgeVerdict string +type JudgeTaskStatus string const ( - JudgeVerdictCompileError JudgeVerdict = "CompileError" // Only for main verdict - JudgeVerdictRuntimeError JudgeVerdict = "RuntimeError" - JudgeVerdictTimeLimitExceeded JudgeVerdict = "TimeLimitExceeded" - JudgeVerdictMemoryLimitExceeded JudgeVerdict = "MemoryLimitExceeded" - JudgeVerdictSystemError JudgeVerdict = "SystemError" // Some runtime unknown error ? - JudgeVerdictWrongAnswer JudgeVerdict = "WrongAnswer" - JudgeVerdictAccepted JudgeVerdict = "Accepted" - JudgeVerdictCancelled JudgeVerdict = "cancelled" // Judge will be cancelled if some point results in Runtime error, Time limit exceeded, Memory limit exceeded + JudgeTaskStatusPending JudgeTaskStatus = "pending" + JudgeTaskStatusWaiting JudgeTaskStatus = "waiting" + JudgeTaskStatusRunning JudgeTaskStatus = "running" + JudgeTaskStatusFinished JudgeTaskStatus = "finished" ) -type JudgeResult struct { - MainVerdict JudgeVerdict `json:"verdict"` // A merge of all TestPoints' verdict, according to the pirority - Detail string `json:"detail"` // A brief description of the result - TestPointCount uint64 `json:"testPointCount"` // Won't be stored in database - TestPointMap map[string]TestPoint `json:"testPoints"` // Won't be stored in database - TestPointsJson string `json:"-"` // Used to store TestPoints in database - AverageTimeMs uint64 `json:"averageTimeMs"` // Won't be stored in database - MaxTimeMs uint64 `json:"maxTimeMs"` // Won't be stored in database - AverageMemory uint64 `json:"averageMemory"` // Won't be stored in database - MaxMemory uint64 `json:"maxMemory"` // Won't be stored in database -} - -type TestPoint struct { - Index string `json:"index"` // The name of *.in/ans file - Verdict JudgeVerdict `json:"verdict"` - Diff *ResultDiff `json:"diff"` // Required if verdict is wrong_answer - TimeUsageMs uint64 `json:"timeUsageMs"` - MemoryUsageByte uint64 `json:"memoryUsageByte"` -} +type ProgrammingLanguage string -type ResultDiff struct { - Expected string `json:"expected"` - Received string `json:"received"` +func (sl ProgrammingLanguage) String() string { + return string(sl) } -type JudgerState string - const ( - JudgerStateIdle JudgerState = "idle" - JudgerStateBusy JudgerState = "busy" - JudgerStateOffline JudgerState = "offline" + ProgrammingLanguageCpp ProgrammingLanguage = "Cpp" + ProgrammingLanguageRust ProgrammingLanguage = "Rust" + ProgrammingLanguagePython ProgrammingLanguage = "Python" ) -type Judger struct { +// Using relationship according to https://gorm.io/docs/belongs_to.html +type Judge struct { models.MetaFields - Host string `gorm:"primaryKey" json:"host"` - State JudgerState `gorm:"default:offline" json:"status"` -} - -type JudgeTask struct { - SubmissionUID string `json:"submissionUID"` - ProblemSlug string `json:"problemSlug"` - Code string `json:"code"` - Language string `json:"language"` - RedisStreamID *string `json:"redisStreamID"` -} - -func (jt *JudgeTask) ToStringMap() map[string]interface{} { - return map[string]interface{}{ - "submission_uid": jt.SubmissionUID, - "problem_slug": jt.ProblemSlug, - "code": jt.Code, - "language": jt.Language, - } -} - -func JudgeTaskFromMap(m map[string]interface{}) *JudgeTask { - return &JudgeTask{ - SubmissionUID: m["submission_uid"].(string), - ProblemSlug: m["problem_slug"].(string), - Code: m["code"].(string), - Language: m["language"].(string), - } + UID uuid.UUID `json:"UID" gorm:"primaryKey"` + RedisStreamID string `json:"redisStreamID"` + UserAccount string `json:"userAccount" gorm:"not null"` + User user_model.User `json:"user"` + ProblemSlug string `json:"problemSlug" gorm:"not null"` + Problem problem_model.Problem `json:"problem"` + Code string `json:"code" gorm:"not null"` + Language ProgrammingLanguage `json:"language" gorm:"not null"` + Status JudgeTaskStatus `json:"status" gorm:"default:pending"` + JudgeResultCount uint `json:"judgeResultCount"` + JudgeResults []JudgeResult `json:"judgeResults" gorm:"foreignKey:JudgeUID"` + MainResult JudgeVerdict `json:"mainResult"` } -func (js JudgerState) CanUpdate(nextStatus JudgerState) bool { - switch js { - case JudgerStateOffline: - return nextStatus == JudgerStateIdle - case JudgerStateIdle: - return nextStatus == JudgerStateBusy || nextStatus == JudgerStateOffline - case JudgerStateBusy: - return nextStatus == JudgerStateIdle || nextStatus == JudgerStateOffline - default: - return false +func NewJudge( + userAccount string, + problemSlug string, + code string, + language ProgrammingLanguage, +) Judge { + return Judge{ + UserAccount: userAccount, + ProblemSlug: problemSlug, + Code: code, + Language: language, + Status: JudgeTaskStatusPending, } } -func StringToJudgerState(state string) JudgerState { - state = strings.ToLower(state) - switch state { - case "idle": - return JudgerStateIdle - case "busy": - return JudgerStateBusy - case "offline": - return JudgerStateOffline - default: - return JudgerStateOffline +func (s *Judge) ToJudgeTask() JudgeTask { + return JudgeTask{ + JudgeUID: s.UID.String(), + ProblemSlug: s.ProblemSlug, + Code: s.Code, + Language: s.Language.String(), } } diff --git a/models/judge/judge_db.go b/models/judge/judge_db.go new file mode 100644 index 0000000..feb8bfe --- /dev/null +++ b/models/judge/judge_db.go @@ -0,0 +1,129 @@ +package judge_model + +import ( + "fmt" + + "github.com/google/uuid" + "github.com/oj-lab/oj-lab-platform/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func CreateJudge(tx *gorm.DB, judge Judge) (*Judge, error) { + judge.UID = uuid.New() + judge.MetaFields = models.NewMetaFields() + if judge.UserAccount == "" { + judge.UserAccount = "anonymous" + } + + return &judge, tx.Create(&judge).Error +} + +func GetJudge(tx *gorm.DB, uid string) (*Judge, error) { + db_judge := Judge{} + err := tx.Model(&Judge{}). + Preload("User").Preload("Problem").Where("UID = ?", uid).First(&db_judge).Error + if err != nil { + return nil, err + } + + return &db_judge, nil +} + +type GetJudgeOptions struct { + Selection []string + Statuses []JudgeTaskStatus + UserAccount *string + ProblemSlug *string + Offset *int + Limit *int + OrderByColumns []models.OrderByColumnOption +} + +func buildGetJudgeTXByOptions( + tx *gorm.DB, options GetJudgeOptions, isCount bool, +) *gorm.DB { + tx = tx.Model(&Judge{}). + Preload(clause.Associations) + // See more in: https://gorm.io/docs/preload.html + // Preload("User.Roles").Preload("Problem.Tags").Preload(clause.Associations) + if len(options.Selection) > 0 { + tx = tx.Select(options.Selection) + } + if options.UserAccount != nil { + tx = tx.Where("user_account = ?", *options.UserAccount) + } + if options.ProblemSlug != nil { + tx = tx.Where("problem_slug = ?", *options.ProblemSlug) + } + if len(options.Statuses) > 0 { + tx = tx.Where("status IN ?", options.Statuses) + } + + if !isCount { + if options.Offset != nil { + tx = tx.Offset(*options.Offset) + } + if options.Limit != nil { + tx = tx.Limit(*options.Limit) + } + for _, option := range options.OrderByColumns { + tx = tx.Order(clause.OrderByColumn{ + Column: clause.Column{Name: option.Column}, + Desc: option.Desc, + }) + } + } + + return tx +} + +func GetJudgeListByOptions( + tx *gorm.DB, options GetJudgeOptions, +) ([]*Judge, int64, error) { + tx = buildGetJudgeTXByOptions(tx, options, false) + var judges []*Judge + err := tx.Find(&judges).Error + if err != nil { + return nil, 0, err + } + + tx = buildGetJudgeTXByOptions(tx, options, true) + var count int64 + err = tx.Count(&count).Error + if err != nil { + return nil, 0, err + } + + return judges, count, nil +} + +func UpdateJudge(tx *gorm.DB, judge Judge) error { + updatingJudge := Judge{} + if judge.UID != uuid.Nil { + err := tx.Where("uid = ?", judge.UID).First(&updatingJudge).Error + if err != nil { + return err + } + } else if judge.RedisStreamID != "" { + err := tx.Where("redis_stream_id = ?", judge.RedisStreamID). + First(&updatingJudge).Error + if err != nil { + return err + } + } else { + return fmt.Errorf("judge uid and redis stream id are both empty") + } + + if judge.Status != "" { + updatingJudge.Status = judge.Status + } + if judge.RedisStreamID != "" { + updatingJudge.RedisStreamID = judge.RedisStreamID + } + if judge.MainResult != "" { + updatingJudge.MainResult = judge.MainResult + } + + return tx.Model(&updatingJudge).Updates(updatingJudge).Error +} diff --git a/models/judge/judge_result.go b/models/judge/judge_result.go new file mode 100644 index 0000000..a07ffcf --- /dev/null +++ b/models/judge/judge_result.go @@ -0,0 +1,33 @@ +package judge_model + +import ( + "github.com/google/uuid" + "github.com/oj-lab/oj-lab-platform/models" +) + +// Should contains a priority definition +// Ex. CompileError > RuntimeError > TimeLimitExceeded > MemoryLimitExceeded > SystemError > WrongAnswer > Accepted +type JudgeVerdict string + +const ( + JudgeVerdictCompileError JudgeVerdict = "CompileError" // Only for main verdict + JudgeVerdictRuntimeError JudgeVerdict = "RuntimeError" + JudgeVerdictTimeLimitExceeded JudgeVerdict = "TimeLimitExceeded" + JudgeVerdictMemoryLimitExceeded JudgeVerdict = "MemoryLimitExceeded" + JudgeVerdictSystemError JudgeVerdict = "SystemError" // Some runtime unknown error ? + JudgeVerdictWrongAnswer JudgeVerdict = "WrongAnswer" + JudgeVerdictAccepted JudgeVerdict = "Accepted" + JudgeVerdictCancelled JudgeVerdict = "cancelled" // Judge will be cancelled if some point results in Runtime error, Time limit exceeded, Memory limit exceeded +) + +type JudgeResult struct { + models.MetaFields + UID uuid.UUID `json:"UID" gorm:"primaryKey"` + JudgeUID uuid.UUID `json:"judgeUID"` + Verdict JudgeVerdict `json:"verdict"` + TimeUsageMs uint64 `json:"timeUsageMs"` + MemoryUsageByte uint64 `json:"memoryUsageByte"` + Output string `json:"output"` + ExpectedOutput string `json:"expectedOutput"` + SystemOutput string `json:"systemOutput"` +} diff --git a/models/judge/judge_result_db.go b/models/judge/judge_result_db.go new file mode 100644 index 0000000..14de8ef --- /dev/null +++ b/models/judge/judge_result_db.go @@ -0,0 +1,14 @@ +package judge_model + +import ( + "github.com/google/uuid" + "github.com/oj-lab/oj-lab-platform/models" + "gorm.io/gorm" +) + +func CreateJudgeResult(tx *gorm.DB, result JudgeResult) (*JudgeResult, error) { + result.UID = uuid.New() + result.MetaFields = models.NewMetaFields() + + return &result, tx.Create(&result).Error +} diff --git a/models/judge/judge_task.go b/models/judge/judge_task.go new file mode 100644 index 0000000..4fd667a --- /dev/null +++ b/models/judge/judge_task.go @@ -0,0 +1,27 @@ +package judge_model + +type JudgeTask struct { + JudgeUID string `json:"JudgeUID"` + ProblemSlug string `json:"problemSlug"` + Code string `json:"code"` + Language string `json:"language"` + RedisStreamID *string `json:"redisStreamID"` +} + +func (jt *JudgeTask) ToStringMap() map[string]interface{} { + return map[string]interface{}{ + "judge_uid": jt.JudgeUID, + "problem_slug": jt.ProblemSlug, + "code": jt.Code, + "language": jt.Language, + } +} + +func JudgeTaskFromMap(m map[string]interface{}) *JudgeTask { + return &JudgeTask{ + JudgeUID: m["judge_uid"].(string), + ProblemSlug: m["problem_slug"].(string), + Code: m["code"].(string), + Language: m["language"].(string), + } +} diff --git a/services/judge/judge_stream.go b/models/judge/judge_task_stream.go similarity index 62% rename from services/judge/judge_stream.go rename to models/judge/judge_task_stream.go index ac824fe..5590d48 100644 --- a/services/judge/judge_stream.go +++ b/models/judge/judge_task_stream.go @@ -1,9 +1,8 @@ -package judge +package judge_model import ( "context" - judge_model "github.com/oj-lab/oj-lab-platform/models/judge" redis_agent "github.com/oj-lab/oj-lab-platform/modules/agent/redis" "github.com/redis/go-redis/v9" ) @@ -16,13 +15,16 @@ const ( func init() { redisAgent := redis_agent.GetDefaultRedisClient() - _, err := redisAgent.XGroupCreateMkStream(context.Background(), streamName, consumerGroupName, "0").Result() - if err != nil && err != redis.Nil && err.Error() != "BUSYGROUP Consumer Group name already exists" { + _, err := redisAgent.XGroupCreateMkStream( + context.Background(), streamName, consumerGroupName, "0").Result() + if err != nil && + err != redis.Nil && + err.Error() != "BUSYGROUP Consumer Group name already exists" { panic(err) } } -func addTaskToStream(ctx context.Context, task *judge_model.JudgeTask) (*string, error) { +func AddTaskToStream(ctx context.Context, task *JudgeTask) (*string, error) { redisAgent := redis_agent.GetDefaultRedisClient() id, err := redisAgent.XAdd(ctx, &redis.XAddArgs{ Stream: streamName, @@ -35,7 +37,7 @@ func addTaskToStream(ctx context.Context, task *judge_model.JudgeTask) (*string, return &id, err } -func getTaskFromStream(ctx context.Context, consumer string) (*judge_model.JudgeTask, error) { +func GetTaskFromStream(ctx context.Context, consumer string) (*JudgeTask, error) { redisAgent := redis_agent.GetDefaultRedisClient() if consumer == "" { consumer = defaultConsumerName @@ -55,21 +57,18 @@ func getTaskFromStream(ctx context.Context, consumer string) (*judge_model.Judge return nil, nil } - task := judge_model.JudgeTask{} + task := JudgeTask{} for _, message := range result[0].Messages { - task = *judge_model.JudgeTaskFromMap(message.Values) + task = *JudgeTaskFromMap(message.Values) task.RedisStreamID = &message.ID } return &task, nil } -func ackTaskFromStream(ctx context.Context, consumer string, streamID string) error { +func AckTaskFromStream(ctx context.Context, streamID string) error { redisAgent := redis_agent.GetDefaultRedisClient() - // TODO: Some ineffectual assignment here, need to find out why - // if consumer == "" { - // consumer = defaultConsumerName - // } + _, err := redisAgent.XAck(ctx, streamName, consumerGroupName, streamID).Result() if err != nil { return err diff --git a/models/judge/submission.go b/models/judge/submission.go deleted file mode 100644 index d8b2463..0000000 --- a/models/judge/submission.go +++ /dev/null @@ -1,69 +0,0 @@ -package judge - -import ( - "github.com/google/uuid" - "github.com/oj-lab/oj-lab-platform/models" - problem_model "github.com/oj-lab/oj-lab-platform/models/problem" - user_model "github.com/oj-lab/oj-lab-platform/models/user" -) - -type SubmissionStatus string - -const ( - SubmissionStatusPending SubmissionStatus = "pending" - SubmissionStatusWaiting SubmissionStatus = "waiting" - SubmissionStatusRunning SubmissionStatus = "running" - SubmissionStatusFinished SubmissionStatus = "finished" -) - -type SubmissionLanguage string - -func (sl SubmissionLanguage) String() string { - return string(sl) -} - -const ( - SubmissionLanguageCpp SubmissionLanguage = "Cpp" - SubmissionLanguageRust SubmissionLanguage = "Rust" - SubmissionLanguagePython SubmissionLanguage = "Python" -) - -// Using relationship according to https://gorm.io/docs/belongs_to.html -type JudgeTaskSubmission struct { - models.MetaFields - UID uuid.UUID `gorm:"primaryKey" json:"UID"` - RedisStreamID string `json:"redisStreamID"` - UserAccount string `gorm:"not null" json:"userAccount"` - User user_model.User `json:"user"` - ProblemSlug string `gorm:"not null" json:"problemSlug"` - Problem problem_model.Problem `json:"problem"` - Code string `gorm:"not null" json:"code"` - Language SubmissionLanguage `gorm:"not null" json:"language"` - Status SubmissionStatus `gorm:"default:pending" json:"status"` - VerdictJson string `json:"verdictJson"` - MainResult JudgeVerdict `json:"mainResult"` -} - -func NewSubmission( - userAccount string, - problemSlug string, - code string, - language SubmissionLanguage, -) JudgeTaskSubmission { - return JudgeTaskSubmission{ - UserAccount: userAccount, - ProblemSlug: problemSlug, - Code: code, - Language: language, - Status: SubmissionStatusPending, - } -} - -func (s *JudgeTaskSubmission) ToJudgeTask() JudgeTask { - return JudgeTask{ - SubmissionUID: s.UID.String(), - ProblemSlug: s.ProblemSlug, - Code: s.Code, - Language: s.Language.String(), - } -} diff --git a/models/judge/submission_db.go b/models/judge/submission_db.go deleted file mode 100644 index cbb377c..0000000 --- a/models/judge/submission_db.go +++ /dev/null @@ -1,123 +0,0 @@ -package judge - -import ( - "fmt" - - "github.com/google/uuid" - "github.com/oj-lab/oj-lab-platform/models" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -func CreateSubmission(tx *gorm.DB, submission JudgeTaskSubmission) (*JudgeTaskSubmission, error) { - submission.UID = uuid.New() - submission.MetaFields = models.NewMetaFields() - if submission.UserAccount == "" { - submission.UserAccount = "anonymous" - } - - return &submission, tx.Create(&submission).Error -} - -func GetSubmission(tx *gorm.DB, uid string) (*JudgeTaskSubmission, error) { - db_submission := JudgeTaskSubmission{} - err := tx.Model(&JudgeTaskSubmission{}).Preload("User").Preload("Problem").Where("UID = ?", uid).First(&db_submission).Error - if err != nil { - return nil, err - } - - return &db_submission, nil -} - -type GetSubmissionOptions struct { - Selection []string - Statuses []SubmissionStatus - UserAccount *string - ProblemSlug *string - Offset *int - Limit *int - OrderByColumns []models.OrderByColumnOption -} - -func BuildGetSubmissionTXByOptions(tx *gorm.DB, options GetSubmissionOptions, isCount bool) *gorm.DB { - tx = tx.Model(&JudgeTaskSubmission{}). - Preload(clause.Associations) - // See more in: https://gorm.io/docs/preload.html - // Preload("User.Roles").Preload("Problem.Tags").Preload(clause.Associations) - if len(options.Selection) > 0 { - tx = tx.Select(options.Selection) - } - if options.UserAccount != nil { - tx = tx.Where("user_account = ?", *options.UserAccount) - } - if options.ProblemSlug != nil { - tx = tx.Where("problem_slug = ?", *options.ProblemSlug) - } - if len(options.Statuses) > 0 { - tx = tx.Where("status IN ?", options.Statuses) - } - if options.Offset != nil { - tx = tx.Offset(*options.Offset) - } - if options.Limit != nil { - tx = tx.Limit(*options.Limit) - } - for _, option := range options.OrderByColumns { - tx = tx.Order(clause.OrderByColumn{ - Column: clause.Column{Name: option.Column}, - Desc: option.Desc, - }) - } - - return tx -} - -func GetSubmissionListByOptions(tx *gorm.DB, options GetSubmissionOptions) ([]*JudgeTaskSubmission, int64, error) { - tx = BuildGetSubmissionTXByOptions(tx, options, false) - var submissions []*JudgeTaskSubmission - err := tx.Find(&submissions).Error - if err != nil { - return nil, 0, err - } - - tx = BuildGetSubmissionTXByOptions(tx, options, true) - var count int64 - err = tx.Count(&count).Error - if err != nil { - return nil, 0, err - } - - return submissions, count, nil -} - -func UpdateSubmission(tx *gorm.DB, submission JudgeTaskSubmission) error { - updatingSubmission := JudgeTaskSubmission{} - if submission.UID != uuid.Nil { - err := tx.Where("uid = ?", submission.UID).First(&updatingSubmission).Error - if err != nil { - return err - } - } else if submission.RedisStreamID != "" { - err := tx.Where("redis_stream_id = ?", submission.RedisStreamID).First(&updatingSubmission).Error - if err != nil { - return err - } - } else { - return fmt.Errorf("submission uid and redis stream id are both empty") - } - - if submission.Status != "" { - updatingSubmission.Status = submission.Status - } - if submission.VerdictJson != "" { - updatingSubmission.VerdictJson = submission.VerdictJson - } - if submission.RedisStreamID != "" { - updatingSubmission.RedisStreamID = submission.RedisStreamID - } - if submission.MainResult != "" { - updatingSubmission.MainResult = submission.MainResult - } - - return tx.Model(&updatingSubmission).Updates(updatingSubmission).Error -} diff --git a/models/problem/problem.go b/models/problem/problem.go index 014258d..fd914a9 100644 --- a/models/problem/problem.go +++ b/models/problem/problem.go @@ -6,7 +6,7 @@ type Problem struct { models.MetaFields Slug string `gorm:"primaryKey" json:"slug"` Title string `gorm:"not null" json:"title"` - Description *string `gorm:"not null" json:"description,omitempty"` + Description *string `json:"description,omitempty"` Tags []*AlgorithmTag `gorm:"many2many:problem_algorithm_tags;" json:"tags"` } diff --git a/models/user/user_db.go b/models/user/user_db.go index 7cf65d3..e8cd76e 100644 --- a/models/user/user_db.go +++ b/models/user/user_db.go @@ -36,7 +36,6 @@ func GetUser(tx *gorm.DB, account string) (*User, error) { } func GetPublicUser(tx *gorm.DB, account string) (*User, error) { - db_user := User{} err := tx.Model(&User{}).Preload("Roles").Select(PublicUserSelection).Where("account = ?", account).First(&db_user).Error if err != nil { diff --git a/modules/agent/gorm/database.go b/modules/agent/gorm/database.go index 95705b1..95099b2 100644 --- a/modules/agent/gorm/database.go +++ b/modules/agent/gorm/database.go @@ -1,4 +1,4 @@ -package gormAgent +package gorm_agent import ( "github.com/oj-lab/oj-lab-platform/modules/config" @@ -25,7 +25,9 @@ func GetDefaultDB() *gorm.DB { db, err = gorm.Open(postgres.New(postgres.Config{ DSN: dsn, PreferSimpleProtocol: true, // disables implicit prepared statement usage - }), &gorm.Config{}) + }), &gorm.Config{ + Logger: getLogger(), + }) if err != nil { panic("failed to connect database") } diff --git a/modules/agent/gorm/logger.go b/modules/agent/gorm/logger.go new file mode 100644 index 0000000..4b9e420 --- /dev/null +++ b/modules/agent/gorm/logger.go @@ -0,0 +1,24 @@ +package gorm_agent + +import ( + "log" + "os" + "time" + + "gorm.io/gorm/logger" +) + +func getLogger() logger.Interface { + logger := logger.New( + log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer + logger.Config{ + SlowThreshold: time.Second, // Slow SQL threshold + LogLevel: logger.Silent, // Log level + IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger + ParameterizedQueries: true, // Don't include params in the SQL log + Colorful: false, // Disable color + }, + ) + + return logger +} diff --git a/modules/agent/judger/client.go b/modules/agent/judger/client.go index 52a1207..6a36c54 100644 --- a/modules/agent/judger/client.go +++ b/modules/agent/judger/client.go @@ -1,4 +1,4 @@ -package judgerAgent +package judger_agent type JudgerClient struct { Host string diff --git a/modules/agent/judger/judge.go b/modules/agent/judger/judge.go index b4ce817..b33c7dd 100644 --- a/modules/agent/judger/judge.go +++ b/modules/agent/judger/judge.go @@ -1,4 +1,4 @@ -package judgerAgent +package judger_agent import ( "bytes" diff --git a/modules/agent/judger/state.go b/modules/agent/judger/state.go index 26c5cab..6f117af 100644 --- a/modules/agent/judger/state.go +++ b/modules/agent/judger/state.go @@ -1,4 +1,4 @@ -package judgerAgent +package judger_agent import ( "io" diff --git a/modules/agent/minio/client.go b/modules/agent/minio/client.go index b32adb6..0a99e56 100644 --- a/modules/agent/minio/client.go +++ b/modules/agent/minio/client.go @@ -1,12 +1,12 @@ -package minioAgent +package minio_agent import ( "context" - "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/oj-lab/oj-lab-platform/modules/config" + "github.com/oj-lab/oj-lab-platform/modules/log" ) const ( @@ -54,17 +54,18 @@ func GetMinioClient() *minio.Client { } ctx := context.Background() + exists, err := minioClient.BucketExists(ctx, bucketName) + if err == nil && exists { + log.AppLogger().WithField("bucket", bucketName).Info("Bucket already exists") + return minioClient + } + err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) if err != nil { - // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := minioClient.BucketExists(ctx, bucketName) - if errBucketExists == nil && exists { - log.Printf("We already own %s\n", bucketName) - } else { - log.Fatalln(err) - } + log.AppLogger().WithError(err). + WithField("bucket", bucketName).Error("Failed to create bucket") } else { - log.Printf("Successfully created %s\n", bucketName) + log.AppLogger().WithField("bucket", bucketName).Info("Successfully created bucket") } } diff --git a/modules/agent/minio/local.go b/modules/agent/minio/local.go index 5e9285b..b53db52 100644 --- a/modules/agent/minio/local.go +++ b/modules/agent/minio/local.go @@ -1,4 +1,4 @@ -package minioAgent +package minio_agent import ( "context" diff --git a/modules/agent/redis/client.go b/modules/agent/redis/client.go index e388f48..9f3aa14 100644 --- a/modules/agent/redis/client.go +++ b/modules/agent/redis/client.go @@ -1,4 +1,4 @@ -package redisAgent +package redis_agent import ( "github.com/oj-lab/oj-lab-platform/modules/config" diff --git a/modules/log/log.go b/modules/log/log.go index 18bac59..b84183c 100644 --- a/modules/log/log.go +++ b/modules/log/log.go @@ -12,7 +12,7 @@ const logLevelProp = "log.level" func AppLogger() *logrus.Entry { return logrus.WithFields(logrus.Fields{ - "CALLER": func() string { + "caller": func() string { pc := make([]uintptr, 1) runtime.Callers(3, pc) f := runtime.FuncForPC(pc[0]) @@ -31,6 +31,8 @@ func setupLog() { println("log level:", lvl) logrus.SetLevel(logLevel) } + // TODO: control log format in config + // logrus.SetFormatter(&logrus.JSONFormatter{}) } func init() { diff --git a/services/judge/judge.go b/services/judge/judge.go new file mode 100644 index 0000000..e513286 --- /dev/null +++ b/services/judge/judge.go @@ -0,0 +1,54 @@ +package judge + +import ( + "context" + + judge_model "github.com/oj-lab/oj-lab-platform/models/judge" + gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" +) + +func GetJudge(ctx context.Context, uid string) (*judge_model.Judge, error) { + db := gormAgent.GetDefaultDB() + submission, err := judge_model.GetJudge(db, uid) + if err != nil { + return nil, err + } + + return submission, nil +} + +func GetJudgeList( + ctx context.Context, options judge_model.GetJudgeOptions, +) ([]*judge_model.Judge, int64, error) { + db := gormAgent.GetDefaultDB() + submissions, total, err := judge_model.GetJudgeListByOptions(db, options) + if err != nil { + return nil, 0, err + } + + return submissions, total, nil +} + +func CreateJudge( + ctx context.Context, submission judge_model.Judge, +) (*judge_model.Judge, error) { + db := gormAgent.GetDefaultDB() + newSubmission, err := judge_model.CreateJudge(db, submission) + if err != nil { + return nil, err + } + + task := newSubmission.ToJudgeTask() + streamId, err := judge_model.AddTaskToStream(ctx, &task) + if err != nil { + return nil, err + } + + newSubmission.RedisStreamID = *streamId + err = judge_model.UpdateJudge(db, *newSubmission) + if err != nil { + return nil, err + } + + return newSubmission, nil +} diff --git a/services/judge/task.go b/services/judge/judge_task.go similarity index 88% rename from services/judge/task.go rename to services/judge/judge_task.go index 3c4e3cd..e9fe100 100644 --- a/services/judge/task.go +++ b/services/judge/judge_task.go @@ -12,15 +12,15 @@ import ( ) func PickJudgeTask(ctx context.Context, consumer string) (*judge_model.JudgeTask, error) { - task, err := getTaskFromStream(ctx, consumer) + task, err := judge_model.GetTaskFromStream(ctx, consumer) if err != nil { return nil, fmt.Errorf("failed to get task from stream: %w", err) } db := gorm_agent.GetDefaultDB() - err = judge_model.UpdateSubmission(db, judge_model.JudgeTaskSubmission{ - UID: uuid.MustParse(task.SubmissionUID), - Status: judge_model.SubmissionStatusRunning, + err = judge_model.UpdateJudge(db, judge_model.Judge{ + UID: uuid.MustParse(task.JudgeUID), + Status: judge_model.JudgeTaskStatusRunning, }) if err != nil { return nil, fmt.Errorf("failed to update submission status: %w", err) @@ -29,7 +29,7 @@ func PickJudgeTask(ctx context.Context, consumer string) (*judge_model.JudgeTask return task, nil } -func ReportJudgeTaskResult( +func ReportJudgeTask( ctx context.Context, consumer string, streamID string, verdictJson string, ) error { @@ -39,10 +39,9 @@ func ReportJudgeTaskResult( if err != nil { return err } - err = judge_model.UpdateSubmission(db, judge_model.JudgeTaskSubmission{ + err = judge_model.UpdateJudge(db, judge_model.Judge{ RedisStreamID: streamID, - Status: judge_model.SubmissionStatusFinished, - VerdictJson: verdictJson, + Status: judge_model.JudgeTaskStatusFinished, MainResult: mainVerdict, }) @@ -50,7 +49,7 @@ func ReportJudgeTaskResult( return err } - err = ackTaskFromStream(ctx, consumer, streamID) + err = judge_model.AckTaskFromStream(ctx, streamID) if err != nil { return err } diff --git a/services/judge/submission.go b/services/judge/submission.go deleted file mode 100644 index 5dccf93..0000000 --- a/services/judge/submission.go +++ /dev/null @@ -1,54 +0,0 @@ -package judge - -import ( - "context" - - judge_model "github.com/oj-lab/oj-lab-platform/models/judge" - gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" -) - -func GetJudgeTaskSubmission(ctx context.Context, uid string) (*judge_model.JudgeTaskSubmission, error) { - db := gormAgent.GetDefaultDB() - submission, err := judge_model.GetSubmission(db, uid) - if err != nil { - return nil, err - } - - return submission, nil -} - -func GetJudgeTaskSubmissionList( - ctx context.Context, options judge_model.GetSubmissionOptions, -) ([]*judge_model.JudgeTaskSubmission, int64, error) { - db := gormAgent.GetDefaultDB() - submissions, total, err := judge_model.GetSubmissionListByOptions(db, options) - if err != nil { - return nil, 0, err - } - - return submissions, total, nil -} - -func CreateJudgeTaskSubmission( - ctx context.Context, submission judge_model.JudgeTaskSubmission, -) (*judge_model.JudgeTaskSubmission, error) { - db := gormAgent.GetDefaultDB() - newSubmission, err := judge_model.CreateSubmission(db, submission) - if err != nil { - return nil, err - } - - task := newSubmission.ToJudgeTask() - streamId, err := addTaskToStream(ctx, &task) - if err != nil { - return nil, err - } - - newSubmission.RedisStreamID = *streamId - err = judge_model.UpdateSubmission(db, *newSubmission) - if err != nil { - return nil, err - } - - return newSubmission, nil -} diff --git a/services/user/user.go b/services/user/user.go index e28b42f..c4a7808 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -44,7 +44,10 @@ func CheckUserExist(ctx context.Context, account string) (bool, error) { } if count > 1 { - log.AppLogger().Warnf("user %s has %d records", account, count) + log.AppLogger(). + WithField("account", account). + WithField("count", count). + Warn("user account is not unique") } return count > 0, nil diff --git a/tests/models/judge_test.go b/tests/models/judge_test.go new file mode 100644 index 0000000..6ff5687 --- /dev/null +++ b/tests/models/judge_test.go @@ -0,0 +1,29 @@ +package models_test + +import ( + "testing" + + judge_model "github.com/oj-lab/oj-lab-platform/models/judge" + problem_model "github.com/oj-lab/oj-lab-platform/models/problem" + gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" +) + +func TestJudgeDB(t *testing.T) { + db := gormAgent.GetDefaultDB() + problem := &problem_model.Problem{ + Slug: "test-judge-db-problem", + } + var err error + err = problem_model.CreateProblem(db, *problem) + if err != nil { + t.Error(err) + } + judge := &judge_model.Judge{ + Language: judge_model.ProgrammingLanguageCpp, + ProblemSlug: problem.Slug, + } + _, err = judge_model.CreateJudge(db, *judge) + if err != nil { + t.Error(err) + } +} diff --git a/tests/models/problem_test.go b/tests/models/problem_test.go index f2f4ded..0faa9df 100644 --- a/tests/models/problem_test.go +++ b/tests/models/problem_test.go @@ -9,7 +9,7 @@ import ( gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" ) -func TestProblemMapper(t *testing.T) { +func TestProblemDB(t *testing.T) { db := gormAgent.GetDefaultDB() description := "Given two integer A and B, please output the answer of A+B." problem := problem_model.Problem{ @@ -60,5 +60,4 @@ func TestProblemMapper(t *testing.T) { if err != nil { t.Error(err) } - } diff --git a/tests/models/user_test.go b/tests/models/user_test.go index 4109454..7d5c410 100644 --- a/tests/models/user_test.go +++ b/tests/models/user_test.go @@ -9,7 +9,7 @@ import ( gormAgent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm" ) -func TestUserMapper(t *testing.T) { +func TestUserDB(t *testing.T) { db := gormAgent.GetDefaultDB() user := user_model.User{ Account: "test", @@ -45,5 +45,4 @@ func TestUserMapper(t *testing.T) { if err != nil { t.Error(err) } - }