Merge pull request #98 from oj-lab/zztrans/main

resolve #97  &&  log format with config.

Makefile

@@ -59,12 +59,22 @@ unset-dependencies:
 	docker compose stop postgres redis minio clickhouse
 	docker compose rm -f postgres redis minio clickhouse
 
+.PHONY: unset-data
+unset-data: build
+	./bin/clean
+
 .PHONY: setup-dependencies
-setup-dependencies: unset-dependencies build get-front get-problem-packages
-	docker compose up -d postgres redis minio clickhouse
-	@echo "Wait 10 seconds for db setup"
-	sleep 10s
-	./bin/init
+setup-dependencies: build get-front get-problem-packages
+	@if [ "$(shell docker ps -f status=running | grep -E "postgres|redis|minio|clickhouse" | wc -l)" -eq 4 ]; then \
+		echo "Containers already up"; \
+		./bin/clean; \
+	else \
+		docker compose up -d postgres redis minio clickhouse; \
+		@echo "Wait 10 seconds for db setup"; \
+		sleep 10s; \
+	fi
+	./bin/init;
+
 
 .PHONY: get-front
 get-front:

cmd/clean/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+
+	"github.com/minio/minio-go/v7"
+	casbin_agent "github.com/oj-lab/oj-lab-platform/modules/agent/casbin"
+
+	judge_model "github.com/oj-lab/oj-lab-platform/models/judge"
+	problem_model "github.com/oj-lab/oj-lab-platform/models/problem"
+	user_model "github.com/oj-lab/oj-lab-platform/models/user"
+	gorm_agent "github.com/oj-lab/oj-lab-platform/modules/agent/gorm"
+	minio_agent "github.com/oj-lab/oj-lab-platform/modules/agent/minio"
+
+	log_module "github.com/oj-lab/oj-lab-platform/modules/log"
+)
+
+func clearCasbin() {
+	enforcer := casbin_agent.GetDefaultCasbinEnforcer()
+	enforcer.ClearPolicy() // no err return
+	log_module.AppLogger().Info("Clear Casbin success")
+}
+
+func removeMinioObjects() {
+	minioClient := minio_agent.GetMinioClient()
+	objectsCh := make(chan minio.ObjectInfo)
+
+	go func() {
+		defer close(objectsCh)
+		opts := minio.ListObjectsOptions{Recursive: true}
+		for object := range minioClient.ListObjects(context.Background(), minio_agent.GetBucketName(), opts) {
+			if object.Err != nil {
+				log_module.AppLogger().WithError(object.Err).Error("Get object error")
+			}
+			objectsCh <- object
+		}
+	}()
+
+	errorCh := minioClient.RemoveObjects(context.Background(), minio_agent.GetBucketName(), objectsCh, minio.RemoveObjectsOptions{})
+	for e := range errorCh {
+		log_module.AppLogger().WithError(e.Err).Error("Failed to remove " + e.ObjectName)
+	}
+
+	log_module.AppLogger().Info("Remove Minio Objects success")
+}
+
+func clearDB() {
+	db := gorm_agent.GetDefaultDB()
+
+	err := db.Migrator().DropTable(
+		&user_model.User{},
+		&problem_model.Problem{},
+		&problem_model.AlgorithmTag{},
+		&judge_model.Judge{},
+		&judge_model.JudgeResult{},
+		"problem_algorithm_tags",
+		"casbin_rule",
+	)
+
+	if err != nil {
+		panic("failed to drop tables")
+	}
+
+	log_module.AppLogger().Info("Clear db success")
+
+}
+func main() {
+	removeMinioObjects()
+	clearCasbin()
+	clearDB()
+
+	println("data clean success")
+}

config.toml

@@ -1,5 +1,8 @@
 [log]
 level = "debug"
+pretty_json = true
+time_on = true
+time_format = "2006-01-02 15:04:05"
 
 [database]
 dsn = "user=postgres password=postgres host=localhost port=5432 dbname=oj_lab sslmode=disable TimeZone=Asia/Shanghai"

modules/log/log.go

@@ -3,20 +3,25 @@ package log_module
 import (
 	"os"
 	"runtime"
+	"strconv"
 
 	config_module "github.com/oj-lab/oj-lab-platform/modules/config"
 	"github.com/sirupsen/logrus"
 )
 
 const logLevelProp = "log.level"
+const logPrettyJson = "log.pretty_json"
+const logTimeOn = "log.time_on"
+const logTimeFormat = "log.time_format"
 
 func AppLogger() *logrus.Entry {
 	return logrus.WithFields(logrus.Fields{
 		"caller": func() string {
 			pc := make([]uintptr, 1)
 			runtime.Callers(3, pc)
 			f := runtime.FuncForPC(pc[0])
-			return f.Name()
+			name, line := f.FileLine(pc[0])
+			return name + ":" + strconv.Itoa(line)
 		}(),
 	})
 }
@@ -26,13 +31,21 @@ func setupLog() {
 
 	logrus.SetLevel(logrus.DebugLevel)
 	lvl := config_module.AppConfig().GetString(logLevelProp)
+	prettyJson := config_module.AppConfig().GetBool(logPrettyJson)
+	timeOn := config_module.AppConfig().GetBool(logTimeOn)
+	timestampFormat := config_module.AppConfig().GetString(logTimeFormat)
+
 	logLevel, err := logrus.ParseLevel(lvl)
 	if err == nil {
 		println("log level:", lvl)
 		logrus.SetLevel(logLevel)
 	}
 	// TODO: control log format in config
-	// logrus.SetFormatter(&logrus.JSONFormatter{})
+	logrus.SetFormatter(&logrus.JSONFormatter{
+		TimestampFormat:  timestampFormat,
+		DisableTimestamp: !timeOn,
+		PrettyPrint:      prettyJson,
+	})
 }
 
 func init() {