feat: migrate to uber-go/zap and support loki stack to collect log fi…

…les (#130)

* chore: migrate to uber-go/zap and support loki stack to collect log files
* feat: allow setting log file path
* feat: added complete settings for promtail and tweaked docker-compose settings
* chore: update deps
* chore: cleanup dockerfile
* fix: logrus field incorrectly handled

.env.example

@@ -48,3 +48,6 @@
 # # Redis client cache enabled, default is `false`, read more about client cache at
 # # https://redis.io/docs/manual/client-side-caching/ and https://github.com/redis/rueidis#client-side-caching for more details.
 # REDIS_CLIENT_CACHE_ENABLED=false
+
+# Log level, default is `info`, available options are `debug`, `info`, `warn`, `error`, `fatal`, `panic`
+# LOG_LEVEL=info

Dockerfile

@@ -1,35 +1,31 @@
 # syntax=docker/dockerfile:1
 
-# 设定构建步骤所使用的来源镜像为基于 Debian 发行版的 Go 1.20 版本镜像
+# --- builder ---
+
 FROM golang:1.20 as builder
 
-# 设定 Go 使用 模块化依赖 管理方式：GO111MODULE
 RUN GO111MODULE=on
 
-# 创建路径 /app
 RUN mkdir /app
 
-# 复制当前目录下 insights-bot 到 /app/insights-bot
 COPY . /app/insights-bot
 
-# 切换到 /app/insights-bot 目录
 WORKDIR /app/insights-bot
 
-# Build
 RUN go env
 RUN go env -w CGO_ENABLED=0
 RUN go mod download
 RUN go build -a -o "release/insights-bot" "github.com/nekomeowww/insights-bot/cmd/insights-bot"
 
-# 设定运行步骤所使用的镜像为基于 Debian 发行版镜像
+# --- runner ---
 FROM debian as runner
 
-RUN apt update && apt upgrade -y && apt install -y ca-certificates
+RUN apt update && apt upgrade -y && apt install -y ca-certificates && update-ca-certificates
 
 COPY --from=builder /app/insights-bot/release/insights-bot /usr/local/bin/
 
-# Health check endpoint
+ENV LOG_FILE_PATH /var/log/insights-bot/insights-bot.log
+
 EXPOSE 7069
 
-# 入点是编译好的应用程序
 CMD [ "/usr/local/bin/insights-bot" ]

README.md

@@ -245,7 +245,8 @@ docker buildx build --platform linux/arm64,linux/amd64 -t <tag> -f Dockerfile .
 | `REDIS_PASSWORD`            | `false`     |                                   | Redis password.                                                                                                                                                                                                                                                                                                                                                                                         |
 | `REDIS_DB`                  | `false`     | `0`                               | Redis database, default is `0`                                                                                                                                                                                                                                                                                                                                                                                          |
 | `REDIS_CLIENT_CACHE_ENABLED` | `false`     | `false`                           | Redis client cache enabled, default is `false`, read more about client cache at [https://redis.io/docs/manual/client-side-caching/](https://redis.io/docs/manual/client-side-caching/) and [https://github.com/redis/rueidis#client-side-caching](https://github.com/redis/rueidis#client-side-caching) for more details.                                                                                                                                                                                                                                                                                                                                                                          |
-| `LOG_LEVEL`                 | `false`     | `info`                            | Log level, available values are `trace`, `debug`, `info`, `warn`, `error`                                                                                                                                                                                                                                                                                                                                               |
+| `LOG_FILE_PATH`             | `false`     | `<insights-bot_executable>/logs/insights-bot.log`                           | Log file path, you can specify one if you want to specify a path to store logs when executed and ran with binary. The default path is `/var/log/insights-bot/insights-bot.log` in Docker volume, you can override the defaults `-e LOG_FILE_PATH=<path>` when executing `docker run` command or modify and prepend a new `LOG_FILE_PATH` the `docker-compose.yml` file.                                                                                                                                                                                                                                          |
+| `LOG_LEVEL`                 | `false`     | `info`                            | Log level, available values are `debug`, `info`, `warn`, `error`, `error`                                                                                                                                                                                                                                                                                                                                               |
 | ~~`CLOVER_DB_PATH`~~        | ~~`false`~~ | ~~`insights_bot_clover_data.db`~~ | **Deprecated**. ~~Path to Clover database file, you can specify one if you want to specify a path to store data when executed and ran with binary. The default path is `/var/lib/insights-bot/insights_bot_clover_data.db` in Docker volume, you can override the defaults `-e CLOVER_DB_PATH=<path>` when executing `docker run` command or modify and prepend a new `CLOVER_DB_PATH` the `docker-compose.yml` file.~~ |
 
 ## Acknowledgements

docker-compose.yml

@@ -1,6 +1,9 @@
 version: "3.9"
 
 services:
+  # default insights-bot service that would pull the image from docker hub
+  # and runs on production environment with db_local service and redis_local
+  # service
   insights_bot:
     profiles:
       - hub
@@ -17,6 +20,7 @@ services:
       - REDIS_PORT=6379
       - REDIS_TLS_ENABLED=false
       - REDIS_PASSWORD=123456
+      - LOG_LEVEL
       # # uncomment the following lines if you want to enable discord bot webhook
       # - DISCORD_BOT_TOKEN
       # - DISCORD_BOT_PUBLIC_KEY
@@ -31,13 +35,21 @@ services:
       retries: 5
       start_period: 1s
       timeout: 1s
+    volumes:
+      - insights_bot_logs:/var/log/insights-bot
     depends_on:
       - db_local
       - redis_local
 
+  # On-demand build insights-bot service that would build the image from
+  # source code and runs on production environment with db_local service
+  # and redis_local service
   insights_bot_local:
     profiles:
       - local
+    build:
+      context: .
+      dockerfile: ./Dockerfile
     image: insights-bot
     restart: always
     environment:
@@ -51,6 +63,7 @@ services:
       - REDIS_PORT=6379
       - REDIS_TLS_ENABLED=false
       - REDIS_PASSWORD=123456
+      - LOG_LEVEL
       # # uncomment the following lines if you want to enable discord bot webhook
       # - DISCORD_BOT_TOKEN
       # - DISCORD_BOT_PUBLIC_KEY
@@ -64,13 +77,20 @@ services:
       interval: 10s
       retries: 5
       timeout: 5s
-    build:
-      context: .
-      dockerfile: ./Dockerfile
+    volumes:
+      - insights_bot_logs:/var/log/insights-bot
     depends_on:
       - db_local
       - redis_local
 
+  # the default PostgreSQL database service that helps insights-bot to store data
+  # you can change the password by changing the POSTGRES_PASSWORD environment variable
+  # and change the database name by changing the POSTGRES_DB environment variable
+  # and change the username by changing the POSTGRES_USER environment variable
+  #
+  # NOTICE: if you have your own PostgreSQL database service, you can remove this service
+  # and change the DB_CONNECTION_STR environment variable in insights_bot or
+  # insights_bot_local service
   db_local:
     image: postgres:alpine
     restart: unless-stopped
@@ -86,6 +106,13 @@ services:
       timeout: 5s
       retries: 5
 
+  # the default Redis service that helps insights-bot to store data
+  # you can change the password by changing the REDIS_PASSWORD environment variable
+  # and change the port by changing the REDIS_PORT environment variable
+  #
+  # NOTICE: if you have your own Redis service, you can remove this service
+  # and change the REDIS_HOST and REDIS_PORT environment variable in insights_bot or
+  # insights_bot_local service
   redis_local:
     image: redis:7
     restart: unless-stopped
@@ -97,3 +124,30 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
+
+  # # promtail service that helps insights-bot to collect logs
+  # # you can uncomment the following lines if you want to enable promtail
+  # promtail:
+  #   image: grafana/promtail:2.8.2
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./production/promtail:/etc/promtail # you may edit the ./production/promtail/config.yaml file to suit your needs
+  #     - insights_bot_logs:/var/log/insights-bot:ro
+  #   command: >
+  #     -config.file=/etc/promtail/config.yaml
+
+  #   # uncomment the following lines if you want to view the web ui of promtail
+  #   # ports:
+  #   #   - 9080:9080
+  #   # uncomment the following lines if you want to use your own loki service
+  #   # networks:
+  #   #   - <loki_network> # replace <loki_network> with your own loki network name
+
+volumes:
+  insights_bot_logs:
+
+# uncomment the following lines if you want to use your own loki service
+# that live in another docker-compose.yml file
+# networks:
+#   <loki_network>: # replace <loki_network> with your own loki network name
+#     external: true

go.mod

@@ -32,6 +32,7 @@ require (
 	go.uber.org/fx v1.19.3
 	go.uber.org/multierr v1.11.0
 	go.uber.org/ratelimit v0.2.0
+	go.uber.org/zap v1.24.0
 )
 
 require (
@@ -63,7 +64,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.16.2 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -84,7 +85,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/goleak v1.2.1 // indirect
-	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.9.0 // indirect
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
 	golang.org/x/mod v0.10.0 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNa
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
-github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.6.1 h1:9XE5ykDiC8eNSqIPkxx0EsV3kMX1oe4kQWRZjIgytUA=

internal/bots/discord/listeners/listeners.go

@@ -1,6 +1,8 @@
 package listeners
 
 import (
+	"fmt"
+
 	"github.com/disgoorg/disgo/discord"
 	"github.com/disgoorg/disgo/events"
 	"github.com/nekomeowww/insights-bot/internal/models/smr"
@@ -9,6 +11,7 @@ import (
 	"github.com/nekomeowww/insights-bot/internal/services/smr/types"
 	"github.com/nekomeowww/insights-bot/pkg/logger"
 	"go.uber.org/fx"
+	"go.uber.org/zap"
 )
 
 func NewModules() fx.Option {
@@ -41,7 +44,7 @@ func NewListeners() func(param NewListenersParam) *Listeners {
 func (b *Listeners) smrCmd(event *events.ApplicationCommandInteractionCreate, data discord.SlashCommandInteractionData) {
 	urlString := data.String("link")
 
-	b.logger.Infof("discord: command received: /smr %s", urlString)
+	b.logger.Info(fmt.Sprintf("discord: command received: /smr %s", urlString))
 
 	// url check
 	err, originErr := smrutils.CheckUrl(urlString)
@@ -53,18 +56,15 @@ func (b *Listeners) smrCmd(event *events.ApplicationCommandInteractionCreate, da
 					Build(),
 			)
 			if err != nil {
-				b.logger.WithField("error", err.Error()).Warn("discord: failed to send error message")
+				b.logger.Warn("discord: failed to send error message", zap.Error(err))
 			}
 
 			return
 		}
 
 		err = event.CreateMessage(discord.NewMessageCreateBuilder().SetContent("出现了一些问题，可以再试试？").Build())
 		if err != nil {
-			b.logger.
-				WithError(err).
-				WithError(originErr).
-				Warn("discord: failed to send error message")
+			b.logger.Warn("discord: failed to send error message", zap.Error(err), zap.NamedError("original_error", originErr))
 		}
 
 		return
@@ -75,7 +75,7 @@ func (b *Listeners) smrCmd(event *events.ApplicationCommandInteractionCreate, da
 		SetContent("请稍等，量子速读中...").
 		Build())
 	if err != nil {
-		b.logger.WithField("error", err.Error()).Warn("discord: failed to send response message")
+		b.logger.Warn("discord: failed to send response message", zap.Error(err))
 		return
 	}
 
@@ -85,11 +85,11 @@ func (b *Listeners) smrCmd(event *events.ApplicationCommandInteractionCreate, da
 		ChannelID: event.Channel().ID.String(),
 	})
 	if err != nil {
-		b.logger.WithField("error", err.Error()).Warn("discord: failed to add task")
+		b.logger.Warn("discord: failed to add task", zap.Error(err))
 
 		err = event.CreateMessage(discord.NewMessageCreateBuilder().SetContent("出现了一些问题，可以再试试？").Build())
 		if err != nil {
-			b.logger.WithField("error", err.Error()).Warn("discord: failed to send error message")
+			b.logger.Warn("discord: failed to send error message", zap.Error(err))
 		}
 
 		return

internal/bots/slack/handlers/handlers.go

@@ -2,10 +2,12 @@ package handlers
 
 import (
 	"context"
+	"fmt"
+	"net/http"
+
 	"github.com/nekomeowww/insights-bot/internal/models/smr"
 	"github.com/nekomeowww/insights-bot/internal/services/smr/smrqueue"
 	"github.com/nekomeowww/insights-bot/internal/services/smr/smrutils"
-	"net/http"
 
 	"github.com/gin-gonic/gin"
 	"github.com/nekomeowww/insights-bot/ent"
@@ -16,9 +18,9 @@ import (
 	"github.com/nekomeowww/insights-bot/pkg/bots/slackbot"
 	"github.com/nekomeowww/insights-bot/pkg/bots/slackbot/services"
 	"github.com/nekomeowww/insights-bot/pkg/logger"
-	"github.com/sirupsen/logrus"
 	"github.com/slack-go/slack"
 	"go.uber.org/fx"
+	"go.uber.org/zap"
 )
 
 func NewModules() fx.Option {
@@ -70,15 +72,15 @@ func (h *Handlers) PostCommandInfo(ctx *gin.Context) {
 	var body receivedCommandInfo
 	if err := ctx.Bind(&body); err != nil {
 		ctx.AbortWithStatus(http.StatusBadRequest)
-		h.logger.WithField("error", err.Error()).Warn("failed to bind request body, type definition of slack request body may have changed")
+		h.logger.Warn("failed to bind request body, type definition of slack request body may have changed", zap.Error(err))
 
 		return
 	}
 
-	h.logger.WithFields(logrus.Fields{
-		"user_id":    body.UserID,
-		"channel_id": body.ChannelID,
-	}).Tracef("slack: command received: /smr %s", body.Text)
+	h.logger.Debug(fmt.Sprintf("slack: command received: /smr %s", body.Text),
+		zap.String("user_id", body.UserID),
+		zap.String("channel_id", body.ChannelID),
+	)
 
 	urlString := body.Text
 
@@ -90,10 +92,7 @@ func (h *Handlers) PostCommandInfo(ctx *gin.Context) {
 		}
 
 		ctx.JSON(http.StatusOK, slackbot.NewSlackWebhookMessage("出现了一些问题，可以再试试？"))
-		h.logger.
-			WithError(err).
-			WithError(originErr).
-			Warn("discord: failed to send error message")
+		h.logger.Warn("discord: failed to send error message", zap.Error(err), zap.NamedError("original_error", originErr))
 
 		return
 	}
@@ -103,7 +102,7 @@ func (h *Handlers) PostCommandInfo(ctx *gin.Context) {
 		slackoauthcredentials.TeamID(body.TeamID),
 	).First(context.Background())
 	if err != nil {
-		h.logger.WithField("error", err.Error()).Warn("slack: failed to get team's access token")
+		h.logger.Warn("slack: failed to get team's access token", zap.Error(err))
 		if ent.IsNotFound(err) {
 			ctx.JSON(http.StatusOK, slackbot.NewSlackWebhookMessage("本应用没有权限向这个频道发送消息，尝试重新安装一下？"))
 			return
@@ -122,7 +121,7 @@ func (h *Handlers) PostCommandInfo(ctx *gin.Context) {
 		TeamID:    body.TeamID,
 	})
 	if err != nil {
-		h.logger.WithError(err).Warn("slack: failed to add task")
+		h.logger.Warn("slack: failed to add task", zap.Error(err))
 		ctx.JSON(http.StatusOK, slackbot.NewSlackWebhookMessage("量子速读请求发送失败了，可以再试试？"))
 
 		return
@@ -142,7 +141,7 @@ func (h *Handlers) GetInstallAuth(ctx *gin.Context) {
 
 	resp, err := slack.GetOAuthV2Response(&http.Client{}, h.config.Slack.ClientID, h.config.Slack.ClientSecret, code, "")
 	if err != nil {
-		h.logger.WithError(err).Error("slack: failed to get access token, interrupt")
+		h.logger.Error("slack: failed to get access token, interrupt", zap.Error(err))
 		ctx.AbortWithStatus(http.StatusServiceUnavailable)
 
 		return