From 4570a1cdc291f8cb518bc167e78abffe50abf952 Mon Sep 17 00:00:00 2001 From: Anton Prokhorov Date: Sun, 21 Jul 2024 20:54:36 +0100 Subject: [PATCH] Better server configuration Switched to viper for configuration --- .air.toml | 2 +- Dockerfile | 2 +- Dockerfile.ci | 2 +- cmd/server/main.go | 95 ++++++++++++--- go.mod | 3 +- go.sum | 2 - internal/actionsdb/actions_test.go | 2 +- internal/cmd/server/config.go | 108 ++++++++++++++++-- internal/cmd/server/dns.go | 9 -- internal/cmd/server/modules.go | 8 +- internal/cmd/server/tls.go | 50 -------- internal/database/config.go | 15 --- internal/database/db.go | 4 +- internal/database/db_test.go | 2 +- internal/dnsdb/dnsdb_test.go | 2 +- internal/httpdb/httpdb_test.go | 2 +- internal/modules/api/api_test.go | 2 +- internal/modules/api/apiclient/client_test.go | 2 +- internal/modules/api/config.go | 4 +- internal/modules/lark/config.go | 16 +-- internal/modules/telegram/config.go | 6 +- internal/utils/struct.go | 31 +++++ 22 files changed, 238 insertions(+), 131 deletions(-) delete mode 100644 internal/database/config.go create mode 100644 internal/utils/struct.go diff --git a/.air.toml b/.air.toml index 171728b..6c6aa9b 100644 --- a/.air.toml +++ b/.air.toml @@ -3,7 +3,7 @@ testdata_dir = "testdata" tmp_dir = ".tmp" [build] - args_bin = [] + args_bin = ["serve", "--config", "server.toml"] bin = "./.tmp/server" cmd = "go build -o ./.tmp/server ./cmd/server/" delay = 1000 diff --git a/Dockerfile b/Dockerfile index 2b0646f..5d751bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,4 @@ RUN apk add --no-cache ca-certificates WORKDIR /opt/app COPY --from=builder /opt/app/server . EXPOSE 53/udp 21 25 80 443 -CMD ["./server"] +CMD ["./server", "serve"] diff --git a/Dockerfile.ci b/Dockerfile.ci index b2a2221..3113354 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -3,4 +3,4 @@ RUN apk add --no-cache ca-certificates WORKDIR /opt/app COPY server /opt/app/ EXPOSE 53/udp 25 80 443 -CMD ["./server"] +CMD ["./server", "serve"] diff --git a/cmd/server/main.go b/cmd/server/main.go index 77542f5..ab9ef8d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,27 +2,69 @@ package main import ( "database/sql" + "errors" "fmt" "net" + "strings" "sync" "time" - "github.com/kelseyhightower/envconfig" + validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/nt0xa/sonar/internal/actionsdb" "github.com/nt0xa/sonar/internal/cache" "github.com/nt0xa/sonar/internal/cmd/server" "github.com/nt0xa/sonar/internal/database" "github.com/nt0xa/sonar/internal/database/models" + "github.com/nt0xa/sonar/internal/utils" "github.com/nt0xa/sonar/pkg/dnsx" "github.com/nt0xa/sonar/pkg/ftpx" "github.com/nt0xa/sonar/pkg/httpx" "github.com/nt0xa/sonar/pkg/smtpx" ) +func init() { + validation.ErrorTag = "mapstructure" +} + func main() { + var ( + cfg server.Config + cfgFile string + ) + root := &cobra.Command{ + Use: "server", + Short: "CLI for sonar server", + PersistentPreRunE: func(_ *cobra.Command, _ []string) error { + if err := initConfig(cfgFile, &cfg); err != nil { + return fmt.Errorf("fail to init config: %w", err) + } + + return nil + }, + SilenceUsage: true, + } + + root.PersistentFlags().StringVar(&cfgFile, "config", "", "config file") + + serve := &cobra.Command{ + Use: "serve", + Short: "start the server", + Run: func(cmd *cobra.Command, args []string) { + serve(&cfg) + }, + } + + root.AddCommand(serve) + + root.Execute() +} + +func serve(cfg *server.Config) { // // Logger // @@ -30,25 +72,11 @@ func main() { log := logrus.New() log.SetFormatter(&logrus.TextFormatter{}) - // - // Config - // - - var cfg server.Config - - if err := envconfig.Process("sonar", &cfg); err != nil { - log.Fatal(err.Error()) - } - - if err := cfg.Validate(); err != nil { - log.Fatal(err) - } - // // DB // - db, err := database.New(&cfg.DB, log) + db, err := database.New(cfg.DB.DSN, log) if err != nil { log.Fatal(err) } @@ -276,3 +304,38 @@ func main() { // Wait forever select {} } + +func initConfig(cfgFile string, cfg *server.Config) error { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + viper.SetConfigName("config") + viper.SetConfigType("toml") + viper.AddConfigPath(".") + } + + viper.SetEnvPrefix("sonar") + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AutomaticEnv() + + for _, key := range utils.StructKeys(*cfg, "mapstructure") { + viper.BindEnv(key) + } + + if err := viper.ReadInConfig(); err != nil { + if !errors.As(err, new(viper.ConfigFileNotFoundError)) { + return err + } + } + + if err := viper.Unmarshal(cfg); err != nil { + return err + } + + if err := cfg.Validate(); err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 76d3767..11229e3 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/gorilla/schema v1.4.0 github.com/invopop/jsonschema v0.8.0 github.com/jmoiron/sqlx v1.4.0 - github.com/kelseyhightower/envconfig v1.4.0 github.com/larksuite/oapi-sdk-go/v3 v3.2.7 github.com/lib/pq v1.10.9 github.com/miekg/dns v1.1.61 @@ -28,6 +27,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 ) require ( @@ -95,7 +95,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect diff --git a/go.sum b/go.sum index 4e25816..a6d829a 100644 --- a/go.sum +++ b/go.sum @@ -290,8 +290,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= diff --git a/internal/actionsdb/actions_test.go b/internal/actionsdb/actions_test.go index a6bb0d7..d7124a7 100644 --- a/internal/actionsdb/actions_test.go +++ b/internal/actionsdb/actions_test.go @@ -33,7 +33,7 @@ func TestMain(m *testing.M) { log := logrus.New() - db, err = database.New(&database.Config{DSN: dsn}, log) + db, err = database.New(dsn, log) if err != nil { fmt.Fprintf(os.Stderr, "fail to init database: %v\n", err) os.Exit(1) diff --git a/internal/cmd/server/config.go b/internal/cmd/server/config.go index b409ada..da71075 100644 --- a/internal/cmd/server/config.go +++ b/internal/cmd/server/config.go @@ -3,28 +3,118 @@ package server import ( validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/go-ozzo/ozzo-validation/v4/is" - "github.com/nt0xa/sonar/internal/database" + "github.com/nt0xa/sonar/internal/utils/valid" + "github.com/spf13/viper" ) -type Config struct { - DB database.Config `json:"db"` - Domain string `json:"domain"` - IP string `json:"ip"` +func init() { + viper.SetDefault("tls.letsencrypt.directory", "./tls") + viper.SetDefault("tls.letsencrypt.ca_dir_url", "https://acme-v02.api.letsencrypt.org/directory") + viper.SetDefault("tls.letsencrypt.ca_insecure", false) - DNS DNSConfig `json:"dns"` + viper.SetDefault("modules.enabled", "api") + viper.SetDefault("modules.api.port", 31337) - TLS TLSConfig `json:"tls"` + viper.SetDefault("modules.lark.tls_enabled", true) +} - Modules ModulesConfig `json:"modules"` +type Config struct { + Domain string `mapstructure:"domain"` + IP string `mapstructure:"ip"` + DB DBConfig `mapstructure:"db"` + DNS DNSConfig `mapstructure:"dns"` + TLS TLSConfig `mapstructure:"tls"` + Modules ModulesConfig `mapstructure:"modules"` } func (c Config) Validate() error { return validation.ValidateStruct(&c, - validation.Field(&c.DB), validation.Field(&c.Domain, validation.Required, is.Domain), validation.Field(&c.IP, validation.Required, is.IP), + validation.Field(&c.DB, validation.Required), validation.Field(&c.DNS), validation.Field(&c.TLS), validation.Field(&c.Modules), ) } + +// +// DB +// + +type DBConfig struct { + DSN string `mapstructure:"dsn"` +} + +func (c DBConfig) Validate() error { + return validation.ValidateStruct(&c, + validation.Field(&c.DSN, validation.Required)) +} + +// +// DNS +// + +type DNSConfig struct { + Zone string `mapstructure:"zone"` +} + +func (c DNSConfig) Validate() error { + return validation.ValidateStruct(&c) +} + +// +// TLS +// + +type TLSConfig struct { + Type string `mapstructure:"type"` + Custom TLSCustomConfig `mapstructure:"custom"` + LetsEncrypt TLSLetsEncryptConfig `mapstructure:"letsencrypt"` +} + +func (c TLSConfig) Validate() error { + rules := make([]*validation.FieldRules, 0) + + rules = append(rules, + validation.Field(&c.Type, validation.Required, validation.In("custom", "letsencrypt"))) + + switch c.Type { + case "custom": + rules = append(rules, validation.Field(&c.Custom)) + case "letsencrypt": + rules = append(rules, validation.Field(&c.LetsEncrypt)) + } + + return validation.ValidateStruct(&c, rules...) +} + +// Custom + +type TLSCustomConfig struct { + Key string `mapstructure:"key"` + Cert string `mapstructure:"cert"` +} + +func (c TLSCustomConfig) Validate() error { + return validation.ValidateStruct(&c, + validation.Field(&c.Key, validation.Required, validation.By(valid.File)), + validation.Field(&c.Cert, validation.Required, validation.By(valid.File)), + ) +} + +// LetsEncrypt + +type TLSLetsEncryptConfig struct { + Email string `mapstructure:"email"` + Directory string `mapstructure:"directory"` + CADirURL string `mapstructure:"ca_dir_url"` + CAInsecure bool `mapstructure:"ca_insecure"` +} + +func (c TLSLetsEncryptConfig) Validate() error { + return validation.ValidateStruct(&c, + validation.Field(&c.Email, validation.Required), + validation.Field(&c.Directory, validation.Required, validation.By(valid.Directory)), + ) +} diff --git a/internal/cmd/server/dns.go b/internal/cmd/server/dns.go index 5cd0b03..d8f3160 100644 --- a/internal/cmd/server/dns.go +++ b/internal/cmd/server/dns.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/fatih/structs" - validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/miekg/dns" "github.com/nt0xa/sonar/internal/database" @@ -40,14 +39,6 @@ var dnsTemplate = tpl.MustParse(` * SOA ns1 hostmaster 1337 86400 7200 4000000 11200 `) -type DNSConfig struct { - Zone string `json:"zone"` -} - -func (c DNSConfig) Validate() error { - return validation.ValidateStruct(&c) -} - func parseDNSRecords(s, origin string, ip net.IP) *dnsx.Records { rrs, err := dnsx.ParseRecords(s, origin) if err != nil { diff --git a/internal/cmd/server/modules.go b/internal/cmd/server/modules.go index d094e35..b6b29fd 100644 --- a/internal/cmd/server/modules.go +++ b/internal/cmd/server/modules.go @@ -20,13 +20,13 @@ type Controller interface { } type ModulesConfig struct { - Enabled []string `json:"enabled"` + Enabled []string `mapstructure:"enabled"` // TODO: dynamic modules config (something like json.RawMessage) to be able to not include // unnecessary modules in binary. - Telegram telegram.Config `json:"telegram"` - API api.Config `json:"api"` - Lark lark.Config `json:"lark"` + Telegram telegram.Config `mapstructure:"telegram"` + API api.Config `mapstructure:"api"` + Lark lark.Config `mapstructure:"lark"` } func (c ModulesConfig) Validate() error { diff --git a/internal/cmd/server/tls.go b/internal/cmd/server/tls.go index 1bae657..044aa70 100644 --- a/internal/cmd/server/tls.go +++ b/internal/cmd/server/tls.go @@ -6,60 +6,10 @@ import ( "sync" "github.com/go-acme/lego/v3/challenge" - validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/nt0xa/sonar/internal/utils/logger" - "github.com/nt0xa/sonar/internal/utils/valid" "github.com/nt0xa/sonar/pkg/certmgr" ) -type TLSConfig struct { - Type string `json:"type"` - Custom TLSCustomConfig `json:"custom"` - LetsEncrypt TLSLetsEncryptConfig `json:"letsencrypt"` -} - -type TLSCustomConfig struct { - Key string `json:"key"` - Cert string `json:"cert"` -} - -type TLSLetsEncryptConfig struct { - Email string `json:"email"` - Directory string `json:"directory"` - CADirURL string `json:"caDirUrl" default:"https://acme-v02.api.letsencrypt.org/directory"` - CAInsecure bool `json:"caInsecure"` -} - -func (c TLSConfig) Validate() error { - rules := make([]*validation.FieldRules, 0) - - rules = append(rules, - validation.Field(&c.Type, validation.Required, validation.In("custom", "letsencrypt"))) - - switch c.Type { - case "custom": - rules = append(rules, validation.Field(&c.Custom)) - case "letsencrypt": - rules = append(rules, validation.Field(&c.LetsEncrypt)) - } - - return validation.ValidateStruct(&c, rules...) -} - -func (c TLSCustomConfig) Validate() error { - return validation.ValidateStruct(&c, - validation.Field(&c.Key, validation.Required, validation.By(valid.File)), - validation.Field(&c.Cert, validation.Required, validation.By(valid.File)), - ) -} - -func (c TLSLetsEncryptConfig) Validate() error { - return validation.ValidateStruct(&c, - validation.Field(&c.Email, validation.Required), - validation.Field(&c.Directory, validation.Required, validation.By(valid.Directory)), - ) -} - type TLS struct { cfg *TLSConfig cm *certmgr.CertMgr diff --git a/internal/database/config.go b/internal/database/config.go deleted file mode 100644 index 2318762..0000000 --- a/internal/database/config.go +++ /dev/null @@ -1,15 +0,0 @@ -package database - -import ( - validation "github.com/go-ozzo/ozzo-validation/v4" -) - -type Config struct { - DSN string `json:"dsn"` -} - -func (c Config) Validate() error { - return validation.ValidateStruct(&c, - validation.Field(&c.DSN, validation.Required), - ) -} diff --git a/internal/database/db.go b/internal/database/db.go index 53d25db..d826d9c 100644 --- a/internal/database/db.go +++ b/internal/database/db.go @@ -23,9 +23,9 @@ type DB struct { obserers []Observer } -func New(cfg *Config, log logger.StdLogger) (*DB, error) { +func New(dsn string, log logger.StdLogger) (*DB, error) { - db, err := sqlx.Connect("postgres", cfg.DSN) + db, err := sqlx.Connect("postgres", dsn) if err != nil { return nil, fmt.Errorf("new: fail to connect to database: %w", err) diff --git a/internal/database/db_test.go b/internal/database/db_test.go index 7fd7191..7b63c7d 100644 --- a/internal/database/db_test.go +++ b/internal/database/db_test.go @@ -29,7 +29,7 @@ func TestMain(m *testing.M) { os.Exit(1) } - db, err = database.New(&database.Config{DSN: dsn}, log) + db, err = database.New(dsn, log) if err != nil { fmt.Fprintf(os.Stderr, "fail to init database: %v\n", err) os.Exit(1) diff --git a/internal/dnsdb/dnsdb_test.go b/internal/dnsdb/dnsdb_test.go index 4786c1d..3b4d274 100644 --- a/internal/dnsdb/dnsdb_test.go +++ b/internal/dnsdb/dnsdb_test.go @@ -32,7 +32,7 @@ func TestMain(m *testing.M) { os.Exit(1) } - db, err = database.New(&database.Config{DSN: dsn}, logrus.New()) + db, err = database.New(dsn, logrus.New()) if err != nil { fmt.Fprintf(os.Stderr, "fail to init database: %v\n", err) os.Exit(1) diff --git a/internal/httpdb/httpdb_test.go b/internal/httpdb/httpdb_test.go index 0766cec..dacf78f 100644 --- a/internal/httpdb/httpdb_test.go +++ b/internal/httpdb/httpdb_test.go @@ -39,7 +39,7 @@ func TestMain(m *testing.M) { os.Exit(1) } - db, err = database.New(&database.Config{DSN: dsn}, logrus.New()) + db, err = database.New(dsn, logrus.New()) if err != nil { fmt.Fprintf(os.Stderr, "fail to init database: %v\n", err) os.Exit(1) diff --git a/internal/modules/api/api_test.go b/internal/modules/api/api_test.go index bc9610f..e34c807 100644 --- a/internal/modules/api/api_test.go +++ b/internal/modules/api/api_test.go @@ -68,7 +68,7 @@ func TestMain(m *testing.M) { log := logrus.New() - db, err = database.New(&database.Config{DSN: dsn}, log) + db, err = database.New(dsn, log) if err != nil { fmt.Fprintf(os.Stderr, "fail to init database: %v\n", err) os.Exit(1) diff --git a/internal/modules/api/apiclient/client_test.go b/internal/modules/api/apiclient/client_test.go index 7d52e04..4d93d1b 100644 --- a/internal/modules/api/apiclient/client_test.go +++ b/internal/modules/api/apiclient/client_test.go @@ -67,7 +67,7 @@ func TestMain(m *testing.M) { log := logrus.New() - db, err = database.New(&database.Config{DSN: dsn}, log) + db, err = database.New(dsn, log) if err != nil { fmt.Fprintf(os.Stderr, "fail to init database: %v\n", err) os.Exit(1) diff --git a/internal/modules/api/config.go b/internal/modules/api/config.go index 0cf4985..0269643 100644 --- a/internal/modules/api/config.go +++ b/internal/modules/api/config.go @@ -5,8 +5,8 @@ import ( ) type Config struct { - Admin string `json:"admin"` - Port int `json:"port" default:"31337"` + Admin string `mapstructure:"admin"` + Port int `mapstructure:"port"` } func (c Config) Validate() error { diff --git a/internal/modules/lark/config.go b/internal/modules/lark/config.go index 1d492cf..de22b5e 100644 --- a/internal/modules/lark/config.go +++ b/internal/modules/lark/config.go @@ -5,14 +5,14 @@ import ( ) type Config struct { - Admin string `json:"admin"` - AppID string `json:"app_id"` - AppSecret string `json:"app_secret"` - VerificationToken string `json:"verification_token"` - EncryptKey string `json:"encrypt_key"` - TLSEnabled bool `json:"tls_enabled" default:"true"` - ProxyURL string `json:"proxy_url"` - ProxyInsecure bool `json:"proxy_insecure"` + Admin string `mapstructure:"admin"` + AppID string `mapstructure:"app_id"` + AppSecret string `mapstructure:"app_secret"` + VerificationToken string `mapstructure:"verification_token"` + EncryptKey string `mapstructure:"encrypt_key"` + TLSEnabled bool `mapstructure:"tls_enabled"` + ProxyURL string `mapstructure:"proxy_url"` + ProxyInsecure bool `mapstructure:"proxy_insecure"` } func (c Config) Validate() error { diff --git a/internal/modules/telegram/config.go b/internal/modules/telegram/config.go index 93636cf..906cb5c 100644 --- a/internal/modules/telegram/config.go +++ b/internal/modules/telegram/config.go @@ -5,9 +5,9 @@ import ( ) type Config struct { - Admin int64 `json:"admin"` - Token string `json:"token"` - Proxy string `json:"proxy"` + Admin int64 `mapstructure:"admin"` + Token string `mapstructure:"token"` + Proxy string `mapstructure:"proxy"` } func (c Config) Validate() error { diff --git a/internal/utils/struct.go b/internal/utils/struct.go new file mode 100644 index 0000000..22a8a6c --- /dev/null +++ b/internal/utils/struct.go @@ -0,0 +1,31 @@ +package utils + +import ( + "fmt" + "reflect" +) + +func StructKeys(s any, tagName string) []string { + keys := []string{} + ct := reflect.TypeOf(s) + + if ct.Kind() != reflect.Struct { + return nil + } + + for i := 0; i < ct.NumField(); i++ { + field := ct.Field(i) + tag := field.Tag.Get(tagName) + + if field.Type.Kind() == reflect.Struct { + res := StructKeys(reflect.New(field.Type).Elem().Interface(), tagName) + for _, k := range res { + keys = append(keys, fmt.Sprintf("%s.%s", tag, k)) + } + } else { + keys = append(keys, tag) + } + } + + return keys +}