refactor config

TODO.md

@@ -37,3 +37,4 @@
 - C++ generates false-positives (cisco/mlspp repository) --> should try with different signatures perhaps (like gitleaks?)
 - "View commit on gitlab" (webserver) when using `local-git-repo`
 - Snyk report!
+- failed cloning repository https://github.com/Senzing/knowledge-base.git, empty git-upload-pack given -> does not result in the tool exiting with 1

cmd/root.go

@@ -7,6 +7,7 @@ import (
 	"os"
 
 	"github.com/rumenvasilev/rvsecret/cmd/scan"
+	"github.com/rumenvasilev/rvsecret/internal/config"
 	"github.com/rumenvasilev/rvsecret/version"
 
 	"github.com/spf13/cobra"
@@ -18,6 +19,9 @@ var (
 		Use:   "rvsecret",
 		Short: "A tool to scan for secrets in various digital hiding spots",
 		Long:  "A tool to scan for secrets in various digital hiding spots - v" + version.AppVersion(), // TODO write a better long description
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			config.SetConfig()
+		},
 	}
 )
 
@@ -32,4 +36,5 @@ func Execute() {
 
 func init() {
 	rootCmd.AddCommand(scan.ScanCmd)
+	rootCmd.PersistentFlags().Bool("debug", false, "Print available debugging information to stdout")
 }

cmd/scan/scan-gitlab.go

@@ -26,8 +26,7 @@ var scanGitlabCmd = &cobra.Command{
 
 func init() {
 	ScanCmd.AddCommand(scanGitlabCmd)
-	scanGitlabCmd.Flags().Bool("add-org-members", false, "Add members to targets when processing organizations")
-	// scanGitlabCmd.Flags().Float64("commit-depth", -1, "Set the commit depth to scan")
+	// scanGitlabCmd.Flags().Bool("add-org-members", false, "Add members to targets when processing organizations")
 	scanGitlabCmd.Flags().StringP("gitlab-api-token", "t", "", "API token for access to gitlab, see doc for necessary scope")
 	scanGitlabCmd.Flags().StringSlice("gitlab-projects", nil, "List of Gitlab projects or users to scan")
 	viper.BindPFlags(scanGithubCmd.Flags()) //nolint:errcheck

cmd/scan/scan.go

@@ -3,7 +3,6 @@
 package scan
 
 import (
-	"github.com/rumenvasilev/rvsecret/internal/config"
 	"github.com/rumenvasilev/rvsecret/version"
 
 	"github.com/spf13/cobra"
@@ -19,15 +18,13 @@ var (
 )
 
 func init() {
-	cobra.OnInitialize(config.SetConfig)
 	// Global flags under `scan` command
 	ScanCmd.PersistentFlags().String("bind-address", "127.0.0.1", "The IP address for the webserver")
 	ScanCmd.PersistentFlags().Int("bind-port", 9393, "The port for the webserver")
 	ScanCmd.PersistentFlags().Int("confidence-level", 3, "The confidence level of the expressions used to find matches")
 	ScanCmd.PersistentFlags().String("config-file", "$HOME/.rvsecret/config.yaml", "config file")
 	ScanCmd.PersistentFlags().Bool("csv", false, "Output csv format")
 	ScanCmd.PersistentFlags().Bool("json", false, "Output json format")
-	ScanCmd.PersistentFlags().Bool("debug", false, "Print available debugging information to stdout")
 	ScanCmd.PersistentFlags().Bool("hide-secrets", false, "Do not print secrets to any supported output")
 	ScanCmd.PersistentFlags().StringSlice("ignore-extension", nil, "List of file extensions to ignore")
 	ScanCmd.PersistentFlags().StringSlice("ignore-path", nil, "List of file paths to ignore")

cmd/updateRules.go

@@ -24,13 +24,12 @@ var updateSignaturesCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(updateSignaturesCmd)
-	updateSignaturesCmd.Flags().StringP("github-api-token", "t", "", "API token for github access, see documentation for necessary scope")
-	updateSignaturesCmd.MarkFlagRequired("github-api-token") //nolint:errcheck
+	updateSignaturesCmd.Flags().StringP("signatures-github-api-token", "t", "", "API token for github access, see documentation for necessary scope")
+	updateSignaturesCmd.MarkFlagRequired("signatures-github-api-token") //nolint:errcheck
 	updateSignaturesCmd.Flags().String("signatures-user-repo", "", "user/repo where signatures can be found, example: rumenvasilev/rvsecret-signatures")
 	updateSignaturesCmd.Flags().String("signatures-url", "https://github.com/rumenvasilev/rvsecret-signatures", "url where the signatures can be found")
 	updateSignaturesCmd.MarkFlagsMutuallyExclusive("signatures-user-repo", "signatures-url")
 	updateSignaturesCmd.Flags().String("signatures-version", "latest", "specific version of the signatures to install (latest, v1.2.0)")
 	updateSignaturesCmd.Flags().Bool("test-signatures", false, "run any tests associated with the signatures and display the output")
-	updateSignaturesCmd.Flags().Bool("debug", false, "Print available debugging information to stdout")
 	viper.BindPFlags(updateSignaturesCmd.Flags()) //nolint:errcheck
 }

internal/config/config.go

@@ -12,8 +12,10 @@ import (
 	"github.com/spf13/viper"
 )
 
+const configFile = "$HOME/.rvsecret/config.yaml"
+
 // cfg holds the configuration data the commands
-var cfg *viper.Viper
+var cfg *Config
 
 // defaultIgnoreExtensions is an array of extensions that if they match a file that file will be excluded
 var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff",
@@ -22,60 +24,12 @@ var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff
 // defaultIgnorePaths is an array of directories that will be excluded from all types of scans.
 var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cache", "/proc/"}
 
-// DefaultValues is a map of all flag default values and other mutable variables
-var defaultValues = map[string]interface{}{
-	"bind-address":     "127.0.0.1",
-	"bind-port":        9393,
-	"commit-depth":     -1,
-	"config-file":      "$HOME/.rvsecret/config.yaml",
-	"csv":              false,
-	"debug":            false,
-	"ignore-extension": nil,
-	"ignore-path":      nil,
-	"in-mem-clone":     false,
-	"json":             false,
-	"max-file-size":    10,
-	"num-threads":      -1,
-	"local-paths":      nil,
-	"scan-forks":       false,
-	"scan-tests":       false,
-	"scan-type":        "",
-	"silent":           false,
-	"confidence-level": 3,
-	"signatures-file":  "$HOME/.rvsecret/signatures/default.yaml",
-	"signatures-path":  "$HOME/.rvsecret/signatures/",
-	"signatures-url":   "https://github.com/rumenvasilev/rvsecret-signatures",
-	// "signatures-version": "",
-	"scan-dir":        nil,
-	"scan-file":       nil,
-	"hide-secrets":    false,
-	"rules-url":       "",
-	"test-signatures": false,
-	"web-server":      false,
-	// Github
-	"add-org-members":       false,
-	"github-enterprise-url": "",
-	"github-url":            "https://api.github.com",
-	"github-api-token":      "",
-	"github-orgs":           nil,
-	"github-repos":          nil,
-	"github-users":          nil,
-	// Gitlab
-	"gitlab-targets": nil,
-	//"gitlab-url":                 "", // TODO set the default
-	"gitlab-api-token": "",
-}
-
 // SetConfig will set the defaults, and load a config file and environment variables if they are present
 func SetConfig() {
-	for key, value := range defaultValues {
-		viper.SetDefault(key, value)
-	}
+	cf := viper.GetString("config-file")
 
-	configFile := viper.GetString("config-file")
-
-	if configFile != defaultValues["config-file"] {
-		viper.SetConfigFile(configFile)
+	if cf != configFile {
+		viper.SetConfigFile(cf)
 	} else {
 		home, err := homedir.Dir()
 		if err != nil {
@@ -90,118 +44,102 @@ func SetConfig() {
 
 	viper.ReadInConfig() //nolint:errcheck
 	viper.AutomaticEnv()
-	cfg = viper.GetViper()
-}
-
-type Config struct {
-	AppVersion          string
-	BindAddress         string
-	BindPort            int
-	CommitDepth         int
-	ConfidenceLevel     int
-	CSVOutput           bool
-	Debug               bool
-	ExpandOrgs          bool
-	GithubAccessToken   string
-	GithubEnterpriseURL string
-	GitlabAccessToken   string
-	GitlabTargets       []string
-	GitlabURL           string
-	HideSecrets         bool
-	InMemClone          bool
-	JSONOutput          bool
-	LocalPaths          []string
-	MaxFileSize         int64
-	ScanFork            bool
-	ScanTests           bool
-	ScanType            api.ScanType
-	Silent              bool
-	SkippableExt        []string
-	SkippablePath       []string
-	SignaturesFile      string
-	SignaturesPath      string
-	SignaturesURL       string
-	SignaturesUserRepo  string
-	SignaturesVersion   string
-	SignaturesTest      bool
-	Threads             int
-	UserDirtyNames      []string
-	UserDirtyOrgs       []string
-	UserDirtyRepos      []string
-	WebServer           bool
-}
-
-// TODO detect scanType automatically
-func Load(scanType api.ScanType) (*Config, error) {
-	c := Config{
-		BindAddress:     cfg.GetString("bind-address"),
-		BindPort:        cfg.GetInt("bind-port"),
-		CommitDepth:     setCommitDepth(cfg.GetFloat64("commit-depth")),
-		CSVOutput:       cfg.GetBool("csv"),
-		Debug:           cfg.GetBool("debug"),
-		ExpandOrgs:      cfg.GetBool("expand-orgs"),
-		HideSecrets:     cfg.GetBool("hide-secrets"),
-		InMemClone:      cfg.GetBool("in-mem-clone"),
-		JSONOutput:      cfg.GetBool("json"),
-		MaxFileSize:     cfg.GetInt64("max-file-size"),
-		ConfidenceLevel: cfg.GetInt("confidence-level"),
-		ScanFork:        cfg.GetBool("scan-forks"),
-		ScanTests:       cfg.GetBool("scan-tests"),
-		ScanType:        scanType,
-		Silent:          cfg.GetBool("silent"),
-		SignaturesFile:  cfg.GetString("signatures-file"),
-		SignaturesPath:  cfg.GetString("signatures-path"),
-		Threads:         cfg.GetInt("num-threads"),
-		AppVersion:      version.AppVersion(),
-		WebServer:       cfg.GetBool("web-server"),
-	}
 
-	switch scanType {
-	case api.LocalGit:
-		c.LocalPaths = cfg.GetStringSlice("local-repos")
-	case api.LocalPath:
-		c.LocalPaths = cfg.GetStringSlice("paths")
-	case api.Gitlab:
-		c.GitlabAccessToken = cfg.GetString("gitlab-api-token")
-		c.GitlabTargets = cfg.GetStringSlice("gitlab-targets")
-	case api.Github, api.GithubEnterprise:
-		c.GithubAccessToken = cfg.GetString("github-api-token")
-		c.UserDirtyRepos = cfg.GetStringSlice("github-repos")
-		c.UserDirtyOrgs = cfg.GetStringSlice("github-orgs")
-		c.UserDirtyNames = cfg.GetStringSlice("github-users")
-		if scanType == api.GithubEnterprise {
-			c.GithubEnterpriseURL = cfg.GetString("github-enterprise-url")
-			if c.GithubEnterpriseURL == "" {
-				return nil, errors.New("Github enterprise URL is not set.")
-			}
-		}
-	case api.UpdateSignatures:
-		c.GithubAccessToken = cfg.GetString("github-api-token")
-		c.SignaturesVersion = viper.GetString("signatures-version")
-		c.SignaturesURL = viper.GetString("signatures-url")
-		c.SignaturesUserRepo = viper.GetString("signatures-user-repo")
-		c.SignaturesTest = viper.GetBool("test-signatures")
+	var c *Config
+	err := viper.Unmarshal(&c)
+	if err != nil {
+		fmt.Println("Failed unmarshaling config to struct")
+		os.Exit(1)
 	}
 
-	// Add the default directories to the sess if they don't already exist
 	c.SkippablePath = util.AppendToSlice(true, defaultIgnorePaths, c.SkippablePath)
 	// add any additional paths the user requested to exclude to the pre-defined slice
-	c.SkippablePath = util.AppendToSlice(true, cfg.GetStringSlice("ignore-path"), c.SkippablePath)
+	// c.SkippablePath = util.AppendToSlice(true, viper.GetStringSlice("ignore-path"), c.SkippablePath)
 
 	// the default ignorable extensions
 	c.SkippableExt = util.AppendToSlice(false, defaultIgnoreExtensions, c.SkippableExt)
 	// add any additional extensions the user requested to ignore
-	c.SkippableExt = util.AppendToSlice(true, cfg.GetStringSlice("ignore-extension"), c.SkippableExt)
+	// c.SkippableExt = util.AppendToSlice(true, viper.GetStringSlice("ignore-extension"), c.SkippableExt)
 
-	return &c, nil
+	c.CommitDepth = setCommitDepth(c.CommitDepth)
+
+	c.AppVersion = version.AppVersion()
+	cfg = c
+}
+
+type Config struct {
+	Global     `mapstructure:",squash"`
+	Github     `mapstructure:",squash"`
+	Gitlab     `mapstructure:",squash"`
+	Signatures `mapstructure:",squash"`
+}
+
+type Global struct {
+	// "add-org-members": false
+	AppVersion      string
+	BindAddress     string       `mapstructure:"bind-address" default:"127.0.0.1"`
+	BindPort        int          `mapstructure:"bind-port" default:"9393"`
+	CommitDepth     int          `mapstructure:"commit-depth" default:"-1"`
+	ConfigFile      string       `mapstructure:"config-file" default:"$HOME/.rvsecret/config.yaml"`
+	ConfidenceLevel int          `mapstructure:"confidence-level" default:"3"`
+	CSVOutput       bool         `mapstructure:"csv"`
+	Debug           bool         `mapstructure:"debug"`
+	ExpandOrgs      bool         `mapstructure:"expand-orgs"`
+	HideSecrets     bool         `mapstructure:"hide-secrets" default:"false"`
+	InMemClone      bool         `mapstructure:"in-mem-clone" default:"false"`
+	JSONOutput      bool         `mapstructure:"json"`
+	LocalPaths      []string     `mapstructure:"paths"`
+	MaxFileSize     int64        `mapstructure:"max-file-size" default:"10"`
+	ScanFork        bool         `mapstructure:"scan-forks" default:"false"`
+	ScanTests       bool         `mapstructure:"scan-tests" default:"false"`
+	ScanType        api.ScanType `mapstructure:"scan-type"`
+	Silent          bool         `mapstructure:"silent" default:"false"`
+	SkippableExt    []string     `mapstructure:"ignore-extension"`
+	SkippablePath   []string     `mapstructure:"ignore-path"`
+	Threads         int          `mapstructure:"num-threads" default:"-1"`
+	WebServer       bool         `mapstructure:"web-server" default:"false"`
+}
+
+type Signatures struct {
+	APIToken           string `mapstructure:"signatures-github-api-token"`
+	SignaturesFile     string `mapstructure:"signatures-file"`
+	SignaturesPath     string `mapstructure:"signatures-path"`
+	SignaturesURL      string `mapstructure:"signatures-url" default:"https://github.com/rumenvasilev/rvsecret-signatures"`
+	SignaturesUserRepo string `mapstructure:"signatures-user-repo"`
+	SignaturesVersion  string `mapstructure:"signatures-version"`
+	SignaturesTest     bool   `mapstructure:"test-signatures"`
+}
+
+type Github struct {
+	APIToken            string   `mapstructure:"github-api-token"`
+	GithubURL           string   `mapstructure:"github-url" default:"https://api.github.com"`
+	GithubEnterpriseURL string   `mapstructure:"github-enterprise-url"`
+	UserDirtyNames      []string `mapstructure:"github-users"`
+	UserDirtyOrgs       []string `mapstructure:"github-orgs"`
+	UserDirtyRepos      []string `mapstructure:"github-repos"`
+}
+
+type Gitlab struct {
+	GitlabAccessToken string   `mapstructure:"gitlab-api-token"`
+	GitlabTargets     []string `mapstructure:"gitlab-targets"`
+	// GitlabURL         string   `yaml:""` // THIS SHOULD BE A CONSTANT
+}
+
+// TODO detect scanType automatically
+func Load(scanType api.ScanType) (*Config, error) {
+	cfg.ScanType = scanType
+	if scanType == api.GithubEnterprise && cfg.GithubEnterpriseURL == "" {
+		return nil, errors.New("Github enterprise URL is not set.")
+	}
+	return cfg, nil
 }
 
 // setCommitDepth will set the commit depth for the current session. This is an ugly way of doing it
 // but for the moment it works fine.
 // TODO dynamically acquire the commit depth of a given repo
-func setCommitDepth(c float64) int {
+func setCommitDepth(c int) int {
 	if c == -1 {
 		return 9999999999
 	}
-	return int(c)
+	return c
 }