diff --git a/TODO.md b/TODO.md index c4c6917..f82e5e0 100644 --- a/TODO.md +++ b/TODO.md @@ -37,3 +37,4 @@ - C++ generates false-positives (cisco/mlspp repository) --> should try with different signatures perhaps (like gitleaks?) - "View commit on gitlab" (webserver) when using `local-git-repo` - Snyk report! +- failed cloning repository https://github.com/Senzing/knowledge-base.git, empty git-upload-pack given -> does not result in the tool exiting with 1 diff --git a/cmd/root.go b/cmd/root.go index 0639f95..398dca5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,6 +7,7 @@ import ( "os" "github.com/rumenvasilev/rvsecret/cmd/scan" + "github.com/rumenvasilev/rvsecret/internal/config" "github.com/rumenvasilev/rvsecret/version" "github.com/spf13/cobra" @@ -18,6 +19,9 @@ var ( Use: "rvsecret", Short: "A tool to scan for secrets in various digital hiding spots", Long: "A tool to scan for secrets in various digital hiding spots - v" + version.AppVersion(), // TODO write a better long description + PersistentPreRun: func(cmd *cobra.Command, args []string) { + config.SetConfig() + }, } ) @@ -32,4 +36,5 @@ func Execute() { func init() { rootCmd.AddCommand(scan.ScanCmd) + rootCmd.PersistentFlags().Bool("debug", false, "Print available debugging information to stdout") } diff --git a/cmd/scan/scan-gitlab.go b/cmd/scan/scan-gitlab.go index 81afafa..6403da4 100644 --- a/cmd/scan/scan-gitlab.go +++ b/cmd/scan/scan-gitlab.go @@ -26,8 +26,7 @@ var scanGitlabCmd = &cobra.Command{ func init() { ScanCmd.AddCommand(scanGitlabCmd) - scanGitlabCmd.Flags().Bool("add-org-members", false, "Add members to targets when processing organizations") - // scanGitlabCmd.Flags().Float64("commit-depth", -1, "Set the commit depth to scan") + // scanGitlabCmd.Flags().Bool("add-org-members", false, "Add members to targets when processing organizations") scanGitlabCmd.Flags().StringP("gitlab-api-token", "t", "", "API token for access to gitlab, see doc for necessary scope") scanGitlabCmd.Flags().StringSlice("gitlab-projects", nil, "List of Gitlab projects or users to scan") viper.BindPFlags(scanGithubCmd.Flags()) //nolint:errcheck diff --git a/cmd/scan/scan.go b/cmd/scan/scan.go index f20f2c1..88f0bcf 100644 --- a/cmd/scan/scan.go +++ b/cmd/scan/scan.go @@ -3,7 +3,6 @@ package scan import ( - "github.com/rumenvasilev/rvsecret/internal/config" "github.com/rumenvasilev/rvsecret/version" "github.com/spf13/cobra" @@ -19,7 +18,6 @@ var ( ) func init() { - cobra.OnInitialize(config.SetConfig) // Global flags under `scan` command ScanCmd.PersistentFlags().String("bind-address", "127.0.0.1", "The IP address for the webserver") ScanCmd.PersistentFlags().Int("bind-port", 9393, "The port for the webserver") @@ -27,7 +25,6 @@ func init() { ScanCmd.PersistentFlags().String("config-file", "$HOME/.rvsecret/config.yaml", "config file") ScanCmd.PersistentFlags().Bool("csv", false, "Output csv format") ScanCmd.PersistentFlags().Bool("json", false, "Output json format") - ScanCmd.PersistentFlags().Bool("debug", false, "Print available debugging information to stdout") ScanCmd.PersistentFlags().Bool("hide-secrets", false, "Do not print secrets to any supported output") ScanCmd.PersistentFlags().StringSlice("ignore-extension", nil, "List of file extensions to ignore") ScanCmd.PersistentFlags().StringSlice("ignore-path", nil, "List of file paths to ignore") diff --git a/cmd/updateRules.go b/cmd/updateRules.go index db1b429..fa83e90 100644 --- a/cmd/updateRules.go +++ b/cmd/updateRules.go @@ -24,13 +24,12 @@ var updateSignaturesCmd = &cobra.Command{ func init() { rootCmd.AddCommand(updateSignaturesCmd) - updateSignaturesCmd.Flags().StringP("github-api-token", "t", "", "API token for github access, see documentation for necessary scope") - updateSignaturesCmd.MarkFlagRequired("github-api-token") //nolint:errcheck + updateSignaturesCmd.Flags().StringP("signatures-github-api-token", "t", "", "API token for github access, see documentation for necessary scope") + updateSignaturesCmd.MarkFlagRequired("signatures-github-api-token") //nolint:errcheck updateSignaturesCmd.Flags().String("signatures-user-repo", "", "user/repo where signatures can be found, example: rumenvasilev/rvsecret-signatures") updateSignaturesCmd.Flags().String("signatures-url", "https://github.com/rumenvasilev/rvsecret-signatures", "url where the signatures can be found") updateSignaturesCmd.MarkFlagsMutuallyExclusive("signatures-user-repo", "signatures-url") updateSignaturesCmd.Flags().String("signatures-version", "latest", "specific version of the signatures to install (latest, v1.2.0)") updateSignaturesCmd.Flags().Bool("test-signatures", false, "run any tests associated with the signatures and display the output") - updateSignaturesCmd.Flags().Bool("debug", false, "Print available debugging information to stdout") viper.BindPFlags(updateSignaturesCmd.Flags()) //nolint:errcheck } diff --git a/internal/config/config.go b/internal/config/config.go index 6925daf..f540b00 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,8 +12,10 @@ import ( "github.com/spf13/viper" ) +const configFile = "$HOME/.rvsecret/config.yaml" + // cfg holds the configuration data the commands -var cfg *viper.Viper +var cfg *Config // defaultIgnoreExtensions is an array of extensions that if they match a file that file will be excluded var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff", @@ -22,60 +24,12 @@ var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff // defaultIgnorePaths is an array of directories that will be excluded from all types of scans. var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cache", "/proc/"} -// DefaultValues is a map of all flag default values and other mutable variables -var defaultValues = map[string]interface{}{ - "bind-address": "127.0.0.1", - "bind-port": 9393, - "commit-depth": -1, - "config-file": "$HOME/.rvsecret/config.yaml", - "csv": false, - "debug": false, - "ignore-extension": nil, - "ignore-path": nil, - "in-mem-clone": false, - "json": false, - "max-file-size": 10, - "num-threads": -1, - "local-paths": nil, - "scan-forks": false, - "scan-tests": false, - "scan-type": "", - "silent": false, - "confidence-level": 3, - "signatures-file": "$HOME/.rvsecret/signatures/default.yaml", - "signatures-path": "$HOME/.rvsecret/signatures/", - "signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures", - // "signatures-version": "", - "scan-dir": nil, - "scan-file": nil, - "hide-secrets": false, - "rules-url": "", - "test-signatures": false, - "web-server": false, - // Github - "add-org-members": false, - "github-enterprise-url": "", - "github-url": "https://api.github.com", - "github-api-token": "", - "github-orgs": nil, - "github-repos": nil, - "github-users": nil, - // Gitlab - "gitlab-targets": nil, - //"gitlab-url": "", // TODO set the default - "gitlab-api-token": "", -} - // SetConfig will set the defaults, and load a config file and environment variables if they are present func SetConfig() { - for key, value := range defaultValues { - viper.SetDefault(key, value) - } + cf := viper.GetString("config-file") - configFile := viper.GetString("config-file") - - if configFile != defaultValues["config-file"] { - viper.SetConfigFile(configFile) + if cf != configFile { + viper.SetConfigFile(cf) } else { home, err := homedir.Dir() if err != nil { @@ -90,118 +44,102 @@ func SetConfig() { viper.ReadInConfig() //nolint:errcheck viper.AutomaticEnv() - cfg = viper.GetViper() -} - -type Config struct { - AppVersion string - BindAddress string - BindPort int - CommitDepth int - ConfidenceLevel int - CSVOutput bool - Debug bool - ExpandOrgs bool - GithubAccessToken string - GithubEnterpriseURL string - GitlabAccessToken string - GitlabTargets []string - GitlabURL string - HideSecrets bool - InMemClone bool - JSONOutput bool - LocalPaths []string - MaxFileSize int64 - ScanFork bool - ScanTests bool - ScanType api.ScanType - Silent bool - SkippableExt []string - SkippablePath []string - SignaturesFile string - SignaturesPath string - SignaturesURL string - SignaturesUserRepo string - SignaturesVersion string - SignaturesTest bool - Threads int - UserDirtyNames []string - UserDirtyOrgs []string - UserDirtyRepos []string - WebServer bool -} - -// TODO detect scanType automatically -func Load(scanType api.ScanType) (*Config, error) { - c := Config{ - BindAddress: cfg.GetString("bind-address"), - BindPort: cfg.GetInt("bind-port"), - CommitDepth: setCommitDepth(cfg.GetFloat64("commit-depth")), - CSVOutput: cfg.GetBool("csv"), - Debug: cfg.GetBool("debug"), - ExpandOrgs: cfg.GetBool("expand-orgs"), - HideSecrets: cfg.GetBool("hide-secrets"), - InMemClone: cfg.GetBool("in-mem-clone"), - JSONOutput: cfg.GetBool("json"), - MaxFileSize: cfg.GetInt64("max-file-size"), - ConfidenceLevel: cfg.GetInt("confidence-level"), - ScanFork: cfg.GetBool("scan-forks"), - ScanTests: cfg.GetBool("scan-tests"), - ScanType: scanType, - Silent: cfg.GetBool("silent"), - SignaturesFile: cfg.GetString("signatures-file"), - SignaturesPath: cfg.GetString("signatures-path"), - Threads: cfg.GetInt("num-threads"), - AppVersion: version.AppVersion(), - WebServer: cfg.GetBool("web-server"), - } - switch scanType { - case api.LocalGit: - c.LocalPaths = cfg.GetStringSlice("local-repos") - case api.LocalPath: - c.LocalPaths = cfg.GetStringSlice("paths") - case api.Gitlab: - c.GitlabAccessToken = cfg.GetString("gitlab-api-token") - c.GitlabTargets = cfg.GetStringSlice("gitlab-targets") - case api.Github, api.GithubEnterprise: - c.GithubAccessToken = cfg.GetString("github-api-token") - c.UserDirtyRepos = cfg.GetStringSlice("github-repos") - c.UserDirtyOrgs = cfg.GetStringSlice("github-orgs") - c.UserDirtyNames = cfg.GetStringSlice("github-users") - if scanType == api.GithubEnterprise { - c.GithubEnterpriseURL = cfg.GetString("github-enterprise-url") - if c.GithubEnterpriseURL == "" { - return nil, errors.New("Github enterprise URL is not set.") - } - } - case api.UpdateSignatures: - c.GithubAccessToken = cfg.GetString("github-api-token") - c.SignaturesVersion = viper.GetString("signatures-version") - c.SignaturesURL = viper.GetString("signatures-url") - c.SignaturesUserRepo = viper.GetString("signatures-user-repo") - c.SignaturesTest = viper.GetBool("test-signatures") + var c *Config + err := viper.Unmarshal(&c) + if err != nil { + fmt.Println("Failed unmarshaling config to struct") + os.Exit(1) } - // Add the default directories to the sess if they don't already exist c.SkippablePath = util.AppendToSlice(true, defaultIgnorePaths, c.SkippablePath) // add any additional paths the user requested to exclude to the pre-defined slice - c.SkippablePath = util.AppendToSlice(true, cfg.GetStringSlice("ignore-path"), c.SkippablePath) + // c.SkippablePath = util.AppendToSlice(true, viper.GetStringSlice("ignore-path"), c.SkippablePath) // the default ignorable extensions c.SkippableExt = util.AppendToSlice(false, defaultIgnoreExtensions, c.SkippableExt) // add any additional extensions the user requested to ignore - c.SkippableExt = util.AppendToSlice(true, cfg.GetStringSlice("ignore-extension"), c.SkippableExt) + // c.SkippableExt = util.AppendToSlice(true, viper.GetStringSlice("ignore-extension"), c.SkippableExt) - return &c, nil + c.CommitDepth = setCommitDepth(c.CommitDepth) + + c.AppVersion = version.AppVersion() + cfg = c +} + +type Config struct { + Global `mapstructure:",squash"` + Github `mapstructure:",squash"` + Gitlab `mapstructure:",squash"` + Signatures `mapstructure:",squash"` +} + +type Global struct { + // "add-org-members": false + AppVersion string + BindAddress string `mapstructure:"bind-address" default:"127.0.0.1"` + BindPort int `mapstructure:"bind-port" default:"9393"` + CommitDepth int `mapstructure:"commit-depth" default:"-1"` + ConfigFile string `mapstructure:"config-file" default:"$HOME/.rvsecret/config.yaml"` + ConfidenceLevel int `mapstructure:"confidence-level" default:"3"` + CSVOutput bool `mapstructure:"csv"` + Debug bool `mapstructure:"debug"` + ExpandOrgs bool `mapstructure:"expand-orgs"` + HideSecrets bool `mapstructure:"hide-secrets" default:"false"` + InMemClone bool `mapstructure:"in-mem-clone" default:"false"` + JSONOutput bool `mapstructure:"json"` + LocalPaths []string `mapstructure:"paths"` + MaxFileSize int64 `mapstructure:"max-file-size" default:"10"` + ScanFork bool `mapstructure:"scan-forks" default:"false"` + ScanTests bool `mapstructure:"scan-tests" default:"false"` + ScanType api.ScanType `mapstructure:"scan-type"` + Silent bool `mapstructure:"silent" default:"false"` + SkippableExt []string `mapstructure:"ignore-extension"` + SkippablePath []string `mapstructure:"ignore-path"` + Threads int `mapstructure:"num-threads" default:"-1"` + WebServer bool `mapstructure:"web-server" default:"false"` +} + +type Signatures struct { + APIToken string `mapstructure:"signatures-github-api-token"` + SignaturesFile string `mapstructure:"signatures-file"` + SignaturesPath string `mapstructure:"signatures-path"` + SignaturesURL string `mapstructure:"signatures-url" default:"https://github.com/rumenvasilev/rvsecret-signatures"` + SignaturesUserRepo string `mapstructure:"signatures-user-repo"` + SignaturesVersion string `mapstructure:"signatures-version"` + SignaturesTest bool `mapstructure:"test-signatures"` +} + +type Github struct { + APIToken string `mapstructure:"github-api-token"` + GithubURL string `mapstructure:"github-url" default:"https://api.github.com"` + GithubEnterpriseURL string `mapstructure:"github-enterprise-url"` + UserDirtyNames []string `mapstructure:"github-users"` + UserDirtyOrgs []string `mapstructure:"github-orgs"` + UserDirtyRepos []string `mapstructure:"github-repos"` +} + +type Gitlab struct { + GitlabAccessToken string `mapstructure:"gitlab-api-token"` + GitlabTargets []string `mapstructure:"gitlab-targets"` + // GitlabURL string `yaml:""` // THIS SHOULD BE A CONSTANT +} + +// TODO detect scanType automatically +func Load(scanType api.ScanType) (*Config, error) { + cfg.ScanType = scanType + if scanType == api.GithubEnterprise && cfg.GithubEnterpriseURL == "" { + return nil, errors.New("Github enterprise URL is not set.") + } + return cfg, nil } // setCommitDepth will set the commit depth for the current session. This is an ugly way of doing it // but for the moment it works fine. // TODO dynamically acquire the commit depth of a given repo -func setCommitDepth(c float64) int { +func setCommitDepth(c int) int { if c == -1 { return 9999999999 } - return int(c) + return c } diff --git a/internal/core/git.go b/internal/core/git.go index 258d7f7..9a2d36c 100644 --- a/internal/core/git.go +++ b/internal/core/git.go @@ -161,19 +161,19 @@ func GetChangeContent(change *object.Change) (result string, contentError error) // InitGitClient will create a new git client of the type given by the input string. func (s *Session) InitGitClient() error { switch s.Config.ScanType { - case api.Github, api.GithubEnterprise, api.UpdateSignatures: - client, err := _github.NewClient(s.Config.GithubAccessToken, "", s.Out) + case api.Github: + client, err := _github.NewClient(s.Config.Github.APIToken, "", s.Out) if err != nil { return err } - if s.Config.ScanType == api.GithubEnterprise { - if s.Config.GithubEnterpriseURL == "" { - return fmt.Errorf("github enterprise URL is missing") - } - client, err = _github.NewClient(s.Config.GithubAccessToken, s.Config.GithubEnterpriseURL, s.Out) - if err != nil { - return err - } + s.Client = client + case api.GithubEnterprise: + if s.Config.GithubEnterpriseURL == "" { + return fmt.Errorf("github enterprise URL is missing") + } + client, err := _github.NewClient(s.Config.Github.APIToken, s.Config.GithubEnterpriseURL, s.Out) + if err != nil { + return err } s.Client = client case api.Gitlab: @@ -184,6 +184,14 @@ func (s *Session) InitGitClient() error { // TODO need to add in the bits to parse the url here as well // TODO set this to some sort of consistent client, look to github for ideas s.Client = client + case api.UpdateSignatures: + client, err := _github.NewClient(s.Config.Signatures.APIToken, "", s.Out) + if err != nil { + return err + } + s.Client = client + default: + return fmt.Errorf("unknown scan type provided") } return nil } @@ -215,10 +223,10 @@ func cloneRepositoryFunc(sess *Session, repo _coreapi.Repository) (*git.Reposito Branch: repo.DefaultBranch, Depth: sess.Config.CommitDepth, InMemClone: sess.Config.InMemClone, - // Token: sess.GithubAccessToken, } auth.Username = "doesn't matter" - auth.Password = sess.Config.GithubAccessToken + auth.Password = sess.Config.Github.APIToken + // case api.UpdateSignatures: This is handled through a different path case api.Gitlab: cloneConfig = CloneConfiguration{ URL: repo.CloneURL, @@ -236,6 +244,8 @@ func cloneRepositoryFunc(sess *Session, repo _coreapi.Repository) (*git.Reposito Depth: sess.Config.CommitDepth, InMemClone: sess.Config.InMemClone, } + default: + return nil, "", fmt.Errorf("unsupported scantype") } return CloneRepositoryGeneric(cloneConfig, &auth) } diff --git a/internal/core/provider/github/client.go b/internal/core/provider/github/client.go index fbdf761..5e86431 100644 --- a/internal/core/provider/github/client.go +++ b/internal/core/provider/github/client.go @@ -3,6 +3,7 @@ package github import ( "context" "fmt" + "net/http" "regexp" "sync" @@ -20,32 +21,45 @@ type Client struct { } // NewClient creates a gitlab api client instance using a token -func NewClient(token string, gheURL string, logger *log.Logger) (*Client, error) { +func NewClient(token, gheURL string, logger *log.Logger) (*Client, error) { err := validateAPIToken(token) if err != nil { return nil, fmt.Errorf("APIToken is invalid, %q", token) } // Get OAuth client oauth := getOauthClient(token) - var c *github.Client - // Github Enterprise if gheURL != "" { - baseURL := fmt.Sprintf("%s/api/v3", gheURL) - uploadURL := fmt.Sprintf("%s/api/uploads", gheURL) - c, err = github.NewEnterpriseClient(baseURL, uploadURL, oauth) - if err != nil { - return nil, fmt.Errorf("unable to parse --github-enterprise-url: <%s>, %w", gheURL, err) - } - } else { - c = github.NewClient(oauth) + return newEnterpriseClient(oauth, gheURL, logger) } + return newStandardClient(oauth, logger), nil +} +func newStandardClient(oauth *http.Client, logger *log.Logger) *Client { + c := github.NewClient(oauth) c.UserAgent = version.UserAgent - client := &Client{ + + return &Client{ apiClient: c, logger: logger, } - return client, nil +} + +func newEnterpriseClient(oauth *http.Client, gheURL string, logger *log.Logger) (*Client, error) { + baseURL := fmt.Sprintf("%s/api/v3", gheURL) + uploadURL := fmt.Sprintf("%s/api/uploads", gheURL) + + var c *github.Client + c, err := github.NewEnterpriseClient(baseURL, uploadURL, oauth) + if err != nil { + return nil, fmt.Errorf("unable to parse --github-enterprise-url: %q, %w", gheURL, err) + } + + c.UserAgent = version.UserAgent + + return &Client{ + apiClient: c, + logger: logger, + }, nil } // validateAPIToken will ensure we have a valid github api token diff --git a/internal/core/signatures.go b/internal/core/signatures.go index c203706..0724795 100644 --- a/internal/core/signatures.go +++ b/internal/core/signatures.go @@ -369,7 +369,6 @@ func (s SafeFunctionSignature) ExtractMatch(file matchfile.MatchFile, sess *Sess // LoadSignatures will load all known signatures for the various match types into the session func LoadSignatures(filePath string, mLevel int, sess *Session) ([]Signature, error) { // TODO we don't need to bring in session here - // ensure that we have the proper home directory fp, err := util.SetHomeDir(filePath) if err != nil { @@ -380,7 +379,6 @@ func LoadSignatures(filePath string, mLevel int, sess *Session) ([]Signature, er if err != nil { return []Signature{}, fmt.Errorf("failed to load signatures file %s: %w", filePath, err) } - signaturesMetaData := SignaturesMetaData{ Version: c.Meta.Version, Date: c.Meta.Date, diff --git a/internal/pkg/signatures/fetch-git.go b/internal/pkg/signatures/fetch-git.go index 9449921..d79506d 100644 --- a/internal/pkg/signatures/fetch-git.go +++ b/internal/pkg/signatures/fetch-git.go @@ -45,7 +45,7 @@ func fetchSignaturesWithGit(version string, sess *core.Session) (string, error) } auth := &githttp.BasicAuth{ Username: "egal", - Password: sess.Config.GithubAccessToken, + Password: sess.Config.Signatures.APIToken, } // If we're gonna use git clone to get a specific tag, we need to pass git.AllTags as parameter here. diff --git a/internal/pkg/signatures/fetch-restapi.go b/internal/pkg/signatures/fetch-restapi.go index 6d82c8c..eb7f37b 100644 --- a/internal/pkg/signatures/fetch-restapi.go +++ b/internal/pkg/signatures/fetch-restapi.go @@ -27,7 +27,7 @@ func fetchSignaturesFromGithubAPI(version string, sess *core.Session) (string, e owner := res[0] repo := res[1] - client, err := _github.NewClient(sess.Config.GithubAccessToken, "", sess.Out) + client, err := _github.NewClient(sess.Config.Signatures.APIToken, "", sess.Out) if err != nil { return "", fmt.Errorf("failed instantiation of Github client, %w", err) } @@ -79,7 +79,7 @@ func downloadAsset(url string, sess *core.Session) (string, error) { // fetch from URL req, _ := http.NewRequest("GET", url, nil) - req.Header.Add("Authorization", fmt.Sprintf("token %s", sess.Config.GithubAccessToken)) + req.Header.Add("Authorization", fmt.Sprintf("token %s", sess.Config.Signatures.APIToken)) req.Header.Add("User-Agent", version.UserAgent) req.Header.Add("Accept", "application/octet-stream") @@ -91,6 +91,12 @@ func downloadAsset(url string, sess *core.Session) (string, error) { } defer resp.Body.Close() //nolint:errcheck + switch resp.StatusCode { + case 200: + default: + return "", &github.ErrorResponse{Response: resp} + } + // store file filename := fmt.Sprintf("%s/signatures/default.yaml", path) f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) diff --git a/internal/pkg/signatures/signatures.go b/internal/pkg/signatures/signatures.go index beff54f..e540cfc 100644 --- a/internal/pkg/signatures/signatures.go +++ b/internal/pkg/signatures/signatures.go @@ -61,12 +61,9 @@ func Update(log *log.Logger) error { log.Warn("Couldn't fetch the signatures from Github REST API, falling back to git method") dir, err = fetchSignaturesWithGit(cfg.SignaturesVersion, sess) if err != nil { - err = fmt.Errorf("couldn't fetch the signatures with git clone either, reason: %w", err) + return fmt.Errorf("couldn't fetch the signatures with git clone either, reason: %w", err) } } - if err != nil { - return err - } // install the signatures if updateSignatures(dir, sess, log) {