From b1d241c205874818caed67cddbddef6d1f179766 Mon Sep 17 00:00:00 2001 From: Rumen Vasilev Date: Thu, 7 Sep 2023 14:51:35 +0200 Subject: [PATCH] extract stats and webserver; reorganise session --- .editorconfig | 23 -- TODO.md | 2 + cmd/scan/scan.go | 7 +- internal/config/config.go | 198 ++++++++++ internal/core/analysis.go | 27 +- internal/core/api/structs.go | 10 + internal/core/banner.go | 17 + internal/core/findings.go | 8 +- internal/core/gh_worker.go | 6 +- internal/core/git.go | 34 +- internal/core/github.go | 52 +-- internal/core/gitlab.go | 20 +- internal/core/localRepo.go | 6 +- internal/core/session.go | 442 ++++++++++------------- internal/core/signatures.go | 2 +- internal/pkg/ghe/ghe.go | 35 +- internal/pkg/github/github.go | 52 ++- internal/pkg/gitlab/gitlab.go | 24 +- internal/pkg/localgit/localgit.go | 16 +- internal/pkg/localpath/localpath-impl.go | 22 +- internal/pkg/localpath/localpath.go | 21 +- internal/{core => stats}/stats.go | 205 ++--------- internal/util/io.go | 10 - internal/util/strings.go | 23 ++ internal/{core => webserver}/router.go | 41 ++- 25 files changed, 648 insertions(+), 655 deletions(-) delete mode 100644 .editorconfig create mode 100644 internal/config/config.go rename internal/{core => stats}/stats.go (52%) rename internal/{core => webserver}/router.go (78%) diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 5ee86ce..0000000 --- a/.editorconfig +++ /dev/null @@ -1,23 +0,0 @@ -# EditorConfig -# editorconfig.org - - -[*] -charset = utf-8 -end_of_line = lf -indent_size = 4 -indent_style = space -insert_final_newline = true -trim_trailing_whitespace = true - -[*.go] -indent_style = tab - -[*.{toml,yml,yaml}] -indent_size = 2 - -[{Makefile, makefile, GNUmakefile}] -indent_style = tab - -[*.md] -trim_trailing_whitespace = false diff --git a/TODO.md b/TODO.md index a387291..dfebc6b 100644 --- a/TODO.md +++ b/TODO.md @@ -19,6 +19,8 @@ - Implement proper server wait, not select{} - Store md5 in session, avoiding duplicate calc - New Makefile +- Overhaul updateRules.go +- Start webserver everywhere ## Features - Add exit codes, so CI could detect if scan failed diff --git a/cmd/scan/scan.go b/cmd/scan/scan.go index 77ff283..fc755a2 100644 --- a/cmd/scan/scan.go +++ b/cmd/scan/scan.go @@ -3,7 +3,7 @@ package scan import ( - "github.com/rumenvasilev/rvsecret/internal/core" + "github.com/rumenvasilev/rvsecret/internal/config" "github.com/rumenvasilev/rvsecret/version" "github.com/spf13/cobra" @@ -19,9 +19,8 @@ var ( ) func init() { - // TODO-RV: Move this to each sub-command - cobra.OnInitialize(core.SetConfig) - + cobra.OnInitialize(config.SetConfig) + // Global flags under `scan` command ScanCmd.PersistentFlags().String("bind-address", "127.0.0.1", "The IP address for the webserver") ScanCmd.PersistentFlags().Int("bind-port", 9393, "The port for the webserver") ScanCmd.PersistentFlags().Int("confidence-level", 3, "The confidence level of the expressions used to find matches") diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..6c090d3 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,198 @@ +package config + +import ( + "errors" + "fmt" + "os" + + "github.com/mitchellh/go-homedir" + "github.com/rumenvasilev/rvsecret/internal/pkg/api" + "github.com/rumenvasilev/rvsecret/internal/util" + "github.com/rumenvasilev/rvsecret/version" + "github.com/spf13/viper" +) + +// cfg holds the configuration data the commands +var cfg *viper.Viper + +// defaultIgnoreExtensions is an array of extensions that if they match a file that file will be excluded +var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff", + "tif", "psd", "xcf"} + +// defaultIgnorePaths is an array of directories that will be excluded from all types of scans. +var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cache", "/proc/"} + +// DefaultValues is a map of all flag default values and other mutable variables +var defaultValues = map[string]interface{}{ + "bind-address": "127.0.0.1", + "bind-port": 9393, + "commit-depth": -1, + "config-file": "$HOME/.rvsecret/config.yaml", + "csv": false, + "debug": false, + "ignore-extension": nil, + "ignore-path": nil, + "in-mem-clone": false, + "json": false, + "max-file-size": 10, + "num-threads": -1, + "local-paths": nil, + "scan-forks": false, + "scan-tests": false, + "scan-type": "", + "silent": false, + "confidence-level": 3, + "signature-file": "$HOME/.rvsecret/signatures/default.yaml", + "signature-path": "$HOME/.rvsecret/signatures/", + "signatures-path": "$HOME/.rvsecret/signatures/", + "signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures", + "signatures-version": "", + "scan-dir": nil, + "scan-file": nil, + "hide-secrets": false, + "rules-url": "", + "test-signatures": false, + "web-server": false, + // Github + "add-org-members": false, + "github-enterprise-url": "", + "github-url": "https://api.github.com", + "github-api-token": "", + "github-orgs": nil, + "github-repos": nil, + "github-users": nil, + // Gitlab + "gitlab-targets": nil, + //"gitlab-url": "", // TODO set the default + "gitlab-api-token": "", +} + +// SetConfig will set the defaults, and load a config file and environment variables if they are present +func SetConfig() { + for key, value := range defaultValues { + viper.SetDefault(key, value) + } + + configFile := viper.GetString("config-file") + + if configFile != defaultValues["config-file"] { + viper.SetConfigFile(configFile) + } else { + home, err := homedir.Dir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + viper.AddConfigPath(home + "/.rvsecret/") + viper.SetConfigName("config") + viper.SetConfigType("yaml") + } + + if err := viper.ReadInConfig(); err != nil { + fmt.Println("Couldn't load Viper config.", err) + // os.Exit(1) + } + + viper.AutomaticEnv() + + cfg = viper.GetViper() +} + +type Config struct { + AppVersion string + BindAddress string + BindPort int + CommitDepth int + ConfidenceLevel int + CSVOutput bool + Debug bool + ExpandOrgs bool + GithubAccessToken string + GithubEnterpriseURL string + GitlabAccessToken string + GitlabTargets []string + GitlabURL string + HideSecrets bool + InMemClone bool + JSONOutput bool + LocalPaths []string + MaxFileSize int64 + ScanFork bool + ScanTests bool + ScanType api.ScanType + Silent bool + SkippableExt []string + SkippablePath []string + SignatureFiles []string + Threads int + UserDirtyNames []string + UserDirtyOrgs []string + UserDirtyRepos []string + WebServer bool +} + +// TODO detect scanType automatically +func Load(scanType api.ScanType) (*Config, error) { + c := Config{ + BindAddress: cfg.GetString("bind-address"), + BindPort: cfg.GetInt("bind-port"), + CommitDepth: setCommitDepth(cfg.GetFloat64("commit-depth")), + CSVOutput: cfg.GetBool("csv"), + Debug: cfg.GetBool("debug"), + ExpandOrgs: cfg.GetBool("expand-orgs"), + GithubEnterpriseURL: cfg.GetString("github-enterprise-url"), + GithubAccessToken: cfg.GetString("github-api-token"), + UserDirtyRepos: cfg.GetStringSlice("github-repos"), + UserDirtyOrgs: cfg.GetStringSlice("github-orgs"), + UserDirtyNames: cfg.GetStringSlice("github-users"), + GitlabAccessToken: cfg.GetString("gitlab-api-token"), + GitlabTargets: cfg.GetStringSlice("gitlab-targets"), + HideSecrets: cfg.GetBool("hide-secrets"), + InMemClone: cfg.GetBool("in-mem-clone"), + JSONOutput: cfg.GetBool("json"), + MaxFileSize: cfg.GetInt64("max-file-size"), + ConfidenceLevel: cfg.GetInt("confidence-level"), + ScanFork: cfg.GetBool("scan-forks"), + ScanTests: cfg.GetBool("scan-tests"), + ScanType: scanType, + Silent: cfg.GetBool("silent"), + SignatureFiles: cfg.GetStringSlice("signature-file"), + Threads: cfg.GetInt("num-threads"), + AppVersion: version.AppVersion(), + WebServer: cfg.GetBool("web-server"), + } + + switch scanType { + case api.LocalGit: + c.LocalPaths = cfg.GetStringSlice("local-repos") + case api.LocalPath: + c.LocalPaths = cfg.GetStringSlice("local-paths") + case api.GithubEnterprise: + if c.GithubEnterpriseURL == "" { + return nil, errors.New("Github enterprise URL is not set.") + } + } + + // Add the default directories to the sess if they don't already exist + c.SkippablePath = util.AppendToSlice(true, defaultIgnorePaths, c.SkippablePath) + // add any additional paths the user requested to exclude to the pre-defined slice + c.SkippablePath = util.AppendToSlice(true, cfg.GetStringSlice("ignore-path"), c.SkippablePath) + + // the default ignorable extensions + c.SkippableExt = util.AppendToSlice(false, defaultIgnoreExtensions, c.SkippableExt) + // add any additional extensions the user requested to ignore + c.SkippableExt = util.AppendToSlice(true, cfg.GetStringSlice("ignore-extension"), c.SkippableExt) + + return &c, nil +} + +// setCommitDepth will set the commit depth for the current session. This is an ugly way of doing it +// but for the moment it works fine. +// TODO dynamically acquire the commit depth of a given repo +func setCommitDepth(c float64) int { + if c == -1 { + return 9999999999 + } + return int(c) +} diff --git a/internal/core/analysis.go b/internal/core/analysis.go index 0874c33..aefd023 100644 --- a/internal/core/analysis.go +++ b/internal/core/analysis.go @@ -10,6 +10,7 @@ import ( _coreapi "github.com/rumenvasilev/rvsecret/internal/core/api" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/matchfile" + "github.com/rumenvasilev/rvsecret/internal/stats" "github.com/rumenvasilev/rvsecret/internal/util" "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -21,9 +22,9 @@ import ( // are controlled by flags. If a directory, file, or the content pass through all of the filters then // it is scanned once per each signature which may lead to a specific secret matching multiple rules // and then generating multiple findings. -func AnalyzeRepositories(sess *Session, stats *Stats, log *log.Logger) { - stats.UpdateStatus(StatusAnalyzing) - repoCnt := len(sess.Repositories) +func AnalyzeRepositories(sess *Session, stats *stats.Stats, log *log.Logger) { + stats.UpdateStatus(_coreapi.StatusAnalyzing) + repoCnt := len(sess.State.Repositories) if repoCnt == 0 { log.Error("No repositories have been gathered.") } @@ -34,11 +35,11 @@ func AnalyzeRepositories(sess *Session, stats *Stats, log *log.Logger) { // Calculate the number of threads based on the flag and the number of repos. If the number of repos // being scanned is less than the number of threads the user requested, then the thread count is the // number of repos. - threadNum := sess.Threads - log.Debug("Defaulting threadNum to %d", sess.Threads) + threadNum := sess.Config.Threads + log.Debug("Defaulting threadNum to %d", sess.Config.Threads) if repoCnt <= 1 { threadNum = 1 - } else if repoCnt <= sess.Threads { + } else if repoCnt <= sess.Config.Threads { log.Debug("Setting threadNum to %d", repoCnt) threadNum = repoCnt } @@ -52,7 +53,7 @@ func AnalyzeRepositories(sess *Session, stats *Stats, log *log.Logger) { } // Feed repos to the analyzer workers - for _, repo := range sess.Repositories { + for _, repo := range sess.State.Repositories { ch <- *repo } @@ -62,7 +63,7 @@ func AnalyzeRepositories(sess *Session, stats *Stats, log *log.Logger) { wg.Wait() } -func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, sess *Session, stats *Stats, log *log.Logger) { +func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, sess *Session, stats *stats.Stats, log *log.Logger) { for { log.Debug("[THREAD #%d] Requesting new repository to analyze...", tid) repo, ok := <-ch @@ -102,7 +103,7 @@ func cleanUpPath(path string, log *log.Logger) { } func (sess *Session) analyzeHistory(clone *git.Repository, tid int, path string, repo _coreapi.Repository) { - stats := sess.Stats + stats := sess.State.Stats log := sess.Out // Get the full commit history for the repo @@ -142,7 +143,7 @@ func (sess *Session) analyzeHistory(clone *git.Repository, tid int, path string, // isDirtyCommit will analyze all the changes and return bool if there's a dirty commit func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Repository, clone *git.Repository, path string, tid int) bool { - stats := sess.Stats + stats := sess.State.Stats log := sess.Out // This will be used to increment the dirty commit stat if any matches are found. A dirty commit @@ -171,7 +172,7 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito mf := matchfile.New(fullFilePath) // Check if file has to be ignored - if ok, msg := ignoredFile(sess.ScanTests, sess.MaxFileSize, fullFilePath, mf, sess.SkippableExt, sess.SkippablePath); ok { + if ok, msg := ignoredFile(sess.Config.ScanTests, sess.Config.MaxFileSize, fullFilePath, mf, sess.Config.SkippableExt, sess.Config.SkippablePath); ok { log.Debug("[THREAD #%d][%s] %s %s", tid, repo.CloneURL, fPath, msg) stats.IncrementFilesIgnored() continue @@ -203,7 +204,7 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito for k, v := range matchMap { // Default to no content, only publish information if explicitly allowed to content = "" - if matchMap != nil && !sess.HideSecrets { + if matchMap != nil && !sess.Config.HideSecrets { // This sets the content for the finding, in this case the actual secret // is the content. This can be removed and hidden via a commandline flag. cleanK := strings.SplitAfterN(k, "_", 2) @@ -274,7 +275,7 @@ func createFinding(changeAction, content string, commit *object.Commit, sig Sign CommitMessage: strings.TrimSpace(commit.Message), Description: sig.Description(), FilePath: fPath, - AppVersion: sess.AppVersion, + AppVersion: sess.Config.AppVersion, LineNumber: strconv.Itoa(lineNum), RepositoryName: repo.Name, RepositoryOwner: repo.Owner, diff --git a/internal/core/api/structs.go b/internal/core/api/structs.go index 9883c57..88e709d 100644 --- a/internal/core/api/structs.go +++ b/internal/core/api/structs.go @@ -35,3 +35,13 @@ type Repository struct { Description string Homepage string } + +type Status string + +// These are various environment variables and tool statuses used in auth and displaying messages +const ( + StatusInitializing Status = "initializing" + StatusGathering Status = "gathering" + StatusAnalyzing Status = "analyzing" + StatusFinished Status = "finished" +) diff --git a/internal/core/banner.go b/internal/core/banner.go index 34d260b..6331690 100644 --- a/internal/core/banner.go +++ b/internal/core/banner.go @@ -3,7 +3,24 @@ package core import ( _ "embed" + "time" + + "github.com/rumenvasilev/rvsecret/internal/config" + "github.com/rumenvasilev/rvsecret/internal/log" + "github.com/rumenvasilev/rvsecret/internal/stats" + "github.com/rumenvasilev/rvsecret/version" ) //go:embed resources/banner.txt var ASCIIBanner string + +func HeaderInfo(cfg config.Config, stats *stats.Stats, log *log.Logger) { + if !cfg.JSONOutput && !cfg.CSVOutput { + log.Warn("%s", ASCIIBanner) + log.Important("%s v%s started at %s", version.Name, cfg.AppVersion, stats.StartedAt.Format(time.RFC3339)) + log.Important("Loaded %d signatures.", len(Signatures)) + if cfg.WebServer { + log.Important("Web interface available at http://%s:%d/public", cfg.BindAddress, cfg.BindPort) + } + } +} diff --git a/internal/core/findings.go b/internal/core/findings.go index ed35965..884f386 100644 --- a/internal/core/findings.go +++ b/internal/core/findings.go @@ -34,7 +34,7 @@ type Finding struct { // setupUrls will set the urls used to search through either github or gitlab for inclusion in the finding data func (f *Finding) setupUrls(sess *Session) { var baseURL string - switch sess.ScanType { + switch sess.Config.ScanType { // case api.GithubEnterprise: // baseURL = sess.GithubEnterpriseURL // SHOULD THIS BE THIS WAY? @@ -44,8 +44,8 @@ func (f *Finding) setupUrls(sess *Session) { case api.Github, api.GithubEnterprise: baseURL = "https://github.com" // TODO: IS THIS CORRECT?? - if sess.ScanType == api.GithubEnterprise { - baseURL = sess.GithubEnterpriseURL + if sess.Config.ScanType == api.GithubEnterprise { + baseURL = sess.Config.GithubEnterpriseURL } f.RepositoryURL = fmt.Sprintf("%s/%s/%s", baseURL, f.RepositoryOwner, f.RepositoryName) f.FileURL = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryURL, f.CommitHash, f.FilePath) @@ -65,7 +65,7 @@ func (f *Finding) Initialize(sess *Session) { } func (f *Finding) RealtimeOutput(sess *Session) { - if !sess.Silent && !sess.CSVOutput && !sess.JSONOutput { + if !sess.Config.Silent && !sess.Config.CSVOutput && !sess.Config.JSONOutput { log := sess.Out log.Warn(" %s", strings.ToUpper(f.Description)) log.Info(" SignatureID..........: %s", f.SignatureID) diff --git a/internal/core/gh_worker.go b/internal/core/gh_worker.go index a7718c4..09e3d51 100644 --- a/internal/core/gh_worker.go +++ b/internal/core/gh_worker.go @@ -46,10 +46,10 @@ func ghWorker(sess *Session, tid int, wg *sync.WaitGroup, ch chan *github.Organi // of the repos gathered for the org and the list pf repos that we care about. for _, repo := range repos { // Increment the total number of repos found even if we are not cloning them - sess.Stats.IncrementRepositoriesTotal() + sess.State.Stats.IncrementRepositoriesTotal() - if sess.UserRepos != nil { - for _, r := range sess.UserRepos { + if sess.GithubUserRepos != nil { + for _, r := range sess.GithubUserRepos { if r == repo.Name { log.Debug(" Retrieved repository %s", repo.FullName) // Add the repo to the sess to be scanned diff --git a/internal/core/git.go b/internal/core/git.go index 3527ba5..1a13320 100644 --- a/internal/core/git.go +++ b/internal/core/git.go @@ -158,24 +158,24 @@ func GetChangeContent(change *object.Change) (result string, contentError error) // InitGitClient will create a new git client of the type given by the input string. func (s *Session) InitGitClient() error { - switch s.ScanType { + switch s.Config.ScanType { case api.Github, api.GithubEnterprise: - client, err := _github.NewClient(s.GithubAccessToken, "", s.Out) + client, err := _github.NewClient(s.Config.GithubAccessToken, "", s.Out) if err != nil { return err } - if s.ScanType == api.GithubEnterprise { - if s.GithubEnterpriseURL == "" { + if s.Config.ScanType == api.GithubEnterprise { + if s.Config.GithubEnterpriseURL == "" { return fmt.Errorf("github enterprise URL is missing") } - client, err = _github.NewClient(s.GithubAccessToken, s.GithubEnterpriseURL, s.Out) + client, err = _github.NewClient(s.Config.GithubAccessToken, s.Config.GithubEnterpriseURL, s.Out) if err != nil { return err } } s.Client = client case api.Gitlab: - client, err := _gitlab.NewClient(s.GitlabAccessToken, s.Out) + client, err := _gitlab.NewClient(s.Config.GitlabAccessToken, s.Out) if err != nil { return fmt.Errorf("error initializing GitLab client: %s", err) } @@ -193,46 +193,46 @@ func cloneRepository(sess *Session, repo _coreapi.Repository) (*git.Repository, if err != nil { switch err.Error() { case "remote repository is empty": - sess.Stats.IncrementRepositoriesCloned() + sess.State.Stats.IncrementRepositoriesCloned() return nil, "", fmt.Errorf("failed cloning repository %s, it is empty, %w", repo.CloneURL, err) default: return nil, "", fmt.Errorf("failed cloning repository %s, %w", repo.CloneURL, err) } } - sess.Stats.IncrementRepositoriesCloned() + sess.State.Stats.IncrementRepositoriesCloned() return clone, path, err } func cloneRepositoryFunc(sess *Session, repo _coreapi.Repository) (*git.Repository, string, error) { var cloneConfig = CloneConfiguration{} var auth = http.BasicAuth{} - switch sess.ScanType { + switch sess.Config.ScanType { case api.Github, api.GithubEnterprise: cloneConfig = CloneConfiguration{ URL: repo.CloneURL, Branch: repo.DefaultBranch, - Depth: sess.CommitDepth, - InMemClone: sess.InMemClone, + Depth: sess.Config.CommitDepth, + InMemClone: sess.Config.InMemClone, // Token: sess.GithubAccessToken, } auth.Username = "doesn't matter" - auth.Password = sess.GithubAccessToken + auth.Password = sess.Config.GithubAccessToken case api.Gitlab: cloneConfig = CloneConfiguration{ URL: repo.CloneURL, Branch: repo.DefaultBranch, - Depth: sess.CommitDepth, - InMemClone: sess.InMemClone, + Depth: sess.Config.CommitDepth, + InMemClone: sess.Config.InMemClone, // Token: , // TODO Is this need since we already have a client? } auth.Username = "oauth2" - auth.Password = sess.GitlabAccessToken + auth.Password = sess.Config.GitlabAccessToken case api.LocalGit: cloneConfig = CloneConfiguration{ URL: repo.CloneURL, Branch: repo.DefaultBranch, - Depth: sess.CommitDepth, - InMemClone: sess.InMemClone, + Depth: sess.Config.CommitDepth, + InMemClone: sess.Config.InMemClone, } } return cloneRepositoryGeneric(cloneConfig, &auth) diff --git a/internal/core/github.go b/internal/core/github.go index 0999751..dc66844 100644 --- a/internal/core/github.go +++ b/internal/core/github.go @@ -18,8 +18,8 @@ import ( // addUser will add a new user to the sess for further scanning and analyzing func (s *Session) addUser(user *_coreapi.Owner) { - s.Lock() - defer s.Unlock() + s.State.Lock() + defer s.State.Unlock() h := md5.New() _, _ = io.WriteString(h, *user.Login) // TODO handle error _, _ = io.WriteString(h, strconv.FormatInt(*user.ID, 10)) // TODO handle error @@ -43,7 +43,7 @@ func GatherUsers(sess *Session) error { log := sess.Out log.Important("Gathering users...") ctx := context.Background() - for _, o := range sess.UserLogins { + for _, o := range sess.GithubUserLogins { owner, err := sess.Client.GetUserOrganization(ctx, o) if err != nil { // Should we not skip here? @@ -52,7 +52,7 @@ func GatherUsers(sess *Session) error { // Add the user to the session and increment the user count sess.addUser(owner) - sess.Stats.IncrementUsers() + sess.State.Stats.IncrementUsers() log.Debug("Added user %s", *owner.Login) } if len(sess.GithubUsers) == 0 { @@ -68,28 +68,28 @@ func (s *Session) ValidateUserInput() error { // if s.ScanType == api.GithubEnterprise { // If no targets are given, fail fast - if s.UserDirtyRepos == nil && s.UserDirtyOrgs == nil && s.UserDirtyNames == nil { + if s.Config.UserDirtyRepos == nil && s.Config.UserDirtyOrgs == nil && s.Config.UserDirtyNames == nil { return errors.New("you must enter either a user, org or repo[s] to scan") } // validate the input does not contain any scary characters exp := regexp.MustCompile(`[A-Za-z0-9,-_]*$`) - for _, v := range s.UserDirtyOrgs { + for _, v := range s.Config.UserDirtyOrgs { if exp.MatchString(v) { - s.UserOrgs = append(s.UserOrgs, v) + s.GithubUserOrgs = append(s.GithubUserOrgs, v) } } - for _, v := range s.UserDirtyRepos { + for _, v := range s.Config.UserDirtyRepos { if exp.MatchString(v) { - s.UserRepos = append(s.UserRepos, v) + s.GithubUserRepos = append(s.GithubUserRepos, v) } } - for _, v := range s.UserDirtyNames { + for _, v := range s.Config.UserDirtyNames { if exp.MatchString(v) { - s.UserLogins = append(s.UserLogins, v) + s.GithubUserLogins = append(s.GithubUserLogins, v) } } return nil @@ -109,7 +109,7 @@ func GatherGithubRepositoriesFromOwner(sess *Session) error { owner := _coreapi.Owner{Kind: util.StringToPointer(_coreapi.TargetTypeUser)} var err error // TODO This should be threaded - for _, ul := range sess.UserLogins { + for _, ul := range sess.GithubUserLogins { // Reset the Page to start for every user // opt.Page = 1 owner.Login = &ul @@ -130,9 +130,9 @@ func GatherGithubRepositoriesFromOwner(sess *Session) error { // of the repos gathered for the org and the list of repos that we care about. for _, repo := range allRepos { // Increment the total number of repos found, regardless if we are cloning them - sess.Stats.IncrementRepositoriesTotal() - if sess.UserRepos != nil { - for _, r := range sess.UserRepos { + sess.State.Stats.IncrementRepositoriesTotal() + if sess.GithubUserRepos != nil { + for _, r := range sess.GithubUserRepos { log.Debug("current repo: %s, comparing to: %s", r, repo.Name) if r == repo.Name { log.Debug(" Retrieved repository %s from user %s", repo.FullName, repo.Owner) @@ -193,7 +193,7 @@ func GatherOrgs(sess *Session, log *log.Logger) error { // } // } else { // This will handle orgs passed in via flags - for _, o := range sess.UserOrgs { + for _, o := range sess.GithubUserOrgs { owner, err := sess.Client.GetUserOrganization(ctx, o) if err != nil { log.Error("Error gathering the Github org %s: %s", o, err) @@ -209,7 +209,7 @@ func GatherOrgs(sess *Session, log *log.Logger) error { // Add the orgs to the list for later enumeration of repos for _, org := range orgList { sess.addOrganization(org) - sess.Stats.IncrementOrgs() + sess.State.Stats.IncrementOrgs() log.Debug("Added org %s", *org.Login) } return nil @@ -240,8 +240,8 @@ func ownerToOrg(owner *_coreapi.Owner) *github.Organization { // addOrganization will add a new organization to the session for further scanning and analyzing func (s *Session) addOrganization(organization *github.Organization) { - s.Lock() - defer s.Unlock() + s.State.Lock() + defer s.State.Unlock() h := md5.New() _, _ = io.WriteString(h, *organization.Login) // TODO handle these errors instead of ignoring them explictly _, _ = io.WriteString(h, strconv.FormatInt(*organization.ID, 10)) @@ -270,10 +270,10 @@ func GatherGithubOrgRepositories(sess *Session, log *log.Logger) error { // Calculate the number of threads based on the flag and the number of orgs // TODO: implement nice in the threading logic to guard against rate limiting and tripping the // security protections - threadNum := sess.Threads + threadNum := sess.Config.Threads if orgsCnt <= 1 { threadNum = 1 - } else if orgsCnt <= sess.Threads { + } else if orgsCnt <= sess.Config.Threads { threadNum = orgsCnt - 1 } wg.Add(threadNum) @@ -289,7 +289,7 @@ func GatherGithubOrgRepositories(sess *Session, log *log.Logger) error { } close(ch) wg.Wait() - if len(sess.Repositories) == 0 { + if len(sess.State.Repositories) == 0 { return fmt.Errorf("no repositories have been found for any of the provided Github organizations") } return nil @@ -317,7 +317,7 @@ func GatherOrgsMembersRepositories(sess *Session) { } for _, v := range members { sess.addUser(v) - sess.Stats.IncrementUsers() + sess.State.Stats.IncrementUsers() log.Debug("Added user %s", *v.Login) // find all repositories allRepos, err = sess.Client.GetRepositoriesFromOwner(ctx, *v) @@ -332,9 +332,9 @@ func GatherOrgsMembersRepositories(sess *Session) { // of the repos gathered for the org and the list of repos that we care about. for _, repo := range allRepos { // Increment the total number of repos found, regardless if we are cloning them - sess.Stats.IncrementRepositoriesTotal() - if sess.UserRepos != nil { - for _, r := range sess.UserRepos { + sess.State.Stats.IncrementRepositoriesTotal() + if sess.GithubUserRepos != nil { + for _, r := range sess.GithubUserRepos { if r == repo.Name { sess.Out.Debug(" Retrieved repository %s from user %s", repo.FullName, repo.Owner) sess.AddRepository(repo) diff --git a/internal/core/gitlab.go b/internal/core/gitlab.go index b13f46a..1d9de8b 100644 --- a/internal/core/gitlab.go +++ b/internal/core/gitlab.go @@ -11,7 +11,7 @@ import ( // GatherTargets will enumerate git targets adding them to a running target list. This will set the targets based // on the scan type set within the cmd package. func GatherTargets(sess *Session) { - sess.Stats.UpdateStatus(StatusGathering) + sess.State.Stats.UpdateStatus(_coreapi.StatusGathering) sess.Out.Important("Gathering targets...") ctx := context.Background() @@ -23,7 +23,7 @@ func GatherTargets(sess *Session) { //case "github": // targets = sess.GithubTargets //case "gitlab": - targets := sess.GitlabTargets + targets := sess.Config.GitlabTargets //} //var target *Owner @@ -53,7 +53,7 @@ func GatherTargets(sess *Session) { sess.Out.Debug("%s (ID: %d) type: %s", *target.Login, *target.ID, *target.Type) sess.AddTarget(target) // If forking is false AND the target type is an Organization as set above in GetUserOrganization - if sess.ExpandOrgs && *target.Type == _coreapi.TargetTypeOrganization { + if sess.Config.ExpandOrgs && *target.Type == _coreapi.TargetTypeOrganization { sess.Out.Debug("Gathering members of %s (ID: %d)...", *target.Login, *target.ID) members, err := sess.Client.GetOrganizationMembers(ctx, *target) if err != nil { @@ -76,16 +76,16 @@ func GatherTargets(sess *Session) { func GatherGitlabRepositories(sess *Session) { log := sess.Out ctx := context.Background() - var ch = make(chan *_coreapi.Owner, len(sess.Targets)) - log.Debug("Number of targets: %d", len(sess.Targets)) + var ch = make(chan *_coreapi.Owner, len(sess.State.Targets)) + log.Debug("Number of targets: %d", len(sess.State.Targets)) var wg sync.WaitGroup var threadNum int - if len(sess.Targets) == 1 { + if len(sess.State.Targets) == 1 { threadNum = 1 - } else if len(sess.Targets) <= sess.Threads { - threadNum = len(sess.Targets) - 1 + } else if len(sess.State.Targets) <= sess.Config.Threads { + threadNum = len(sess.State.Targets) - 1 } else { - threadNum = sess.Threads + threadNum = sess.Config.Threads } wg.Add(threadNum) log.Debug("Threads for repository gathering: %d", threadNum) @@ -113,7 +113,7 @@ func GatherGitlabRepositories(sess *Session) { }() } - for _, target := range sess.Targets { + for _, target := range sess.State.Targets { ch <- target } close(ch) diff --git a/internal/core/localRepo.go b/internal/core/localRepo.go index 1e8e729..a6843f1 100644 --- a/internal/core/localRepo.go +++ b/internal/core/localRepo.go @@ -21,11 +21,11 @@ func GatherLocalRepositories(sess *Session) error { // This is the number of targets as we don't do forks or anything else. // It will contain directories, that will then be added to the repo count // if they contain a .git directory - sess.Stats.Targets = len(sess.LocalPaths) - sess.Stats.UpdateStatus(StatusGathering) + sess.State.Stats.Targets = len(sess.Config.LocalPaths) + sess.State.Stats.UpdateStatus(_coreapi.StatusGathering) sess.Out.Important("Gathering Local Repositories...") - for _, pth := range sess.LocalPaths { + for _, pth := range sess.Config.LocalPaths { if !util.PathExists(pth, log) { return fmt.Errorf("[*] <%s> does not exist! Quitting.", pth) diff --git a/internal/core/session.go b/internal/core/session.go index a07b014..52113b9 100644 --- a/internal/core/session.go +++ b/internal/core/session.go @@ -2,279 +2,97 @@ package core import ( - "errors" + "encoding/csv" + "encoding/json" "fmt" "os" "runtime" + "sort" "strings" "sync" "time" + "github.com/gin-gonic/gin" "github.com/google/go-github/github" - "github.com/mitchellh/go-homedir" + "github.com/rumenvasilev/rvsecret/internal/config" _coreapi "github.com/rumenvasilev/rvsecret/internal/core/api" "github.com/rumenvasilev/rvsecret/internal/core/provider" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" + "github.com/rumenvasilev/rvsecret/internal/stats" "github.com/rumenvasilev/rvsecret/internal/util" - "github.com/rumenvasilev/rvsecret/version" - - "github.com/gin-gonic/gin" - "github.com/spf13/viper" -) - -// cfg holds the configuration data the commands -var cfg *viper.Viper - -type Status string - -// These are various environment variables and tool statuses used in auth and displaying messages -const ( - StatusInitializing Status = "initializing" - StatusGathering Status = "gathering" - StatusAnalyzing Status = "analyzing" - StatusFinished Status = "finished" ) -// defaultIgnoreExtensions is an array of extensions that if they match a file that file will be excluded -var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff", - "tif", "psd", "xcf"} - -// defaultIgnorePaths is an array of directories that will be excluded from all types of scans. -var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cache", "/proc/"} - -// DefaultValues is a map of all flag default values and other mutable variables -var DefaultValues = map[string]interface{}{ - "bind-address": "127.0.0.1", - "bind-port": 9393, - // "commit-depth": -1, - "config-file": "$HOME/.rvsecret/config.yaml", - "csv": false, - "debug": false, - "add-org-members": false, - "github-enterprise-url": "", - "github-url": "https://api.github.com", - "github-api-token": "", - "github-orgs": nil, - "github-repos": nil, - "github-users": nil, - "gitlab-targets": nil, - //"gitlab-url": "", // TODO set the default - "gitlab-api-token": "", - "ignore-extension": nil, - "ignore-path": nil, - "in-mem-clone": false, - "json": false, - "max-file-size": 10, - "num-threads": -1, - "local-paths": nil, - "scan-forks": false, - "scan-tests": false, - "scan-type": "", - "silent": false, - "confidence-level": 3, - "signature-file": "$HOME/.rvsecret/signatures/default.yaml", - "signature-path": "$HOME/.rvsecret/signatures/", - "scan-dir": nil, - "scan-file": nil, - "hide-secrets": false, - "rules-url": "", - "signatures-path": "$HOME/.rvsecret/signatures/", - "signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures", - "signatures-version": "", - "test-signatures": false, - "web-server": false, -} - // Session contains all the necessary values and parameters used during a scan type Session struct { - sync.Mutex - AppVersion string - BindAddress string - BindPort int + Config *config.Config // Client holds the client for the target git server (github, gitlab) - Client provider.IClient `json:"-"` - CommitDepth int - ConfidenceLevel int - CSVOutput bool - Debug bool - ExpandOrgs bool - Findings []*Finding - GithubAccessToken string - // GithubClient *github.Client `json:"-"` - GithubEnterpriseURL string - // GithubURL string - GitlabAccessToken string - GitlabTargets []string - GitlabURL string + Client provider.IClient `json:"-"` + State *State // GithubUsers []*github.User GithubUsers []*_coreapi.Owner - HideSecrets bool - InMemClone bool - JSONOutput bool - LocalPaths []string - MaxFileSize int64 + GithubUserLogins []string + GithubUserOrgs []string + GithubUserRepos []string Organizations []*github.Organization Out *log.Logger `json:"-"` - Repositories []*_coreapi.Repository Router *gin.Engine `json:"-"` - SignatureVersion string - ScanFork bool - ScanTests bool - ScanType api.ScanType Signatures []*Signature - Silent bool - SkippableExt []string - SkippablePath []string - Stats *Stats - Targets []*_coreapi.Owner - Threads int - UserDirtyNames []string - UserDirtyOrgs []string - UserDirtyRepos []string - UserLogins []string - UserOrgs []string - UserRepos []string - WebServer bool + SignatureVersion string } -// githubRepository is the holds the necessary fields in a simpler structure -// type githubRepository struct { -// Owner *string -// ID *int64 -// Name *string -// FullName *string -// CloneURL *string -// URL *string -// DefaultBranch *string -// Description *string -// Homepage *string -// } +type State struct { + sync.Mutex + Stats *stats.Stats + Findings []*Finding + Targets []*_coreapi.Owner + Repositories []*_coreapi.Repository +} // NewSession is the entry point for starting a new scan session func NewSession(scanType api.ScanType, log *log.Logger) (*Session, error) { - var session Session - err := session.Initialize(scanType, log) + cfg, err := config.Load(scanType) if err != nil { return nil, err } - return &session, nil + return NewSessionWithConfig(cfg, log) } -// SetConfig will set the defaults, and load a config file and environment variables if they are present -func SetConfig() { - for key, value := range DefaultValues { - viper.SetDefault(key, value) - } - - configFile := viper.GetString("config-file") - - if configFile != DefaultValues["config-file"] { - viper.SetConfigFile(configFile) - } else { - home, err := homedir.Dir() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - viper.AddConfigPath(home + "/.rvsecret/") - viper.SetConfigName("config") - viper.SetConfigType("yaml") - } - - if err := viper.ReadInConfig(); err != nil { - fmt.Println("Couldn't load Viper config") - os.Exit(1) +func NewSessionWithConfig(cfg *config.Config, log *log.Logger) (*Session, error) { + var session Session + err := session.Initialize(cfg, log) + if err != nil { + return nil, err } - - viper.AutomaticEnv() - - cfg = viper.GetViper() + return &session, nil } // Initialize will set the initial values and options used during a scan session -func (s *Session) Initialize(scanType api.ScanType, log *log.Logger) error { +func (s *Session) Initialize(cfg *config.Config, log *log.Logger) error { s.Out = log - s.BindAddress = cfg.GetString("bind-address") - s.BindPort = cfg.GetInt("bind-port") - s.CommitDepth = setCommitDepth(cfg.GetFloat64("commit-depth")) - s.CSVOutput = cfg.GetBool("csv") - s.Debug = cfg.GetBool("debug") - s.ExpandOrgs = cfg.GetBool("expand-orgs") - s.GithubEnterpriseURL = cfg.GetString("github-enterprise-url") - s.GithubAccessToken = cfg.GetString("github-api-token") - s.UserDirtyRepos = cfg.GetStringSlice("github-repos") - s.UserDirtyOrgs = cfg.GetStringSlice("github-orgs") - s.UserDirtyNames = cfg.GetStringSlice("github-users") - s.GitlabAccessToken = cfg.GetString("gitlab-api-token") - s.GitlabTargets = cfg.GetStringSlice("gitlab-targets") - s.HideSecrets = cfg.GetBool("hide-secrets") - s.InMemClone = cfg.GetBool("in-mem-clone") - s.JSONOutput = cfg.GetBool("json") - s.MaxFileSize = cfg.GetInt64("max-file-size") - s.ConfidenceLevel = cfg.GetInt("confidence-level") - s.ScanFork = cfg.GetBool("scan-forks") - s.ScanTests = cfg.GetBool("scan-tests") - s.ScanType = scanType - s.Silent = cfg.GetBool("silent") - s.Threads = cfg.GetInt("num-threads") - s.AppVersion = version.AppVersion() - s.WebServer = cfg.GetBool("web-server") - - switch s.ScanType { - case api.LocalGit: - s.LocalPaths = cfg.GetStringSlice("local-repos") - case api.LocalPath: - s.LocalPaths = cfg.GetStringSlice("local-paths") - case api.GithubEnterprise: - if s.GithubEnterpriseURL == "" { - return errors.New("Github enterprise URL is not set.") - } - } - - // Add the default directories to the sess if they don't already exist - for _, e := range defaultIgnorePaths { - e = strings.TrimSpace(e) - s.SkippablePath = util.AppendIfMissing(s.SkippablePath, e) - } - - // add any additional paths the user requested to exclude to the pre-defined slice - for _, e := range cfg.GetStringSlice("ignore-path") { - e = strings.TrimSpace(e) - s.SkippablePath = util.AppendIfMissing(s.SkippablePath, e) - } - - // the default ignorable extensions - for _, e := range defaultIgnoreExtensions { - s.SkippableExt = util.AppendIfMissing(s.SkippableExt, e) - } - - // add any additional extensions the user requested to ignore - for _, f := range cfg.GetStringSlice("ignore-extension") { - f = strings.TrimSpace(f) - s.SkippableExt = util.AppendIfMissing(s.SkippableExt, f) - } + s.Config = cfg + s.State = &State{} + s.State.Stats = stats.Init() - s.InitStats() s.InitThreads() - if !s.Silent && s.WebServer { - s.InitRouter() - } + // if !s.Silent && s.WebServer { + // s.InitRouter() + // } var curSig []Signature var combinedSig []Signature + // signaturessss // TODO need to catch this error here - for _, f := range cfg.GetStringSlice("signature-file") { + for _, f := range cfg.SignatureFiles { f = strings.TrimSpace(f) h, err := util.SetHomeDir(f) if err != nil { return err } if util.PathExists(h, s.Out) { - curSig, err = LoadSignatures(h, s.ConfidenceLevel, s) + curSig, err = LoadSignatures(h, cfg.ConfidenceLevel, s) if err != nil { return err } @@ -285,79 +103,58 @@ func (s *Session) Initialize(scanType api.ScanType, log *log.Logger) error { return nil } -// setCommitDepth will set the commit depth for the current session. This is an ugly way of doing it -// but for the moment it works fine. -// TODO dynamically acquire the commit depth of a given repo -func setCommitDepth(c float64) int { - if c == -1 { - return 9999999999 - } - return int(c) -} - // Finish is called at the end of a scan session and used to generate discrete data points // for a given scan session including setting the status of a scan to finished. func (s *Session) Finish() { - s.Stats.FinishedAt = time.Now() - s.Stats.UpdateStatus(StatusFinished) + s.State.Stats.FinishedAt = time.Now() + s.State.Stats.UpdateStatus(_coreapi.StatusFinished) } // AddTarget will add a new target to a session to be scanned during that session func (s *Session) AddTarget(target *_coreapi.Owner) { - s.Lock() - defer s.Unlock() - for _, t := range s.Targets { + s.State.Lock() + defer s.State.Unlock() + for _, t := range s.State.Targets { if *target.ID == *t.ID { return } } - s.Targets = append(s.Targets, target) - s.Stats.IncrementTargets() + s.State.Targets = append(s.State.Targets, target) + s.State.Stats.IncrementTargets() } // AddRepository will add a given repository to be scanned to a session. This counts as // the total number of repos that have been gathered during a session. func (s *Session) AddRepository(repository *_coreapi.Repository) { - s.Lock() - defer s.Unlock() - for _, r := range s.Repositories { + s.State.Lock() + defer s.State.Unlock() + for _, r := range s.State.Repositories { if repository.ID == r.ID { return } } - s.Repositories = append(s.Repositories, repository) + s.State.Repositories = append(s.State.Repositories, repository) } // AddFinding will add a finding that has been discovered during a session to the list of findings // for that session func (s *Session) AddFinding(finding *Finding) { - s.Lock() - defer s.Unlock() + s.State.Lock() + defer s.State.Unlock() // const MaxStrLen = 100 - s.Findings = append(s.Findings, finding) - s.Stats.IncrementFindingsTotal() + s.State.Findings = append(s.State.Findings, finding) + s.State.Stats.IncrementFindingsTotal() } // InitThreads will set the correct number of threads based on the commandline flags func (s *Session) InitThreads() { - if s.Threads <= 0 { + if s.Config.Threads <= 0 { numCPUs := runtime.NumCPU() - s.Threads = numCPUs + s.Config.Threads = numCPUs s.Out.Debug("Setting threads to %d", numCPUs) } - runtime.GOMAXPROCS(s.Threads + 2) // thread count + main + web server -} - -// InitRouter will configure and start the webserver for graphical output and status messages -func (s *Session) InitRouter() { - bind := fmt.Sprintf("%s:%d", s.BindAddress, s.BindPort) - s.Router = NewRouter(s) - go func(sess *Session) { - if err := sess.Router.Run(bind); err != nil { - sess.Out.Fatal("Error when starting web server: %s", err) - } - }(s) + runtime.GOMAXPROCS(s.Config.Threads + 2) // thread count + main + web server } // SaveToFile will save a json representation of the session output to a file @@ -372,3 +169,130 @@ func (s *Session) InitRouter() { // } // return nil // } + +// PrintDebug will print a debug header at the start of the session that displays specific setting +func PrintDebug(sess *Session) { + maxFileSize := sess.Config.MaxFileSize * 1024 * 1024 + sess.Out.Debug("\n") + sess.Out.Debug("Debug Info") + sess.Out.Debug("App version..............%v", sess.Config.AppVersion) + sess.Out.Debug("Signatures version.......%v", sess.SignatureVersion) + sess.Out.Debug("Scanning tests...........%v", sess.Config.ScanTests) + sess.Out.Debug("Max file size............%d", maxFileSize) + sess.Out.Debug("JSON output..............%v", sess.Config.JSONOutput) + sess.Out.Debug("CSV output...............%v", sess.Config.CSVOutput) + sess.Out.Debug("Silent output............%v", sess.Config.Silent) + sess.Out.Debug("Web server enabled.......%v", sess.Config.WebServer) + // log.Debug("") +} + +// PrintSessionStats will print the performance and sessions stats to stdout at the conclusion of a session scan +func printSessionStats(s *stats.Stats, log *log.Logger, appVersion, signatureVersion string) { + log.Important("\n--------Results--------") + log.Important("") + log.Important("-------Findings------") + log.Info("Total Findings......: %d", s.Findings) + log.Important("") + log.Important("--------Files--------") + log.Info("Total Files.........: %d", s.FilesTotal) + log.Info("Files Scanned.......: %d", s.FilesScanned) + log.Info("Files Ignored.......: %d", s.FilesIgnored) + log.Info("Files Dirty.........: %d", s.FilesDirty) + log.Important("") + log.Important("---------SCM---------") + log.Info("Repos Found.........: %d", s.RepositoriesTotal) + log.Info("Repos Cloned........: %d", s.RepositoriesCloned) + log.Info("Repos Scanned.......: %d", s.RepositoriesScanned) + log.Info("Commits Total.......: %d", s.CommitsTotal) + log.Info("Commits Scanned.....: %d", s.CommitsScanned) + log.Info("Commits Dirty.......: %d", s.CommitsDirty) + log.Important("") + log.Important("-------General-------") + log.Info("App Version.........: %s", appVersion) + log.Info("Signatures Version..: %s", signatureVersion) + log.Info("Elapsed Time........: %s", time.Since(s.StartedAt)) + log.Info("") +} + +// SummaryOutput will spit out the results of the hunt along with performance data +func SummaryOutput(sess *Session) { + + // alpha sort the findings to make the results idempotent + if len(sess.State.Findings) > 0 { + sort.Slice(sess.State.Findings, func(i, j int) bool { + return sess.State.Findings[i].SecretID < sess.State.Findings[j].SecretID + }) + } + + if sess.Config.JSONOutput { + if len(sess.State.Findings) > 0 { + b, err := json.MarshalIndent(sess.State.Findings, "", " ") + if err != nil { + fmt.Println(err) + return + } + c := string(b) + if c == "null" { + fmt.Println("[]") + } else { + fmt.Println(c) + } + } else { + fmt.Println("[]") + } + } + + if sess.Config.CSVOutput { + w := csv.NewWriter(os.Stdout) + defer w.Flush() + header := []string{ + "FilePath", + "Line Number", + "Action", + "Description", + "SignatureID", + "Finding List", + "Repo Owner", + "Repo Name", + "Commit Hash", + "Commit Message", + "Commit Author", + "File URL", + "Secret ID", + "App Version", + "Signatures Version", + } + err := w.Write(header) + if err != nil { + sess.Out.Error(err.Error()) + } + + for _, v := range sess.State.Findings { + line := []string{ + v.FilePath, + v.LineNumber, + v.Action, + v.Description, + v.SignatureID, + v.Content, + v.RepositoryOwner, + v.RepositoryName, + v.CommitHash, + v.CommitMessage, + v.CommitAuthor, + v.FileURL, + v.SecretID, + v.AppVersion, + v.SignatureVersion, + } + err := w.Write(line) + if err != nil { + sess.Out.Error(err.Error()) + } + } + } + + if !sess.Config.JSONOutput && !sess.Config.CSVOutput { + printSessionStats(sess.State.Stats, sess.Out, sess.Config.AppVersion, sess.SignatureVersion) + } +} diff --git a/internal/core/signatures.go b/internal/core/signatures.go index 629ea41..a5663f9 100644 --- a/internal/core/signatures.go +++ b/internal/core/signatures.go @@ -251,7 +251,7 @@ func (s PatternSignature) ExtractMatch(file matchfile.MatchFile, sess *Session, } } - if sess.ScanType != "localPath" { + if sess.Config.ScanType != "localPath" { content, err := GetChangeContent(change) if err != nil { sess.Out.Error("Error retrieving content in commit %s, change %s: %s", "commit.String()", change.String(), err) diff --git a/internal/pkg/ghe/ghe.go b/internal/pkg/ghe/ghe.go index d93b29b..66ebd22 100644 --- a/internal/pkg/ghe/ghe.go +++ b/internal/pkg/ghe/ghe.go @@ -2,12 +2,10 @@ package ghe import ( "fmt" - "time" "github.com/rumenvasilev/rvsecret/internal/core" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" - "github.com/rumenvasilev/rvsecret/version" ) func Scan(log *log.Logger) error { @@ -24,18 +22,11 @@ func Scan(log *log.Logger) error { // By default we display a header to the user giving basic info about application. This will not be displayed // during a silent run which is the default when using this in an automated fashion. - if !sess.JSONOutput && !sess.CSVOutput { - log.Warn("%s", core.ASCIIBanner) - log.Important("%s v%s started at %s", version.Name, sess.AppVersion, sess.Stats.StartedAt.Format(time.RFC3339)) - log.Important("Loaded %d signatures.", len(core.Signatures)) - if sess.WebServer { - log.Important("Web interface available at http://%s:%d/public", sess.BindAddress, sess.BindPort) - } - } + core.HeaderInfo(*sess.Config, sess.State.Stats, sess.Out) - log.Debug("We have these orgs: %s", sess.UserOrgs) - log.Debug("We have these users: %s", sess.UserLogins) - log.Debug("We have these repos: %s", sess.UserRepos) + log.Debug("We have these orgs: %s", sess.GithubUserOrgs) + log.Debug("We have these users: %s", sess.GithubUserLogins) + log.Debug("We have these repos: %s", sess.GithubUserRepos) // Create a github client to be used for the session err = sess.InitGitClient() @@ -45,8 +36,8 @@ func Scan(log *log.Logger) error { // If we have github users and no orgs or repos then we default to scan // the visible repos of that user. - if sess.UserLogins != nil { - if sess.UserOrgs == nil && sess.UserRepos == nil { + if sess.GithubUserLogins != nil { + if sess.GithubUserOrgs == nil && sess.GithubUserRepos == nil { err = core.GatherUsers(sess) if err != nil { return err @@ -55,8 +46,8 @@ func Scan(log *log.Logger) error { } // If the user has only given orgs then we grab all te repos from those orgs - if sess.UserOrgs != nil { - if sess.UserLogins == nil && sess.UserRepos == nil { + if sess.GithubUserOrgs != nil { + if sess.GithubUserLogins == nil && sess.GithubUserRepos == nil { err = core.GatherOrgs(sess, log) if err != nil { return err @@ -66,8 +57,8 @@ func Scan(log *log.Logger) error { // If we have repo(s) given we need to ensure that we also have orgs or users. rvsecret will then // look for the repo in the user or login lists and scan it. - if sess.UserRepos != nil { - if sess.UserOrgs != nil { + if sess.GithubUserRepos != nil { + if sess.GithubUserOrgs != nil { err = core.GatherOrgs(sess, log) if err != nil { return err @@ -76,7 +67,7 @@ func Scan(log *log.Logger) error { if err != nil { return err } - } else if sess.UserLogins != nil { + } else if sess.GithubUserLogins != nil { err = core.GatherUsers(sess) if err != nil { return err @@ -90,12 +81,12 @@ func Scan(log *log.Logger) error { } } - core.AnalyzeRepositories(sess, sess.Stats, log) + core.AnalyzeRepositories(sess, sess.State.Stats, log) sess.Finish() core.SummaryOutput(sess) - if !sess.Silent && sess.WebServer { + if !sess.Config.Silent && sess.Config.WebServer { log.Important("Press Ctrl+C to stop web server and exit.") select {} } diff --git a/internal/pkg/github/github.go b/internal/pkg/github/github.go index 58a80ed..44b6530 100644 --- a/internal/pkg/github/github.go +++ b/internal/pkg/github/github.go @@ -2,16 +2,23 @@ package github import ( "fmt" - "time" + "github.com/rumenvasilev/rvsecret/internal/config" "github.com/rumenvasilev/rvsecret/internal/core" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" - "github.com/rumenvasilev/rvsecret/version" + "github.com/rumenvasilev/rvsecret/internal/webserver" ) func Scan(log *log.Logger) error { - sess, err := core.NewSession(api.Github, log) + // load config + cfg, err := config.Load(api.Github) + if err != nil { + return err + } + + // create session + sess, err := core.NewSessionWithConfig(cfg, log) if err != nil { return err } @@ -22,24 +29,29 @@ func Scan(log *log.Logger) error { return err } + // Start webserver + // TODO all of that could be a normalized session struct + if sess.Config.WebServer && !sess.Config.Silent { + ws := webserver.New(webserver.Configuration{ + ScanType: api.Github, + Debug: cfg.Debug, + Logger: log, + State: sess.State, + }) + go webserver.Start(ws, sess.Config.BindAddress, sess.Config.BindPort, log) + } + // By default we display a header to the user giving basic info about application. This will not be displayed // during a silent run which is the default when using this in an automated fashion. - if !sess.JSONOutput && !sess.CSVOutput { - log.Warn("%s", core.ASCIIBanner) - log.Important("%s v%s started at %s", version.Name, sess.AppVersion, sess.Stats.StartedAt.Format(time.RFC3339)) - log.Important("Loaded %d signatures.", len(core.Signatures)) - if sess.WebServer { - log.Important("Web interface available at http://%s:%d/public", sess.BindAddress, sess.BindPort) - } - } + core.HeaderInfo(*sess.Config, sess.State.Stats, sess.Out) - if sess.Debug { + if sess.Config.Debug { core.PrintDebug(sess) } - log.Debug("We have these orgs: %s", sess.UserOrgs) - log.Debug("We have these users: %s", sess.UserLogins) - log.Debug("We have these repos: %s", sess.UserRepos) + log.Debug("We have these orgs: %s", sess.GithubUserOrgs) + log.Debug("We have these users: %s", sess.GithubUserLogins) + log.Debug("We have these repos: %s", sess.GithubUserRepos) //Create a github client to be used for the session err = sess.InitGitClient() @@ -47,7 +59,7 @@ func Scan(log *log.Logger) error { return err } - if sess.UserLogins != nil { + if sess.GithubUserLogins != nil { err = core.GatherUsers(sess) if err != nil { return err @@ -56,10 +68,10 @@ func Scan(log *log.Logger) error { if err != nil { return err } - } else if sess.ExpandOrgs && sess.UserOrgs != nil { + } else if sess.Config.ExpandOrgs && sess.GithubUserOrgs != nil { // FIXME: this should be from --add-org-members core.GatherOrgsMembersRepositories(sess) - } else if sess.UserOrgs != nil { + } else if sess.GithubUserOrgs != nil { err = core.GatherOrgs(sess, log) if err != nil { return err @@ -74,12 +86,12 @@ func Scan(log *log.Logger) error { return fmt.Errorf("please specify an org or user that contains the repo(s)") } - core.AnalyzeRepositories(sess, sess.Stats, log) + core.AnalyzeRepositories(sess, sess.State.Stats, log) sess.Finish() core.SummaryOutput(sess) - if !sess.Silent && sess.WebServer { + if !sess.Config.Silent && sess.Config.WebServer { log.Important("Press Ctrl+C to stop web server and exit.") select {} } diff --git a/internal/pkg/gitlab/gitlab.go b/internal/pkg/gitlab/gitlab.go index 8d44958..b8dc973 100644 --- a/internal/pkg/gitlab/gitlab.go +++ b/internal/pkg/gitlab/gitlab.go @@ -1,12 +1,9 @@ package gitlab import ( - "time" - "github.com/rumenvasilev/rvsecret/internal/core" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" - "github.com/rumenvasilev/rvsecret/version" "github.com/spf13/viper" ) @@ -17,22 +14,9 @@ func Scan(log *log.Logger) error { } // By default we display a header to the user giving basic info about application. This will not be displayed // during a silent run which is the default when using this in an automated fashion. - if !sess.JSONOutput && !sess.CSVOutput { - log.Warn("%s", core.ASCIIBanner) - log.Important("%s v%s started at %s", version.Name, sess.AppVersion, sess.Stats.StartedAt.Format(time.RFC3339)) - log.Important("Loaded %d signatures.", len(core.Signatures)) - if sess.WebServer { - log.Important("Web interface available at http://%s:%d/public", sess.BindAddress, sess.BindPort) - } - } - - if sess.Debug { - log.Debug("We have these orgs: %s", sess.UserOrgs) - log.Debug("We have these users: %s", sess.UserLogins) - log.Debug("We have these repos: %s", sess.UserRepos) - } + core.HeaderInfo(*sess.Config, sess.State.Stats, sess.Out) - sess.GitlabAccessToken = viper.GetString("gitlab-api-token") + sess.Config.GitlabAccessToken = viper.GetString("gitlab-api-token") err = sess.InitGitClient() if err != nil { @@ -41,12 +25,12 @@ func Scan(log *log.Logger) error { core.GatherTargets(sess) core.GatherGitlabRepositories(sess) - core.AnalyzeRepositories(sess, sess.Stats, log) + core.AnalyzeRepositories(sess, sess.State.Stats, log) sess.Finish() core.SummaryOutput(sess) - if !sess.Silent && sess.WebServer { + if !sess.Config.Silent && sess.Config.WebServer { log.Important("%s", core.ASCIIBanner) log.Important("Press Ctrl+C to stop web server and exit.") select {} diff --git a/internal/pkg/localgit/localgit.go b/internal/pkg/localgit/localgit.go index fc6fb23..a745675 100644 --- a/internal/pkg/localgit/localgit.go +++ b/internal/pkg/localgit/localgit.go @@ -1,12 +1,9 @@ package localgit import ( - "time" - "github.com/rumenvasilev/rvsecret/internal/core" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" - "github.com/rumenvasilev/rvsecret/version" ) func Scan(log *log.Logger) error { @@ -16,25 +13,18 @@ func Scan(log *log.Logger) error { } // By default we display a header to the user giving basic info about application. This will not be displayed // during a silent run which is the default when using this in an automated fashion. - if !sess.JSONOutput && !sess.CSVOutput { - log.Warn("%s", core.ASCIIBanner) - log.Important("%s v%s started at %s", version.Name, sess.AppVersion, sess.Stats.StartedAt.Format(time.RFC3339)) - log.Important("Loaded %d signatures.", len(core.Signatures)) - if sess.WebServer { - log.Important("Web interface available at http://%s:%d/public", sess.BindAddress, sess.BindPort) - } - } + core.HeaderInfo(*sess.Config, sess.State.Stats, sess.Out) err = core.GatherLocalRepositories(sess) if err != nil { return err } - core.AnalyzeRepositories(sess, sess.Stats, log) + core.AnalyzeRepositories(sess, sess.State.Stats, log) sess.Finish() core.SummaryOutput(sess) - if !sess.Silent && sess.WebServer { + if !sess.Config.Silent && sess.Config.WebServer { log.Important("Press Ctrl+C to stop web server and exit.") select {} } diff --git a/internal/pkg/localpath/localpath-impl.go b/internal/pkg/localpath/localpath-impl.go index 1c0c3c4..c929c47 100644 --- a/internal/pkg/localpath/localpath-impl.go +++ b/internal/pkg/localpath/localpath-impl.go @@ -24,45 +24,45 @@ func doFileScan(filename string, sess *core.Session) { likelyTestFile := false // This is the total number of files that we know exist in out path. This does not care about the scan, it is simply the total number of files found - sess.Stats.IncrementFilesTotal() + sess.State.Stats.IncrementFilesTotal() mf := matchfile.New(filename) - if mf.IsSkippable(sess.SkippableExt, sess.SkippablePath) { + if mf.IsSkippable(sess.Config.SkippableExt, sess.Config.SkippablePath) { sess.Out.Debug("%s is listed as skippable and is being ignored", filename) - sess.Stats.IncrementFilesIgnored() + sess.State.Stats.IncrementFilesIgnored() return } // If we are not scanning tests then drop all files that match common test file patterns // If we do not want to scan any test files or paths we check for them and then exclude them if they are found // The default is to not scan test files or common test paths - if !sess.ScanTests { + if !sess.Config.ScanTests { likelyTestFile = util.IsTestFileOrPath(filename) } if likelyTestFile { // We want to know how many files have been ignored - sess.Stats.IncrementFilesIgnored() + sess.State.Stats.IncrementFilesIgnored() sess.Out.Debug("%s is a test file and being ignored", filename) return } // Check the file size of the file. If it is greater than the default size // then we increment the ignored file count and pass on through. - val, msg := util.IsMaxFileSize(filename, sess.MaxFileSize) + val, msg := util.IsMaxFileSize(filename, sess.Config.MaxFileSize) if val { - sess.Stats.IncrementFilesIgnored() + sess.State.Stats.IncrementFilesIgnored() sess.Out.Debug("%s %s", filename, msg) return } - if sess.Debug { + if sess.Config.Debug { // Print the filename of every file being scanned sess.Out.Debug("Analyzing %s", filename) } // Increment the number of files scanned - sess.Stats.IncrementFilesScanned() + sess.State.Stats.IncrementFilesScanned() var content string // Scan the file for know signatures for _, signature := range core.Signatures { @@ -73,7 +73,7 @@ func doFileScan(filename string, sess *core.Session) { content = cleanK[1] // destroy the secret if the flag is set - if sess.HideSecrets { + if sess.Config.HideSecrets { content = "" } @@ -112,7 +112,7 @@ func scanDir(path string, sess *core.Session) { defer cancel() // get an slice of of all paths - files, err := search(ctx, path, sess.SkippablePath, sess) + files, err := search(ctx, path, sess.Config.SkippablePath, sess) if err != nil { sess.Out.Error("There is an error scanning %s: %s", path, err.Error()) } diff --git a/internal/pkg/localpath/localpath.go b/internal/pkg/localpath/localpath.go index ee31c00..5765333 100644 --- a/internal/pkg/localpath/localpath.go +++ b/internal/pkg/localpath/localpath.go @@ -2,13 +2,11 @@ package localpath import ( "fmt" - "time" "github.com/rumenvasilev/rvsecret/internal/core" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" "github.com/rumenvasilev/rvsecret/internal/util" - "github.com/rumenvasilev/rvsecret/version" ) func Scan(log *log.Logger) error { @@ -19,24 +17,17 @@ func Scan(log *log.Logger) error { // re-assign logger sess.Out = log // exclude the .git directory from local scans as it is not handled properly here - sess.SkippablePath = util.AppendIfMissing(sess.SkippablePath, ".git/") + sess.Config.SkippablePath = util.AppendIfMissing(sess.Config.SkippablePath, ".git/") - if sess.Debug { + if sess.Config.Debug { core.PrintDebug(sess) } // By default we display a header to the user giving basic info about application. This will not be displayed // during a silent run which is the default when using this in an automated fashion. - if !sess.JSONOutput && !sess.CSVOutput { - log.Warn("%s", core.ASCIIBanner) - log.Important("%s v%s started at %s", version.Name, sess.AppVersion, sess.Stats.StartedAt.Format(time.RFC3339)) - log.Important("Loaded %d signatures.", len(core.Signatures)) - if sess.WebServer { - log.Important("Web interface available at http://%s:%d/public", sess.BindAddress, sess.BindPort) - } - } + core.HeaderInfo(*sess.Config, sess.State.Stats, sess.Out) - for _, p := range sess.LocalPaths { + for _, p := range sess.Config.LocalPaths { if util.PathExists(p, log) { last := p[len(p)-1:] if last == "/" { @@ -50,9 +41,9 @@ func Scan(log *log.Logger) error { sess.Finish() core.SummaryOutput(sess) - fmt.Println("Webserver: ", sess.WebServer) + fmt.Println("Webserver: ", sess.Config.WebServer) - if !sess.Silent && sess.WebServer { + if !sess.Config.Silent && sess.Config.WebServer { log.Important("Press Ctrl+C to stop web server and exit.") select {} } diff --git a/internal/core/stats.go b/internal/stats/stats.go similarity index 52% rename from internal/core/stats.go rename to internal/stats/stats.go index 31c9f54..1f1eefd 100644 --- a/internal/core/stats.go +++ b/internal/stats/stats.go @@ -1,15 +1,10 @@ -package core +package stats import ( - "encoding/csv" - "encoding/json" - "fmt" - "os" - "sort" "sync" "time" - "github.com/rumenvasilev/rvsecret/internal/log" + _coreapi "github.com/rumenvasilev/rvsecret/internal/core/api" ) // Stats hold various runtime statistics used for perf data as well generating various reports @@ -17,19 +12,19 @@ import ( type Stats struct { // TODO alpha sort this sync.Mutex - StartedAt time.Time // The time we started the scan - FinishedAt time.Time // The time we finished the scan - Status Status // The running status of a scan for the web interface - Progress float64 // The running progress for the bar on the web interface - RepositoriesTotal int // The toatal number of repos discovered - RepositoriesScanned int // The total number of repos scanned (not excluded, errors, empty) - RepositoriesCloned int // The total number of repos cloned (excludes errors and excluded, includes empty) - Organizations int // The number of github orgs - CommitsScanned int // The number of commits scanned in a repo - CommitsDirty int // The number of commits in a repo found to have secrets - FilesScanned int // The number of files actually scanned - FilesIgnored int // The number of files ignored (tests, extensions, paths) - FilesTotal int // The total number of files that were processed + StartedAt time.Time // The time we started the scan + FinishedAt time.Time // The time we finished the scan + Status _coreapi.Status // The running status of a scan for the web interface + Progress float64 // The running progress for the bar on the web interface + RepositoriesTotal int // The toatal number of repos discovered + RepositoriesScanned int // The total number of repos scanned (not excluded, errors, empty) + RepositoriesCloned int // The total number of repos cloned (excludes errors and excluded, includes empty) + Organizations int // The number of github orgs + CommitsScanned int // The number of commits scanned in a repo + CommitsDirty int // The number of commits in a repo found to have secrets + FilesScanned int // The number of files actually scanned + FilesIgnored int // The number of files ignored (tests, extensions, paths) + FilesTotal int // The total number of files that were processed FilesDirty int FindingsTotal int // The total number of findings. There can be more than one finding per file and more than one finding of the same type in a file Users int // Github users @@ -41,7 +36,26 @@ type Stats struct { // TODO alpha sort this Commits int // This will point to CommitsScanned } -func (s *Stats) UpdateStatus(to Status) { +// InitStats will set the initial values for a session +func Init() *Stats { + return &Stats{ + FilesIgnored: 0, + FilesScanned: 0, + FindingsTotal: 0, + Organizations: 0, + Progress: 0.0, + StartedAt: time.Now(), + Status: _coreapi.StatusFinished, + Users: 0, + Targets: 0, + Repositories: 0, + CommitsTotal: 0, + Findings: 0, + Files: 0, + } +} + +func (s *Stats) UpdateStatus(to _coreapi.Status) { s.Status = to } @@ -191,152 +205,3 @@ func (s *Stats) IncrementCommitsDirty() { defer s.Unlock() s.CommitsDirty++ } - -// InitStats will set the initial values for a session -func (s *Session) InitStats() { - if s.Stats != nil { - return - } - s.Stats = &Stats{ - FilesIgnored: 0, - FilesScanned: 0, - FindingsTotal: 0, - Organizations: 0, - Progress: 0.0, - StartedAt: time.Now(), - Status: StatusFinished, - Users: 0, - Targets: 0, - Repositories: 0, - CommitsTotal: 0, - Findings: 0, - Files: 0, - } -} - -// PrintDebug will print a debug header at the start of the session that displays specific setting -func PrintDebug(sess *Session) { - maxFileSize := sess.MaxFileSize * 1024 * 1024 - sess.Out.Debug("\n") - sess.Out.Debug("Debug Info") - sess.Out.Debug("App version..............%v", sess.AppVersion) - sess.Out.Debug("Signatures version.......%v", sess.SignatureVersion) - sess.Out.Debug("Scanning tests...........%v", sess.ScanTests) - sess.Out.Debug("Max file size............%d", maxFileSize) - sess.Out.Debug("JSON output..............%v", sess.JSONOutput) - sess.Out.Debug("CSV output...............%v", sess.CSVOutput) - sess.Out.Debug("Silent output............%v", sess.Silent) - sess.Out.Debug("Web server enabled.......%v", sess.WebServer) - // log.Debug("") -} - -// PrintSessionStats will print the performance and sessions stats to stdout at the conclusion of a session scan -func printSessionStats(s *Stats, log *log.Logger, appVersion, signatureVersion string) { - log.Important("\n--------Results--------\n") - log.Important("\n") - log.Important("-------Findings------") - log.Info("Total Findings......: %d", s.Findings) - log.Important("\n") - log.Important("--------Files--------") - log.Info("Total Files.........: %d", s.FilesTotal) - log.Info("Files Scanned.......: %d", s.FilesScanned) - log.Info("Files Ignored.......: %d", s.FilesIgnored) - log.Info("Files Dirty.........: %d", s.FilesDirty) - log.Important("\n") - log.Important("---------SCM---------") - log.Info("Repos Found.........: %d", s.RepositoriesTotal) - log.Info("Repos Cloned........: %d", s.RepositoriesCloned) - log.Info("Repos Scanned.......: %d", s.RepositoriesScanned) - log.Info("Commits Total.......: %d", s.CommitsTotal) - log.Info("Commits Scanned.....: %d", s.CommitsScanned) - log.Info("Commits Dirty.......: %d", s.CommitsDirty) - log.Important("\n") - log.Important("-------General-------") - log.Info("App Version.........: %s", appVersion) - log.Info("Signatures Version..: %s", signatureVersion) - log.Info("Elapsed Time........: %s", time.Since(s.StartedAt)) - log.Info("") -} - -// SummaryOutput will spit out the results of the hunt along with performance data -func SummaryOutput(sess *Session) { - - // alpha sort the findings to make the results idempotent - if len(sess.Findings) > 0 { - sort.Slice(sess.Findings, func(i, j int) bool { - return sess.Findings[i].SecretID < sess.Findings[j].SecretID - }) - } - - if sess.JSONOutput { - if len(sess.Findings) > 0 { - b, err := json.MarshalIndent(sess.Findings, "", " ") - if err != nil { - fmt.Println(err) - return - } - c := string(b) - if c == "null" { - fmt.Println("[]") - } else { - fmt.Println(c) - } - } else { - fmt.Println("[]") - } - } - - if sess.CSVOutput { - w := csv.NewWriter(os.Stdout) - defer w.Flush() - header := []string{ - "FilePath", - "Line Number", - "Action", - "Description", - "SignatureID", - "Finding List", - "Repo Owner", - "Repo Name", - "Commit Hash", - "Commit Message", - "Commit Author", - "File URL", - "Secret ID", - "App Version", - "Signatures Version", - } - err := w.Write(header) - if err != nil { - sess.Out.Error(err.Error()) - } - - for _, v := range sess.Findings { - line := []string{ - v.FilePath, - v.LineNumber, - v.Action, - v.Description, - v.SignatureID, - v.Content, - v.RepositoryOwner, - v.RepositoryName, - v.CommitHash, - v.CommitMessage, - v.CommitAuthor, - v.FileURL, - v.SecretID, - v.AppVersion, - v.SignatureVersion, - } - err := w.Write(line) - if err != nil { - sess.Out.Error(err.Error()) - } - } - } - - if !sess.JSONOutput && !sess.CSVOutput { - printSessionStats(sess.Stats, sess.Out, sess.AppVersion, sess.SignatureVersion) - } -} diff --git a/internal/util/io.go b/internal/util/io.go index c214e71..900166f 100644 --- a/internal/util/io.go +++ b/internal/util/io.go @@ -44,16 +44,6 @@ func FileExists(path string) bool { return true } -// AppendIfMissing will check a slice for a value before appending it -func AppendIfMissing(slice []string, s string) []string { - for _, ele := range slice { - if ele == s { - return slice - } - } - return append(slice, s) -} - // SetHomeDir will set the correct homedir. func SetHomeDir(h string) (string, error) { if strings.Contains(h, "$HOME") { diff --git a/internal/util/strings.go b/internal/util/strings.go index aef1178..eaed48c 100644 --- a/internal/util/strings.go +++ b/internal/util/strings.go @@ -91,3 +91,26 @@ func PointerToInt64(p *int64) int64 { } return *p } + +// AppendIfMissing will check a slice for a value before appending it +func AppendIfMissing(slice []string, s string) []string { + for _, ele := range slice { + if ele == s { + return slice + } + } + return append(slice, s) +} + +// AppendToSlice will append additional items to slice if not present already and return a new slice +// additional trim support is present, if necessary +func AppendToSlice(trim bool, input, target []string) []string { + var result []string + for _, v := range input { + if trim { + v = strings.TrimSpace(v) + } + result = AppendIfMissing(target, v) + } + return result +} diff --git a/internal/core/router.go b/internal/webserver/router.go similarity index 78% rename from internal/core/router.go rename to internal/webserver/router.go index 3f96a91..cad9843 100644 --- a/internal/core/router.go +++ b/internal/webserver/router.go @@ -1,16 +1,17 @@ -package core +package webserver import ( "fmt" "io" "io/fs" - "log" "net/http" "github.com/gin-contrib/logger" "github.com/gin-contrib/secure" "github.com/gin-gonic/gin" "github.com/rumenvasilev/rvsecret/assets" + "github.com/rumenvasilev/rvsecret/internal/core" + "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" "github.com/rumenvasilev/rvsecret/internal/util" ) @@ -27,20 +28,38 @@ const ( // Is this a github repo/org var isGithub bool -// NewRouter will create an instance of the web frontend, setting the necessary parameters. -func NewRouter(s *Session) *gin.Engine { - if s.ScanType == api.Github { +// Start will configure and start the webserver for graphical output and status messages +// It's a blocking call, so it should be run in a goroutine +func Start(engine *gin.Engine, address string, port int, log *log.Logger) { + bind := fmt.Sprintf("%s:%d", address, port) + if err := engine.Run(bind); err != nil { + log.Fatal("Error when starting web server: %s", err) + } +} + +type Configuration struct { + ScanType api.ScanType + Debug bool + Logger *log.Logger + // session + State *core.State +} + +// New will create an instance of the web frontend, setting the necessary parameters. +func New(input Configuration) *gin.Engine { + log := input.Logger + if input.ScanType == api.Github { isGithub = true } gin.SetMode(gin.ReleaseMode) - if s.Debug { + if input.Debug { gin.SetMode(gin.DebugMode) } serverRoot, err := fs.Sub(&assets.Assets, "static") if err != nil { - log.Fatal(err) + log.Fatal(err.Error()) } router := gin.New() @@ -63,16 +82,16 @@ func NewRouter(s *Session) *gin.Engine { ReferrerPolicy: ReferrerPolicy, })) router.GET("/api/stats", func(c *gin.Context) { - c.JSON(200, s.Stats) + c.JSON(200, input.State.Stats) }) router.GET("/api/findings", func(c *gin.Context) { - c.JSON(200, s.Findings) + c.JSON(200, input.State.Findings) }) router.GET("/api/targets", func(c *gin.Context) { - c.JSON(200, s.Targets) + c.JSON(200, input.State.Targets) }) router.GET("/api/repositories", func(c *gin.Context) { - c.JSON(200, s.Repositories) + c.JSON(200, input.State.Repositories) }) router.GET("/api/files/:owner/:repo/:commit/*path", fetchFile)