make log global; rename Scanner.Do to Scanner.Run

Signed-off-by: Rumen Vasilev <[email protected]>

TODO.md

@@ -1,26 +1,16 @@
 # TODO
 ## Refactor
-- x Remove dependency on bindata - github.com/elazarl/go-bindata-assetfs
-- x Change banner
-- x Remove search for all github orgs if none is provided
 - Integrate with GHA for build/test
-  - x build,test pipeline
-  - x unit test, coverage (https://codecov.io/bash?), race
   - add release pipeline
   - Dynamic Changelog (in releases?)
 - Increase test coverage
 - Implement integration tests
-- x Refactor core package - remove multi-level nesting
 - Remove dependencies on Hashicorp as much as possible (viper?)
-- x Create config package
 - Remove global cfg var, explicitly pass it as arg
-- x os.Exit() everywhere, raise error instead
-- x replace pkg.Scan with interface for easier testing
 - Implement proper server wait, not select{}
 - Store md5 in session, avoiding duplicate calc
 - New Makefile
-- x Overhaul updateRules.go
-- x Start webserver everywhere
+- Propagate context
 
 ## Features
 - Add exit codes, so CI could detect if scan failed

cmd/init-config.go

@@ -1,18 +1,15 @@
 package cmd
 
 import (
-	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 var initConfigCmd = &cobra.Command{
 	Use:   "init-config",
 	Short: "Creates configuration file",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		log := log.NewLogger(viper.GetBool("debug"), viper.GetBool("silent"))
-		return pkg.SaveConfig(log)
+		return pkg.SaveConfig()
 	},
 }
 

cmd/scan/scan-github.go

@@ -4,7 +4,6 @@ package scan
 
 import (
 	"github.com/rumenvasilev/rvsecret/internal/config"
-	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/spf13/cobra"
@@ -25,8 +24,7 @@ var scanGithubCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
-		return scan.New(cfg, log).Do()
+		return scan.New(cfg).Run()
 	},
 }
 

cmd/scan/scan-gitlab.go

@@ -4,7 +4,6 @@ package scan
 
 import (
 	"github.com/rumenvasilev/rvsecret/internal/config"
-	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/spf13/cobra"
@@ -21,8 +20,7 @@ var scanGitlabCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
-		return scan.New(cfg, log).Do()
+		return scan.New(cfg).Run()
 	},
 }
 

cmd/scan/scan-local-git-repo.go

@@ -4,7 +4,6 @@ package scan
 
 import (
 	"github.com/rumenvasilev/rvsecret/internal/config"
-	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/spf13/cobra"
@@ -20,8 +19,7 @@ var scanLocalGitRepoCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
-		return scan.New(cfg, log).Do()
+		return scan.New(cfg).Run()
 	},
 }
 

cmd/scan/scan-localpath.go

@@ -4,7 +4,6 @@ package scan
 
 import (
 	"github.com/rumenvasilev/rvsecret/internal/config"
-	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/spf13/cobra"
@@ -21,8 +20,7 @@ var scanLocalPathCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
-		return scan.New(cfg, log).Do()
+		return scan.New(cfg).Run()
 	},
 }
 

cmd/updateRules.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"github.com/rumenvasilev/rvsecret/internal/config"
-	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/signatures"
 
@@ -25,8 +24,7 @@ var updateSignaturesCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
-		return signatures.Update(cfg, log)
+		return signatures.Update(cfg)
 	},
 }
 

internal/config/config.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/fatih/structs"
 	"github.com/mitchellh/go-homedir"
+	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/rumenvasilev/rvsecret/internal/util"
 	"github.com/rumenvasilev/rvsecret/version"
@@ -194,6 +195,8 @@ func SetConfig(cmd *cobra.Command) {
 // Load depends on cfg being initialized and populated, otherwise it will panic
 func Load(scanType api.ScanType) (*Config, error) {
 	// set configuration
+	log.SetDebug(cfg.Global.Debug)
+	log.SetSilent(cfg.Global.Silent)
 	cfg.Global.ScanType = scanType
 
 	switch scanType {

internal/core/analysis.go

@@ -25,7 +25,8 @@ import (
 //	are controlled by flags. If a directory, file, or the content pass through all of the filters then
 //	it is scanned once per each signature which may lead to a specific secret matching multiple rules
 //	and then generating multiple findings.
-func AnalyzeRepositories(sess *Session, stats *stats.Stats, log *log.Logger) {
+func AnalyzeRepositories(sess *Session, stats *stats.Stats) {
+	log := log.Log
 	stats.UpdateStatus(_coreapi.StatusAnalyzing)
 	repoCnt := len(sess.State.Repositories)
 	if repoCnt == 0 {
@@ -52,7 +53,7 @@ func AnalyzeRepositories(sess *Session, stats *stats.Stats, log *log.Logger) {
 
 	// Start analyzer workers
 	for i := 0; i < threadNum; i++ {
-		go analyzeWorker(i, ch, &wg, sess, stats, log)
+		go analyzeWorker(i, ch, &wg, sess, stats)
 	}
 
 	// Feed repos to the analyzer workers
@@ -66,7 +67,8 @@ func AnalyzeRepositories(sess *Session, stats *stats.Stats, log *log.Logger) {
 	wg.Wait()
 }
 
-func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, sess *Session, stats *stats.Stats, log *log.Logger) {
+func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, sess *Session, stats *stats.Stats) {
+	log := log.Log
 	for {
 		log.Debug("[THREAD #%d] Requesting new repository to analyze...", tid)
 		repo, ok := <-ch
@@ -83,7 +85,7 @@ func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, ses
 		clone, path, err := cloneRepository(sess.Config, stats.IncrementRepositoriesCloned, repo)
 		if err != nil {
 			log.Error("%v", err)
-			cleanUpPath(path, log)
+			cleanUpPath(path)
 			continue
 		}
 		log.Debug("[THREAD #%d][%s] Cloned repository to: %s", tid, repo.CloneURL, path)
@@ -93,21 +95,21 @@ func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, ses
 		log.Debug("[THREAD #%d][%s] Done analyzing commits", tid, repo.CloneURL)
 		log.Debug("[THREAD #%d][%s] Deleted %s", tid, repo.CloneURL, path)
 
-		cleanUpPath(path, log)
+		cleanUpPath(path)
 		stats.IncrementRepositoriesScanned()
 	}
 }
 
-func cleanUpPath(path string, log *log.Logger) {
+func cleanUpPath(path string) {
 	err := os.RemoveAll(path)
 	if err != nil {
-		log.Error("Could not remove path from disk: %s", err.Error())
+		log.Log.Error("Could not remove path from disk: %s", err.Error())
 	}
 }
 
 func (sess *Session) analyzeHistory(clone *git.Repository, tid int, path string, repo _coreapi.Repository) {
 	stats := sess.State.Stats
-	log := sess.Out
+	log := log.Log
 
 	// Get the full commit history for the repo
 	history, err := _git.GetRepositoryHistory(clone)
@@ -147,7 +149,7 @@ func (sess *Session) analyzeHistory(clone *git.Repository, tid int, path string,
 // isDirtyCommit will analyze all the changes and return bool if there's a dirty commit
 func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Repository, clone *git.Repository, path string, tid int) bool {
 	stats := sess.State.Stats
-	log := sess.Out
+	log := log.Log
 
 	// This will be used to increment the dirty commit stat if any matches are found. A dirty commit
 	// means that a secret was found in that commit. This provides an easier way to manually to look
@@ -190,7 +192,7 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito
 		// dirtyFile := false
 
 		// call signaturesfunc
-		dirtyFile, dcommit, ignored, out := signatures.Discover(mf, change, sess.Config, sess.Signatures, log)
+		dirtyFile, dcommit, ignored, out := signatures.Discover(mf, change, sess.Config, sess.Signatures)
 		for _, v := range out {
 			fin := &finding.Finding{
 				Action:           changeAction,
@@ -214,7 +216,7 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito
 			log.Debug("[THREAD #%d][%s] Done analyzing changes in %s", tid, repo.CloneURL, commit.Hash)
 
 			// // Print realtime data to stdout
-			fin.RealtimeOutput(sess.Config.Global, log)
+			fin.RealtimeOutput(sess.Config.Global)
 		}
 
 		if dirtyFile {

internal/core/banner/banner.go

@@ -12,7 +12,8 @@ import (
 //go:embed banner.txt
 var ASCIIBanner string
 
-func HeaderInfo(cfg config.Global, startTime string, sigs int, log *log.Logger) {
+func HeaderInfo(cfg config.Global, startTime string, sigs int) {
+	log := log.Log
 	if !cfg.JSONOutput && !cfg.CSVOutput {
 		log.Warn("%s", ASCIIBanner)
 		log.Important("%s v%s started at %s", version.Name, cfg.AppVersion, startTime)

internal/core/finding/finding.go

@@ -68,7 +68,8 @@ func (f *Finding) setupUrls(scanType api.ScanType, gheURL string) {
 	}
 }
 
-func (f *Finding) RealtimeOutput(cfg config.Global, log *log.Logger) {
+func (f *Finding) RealtimeOutput(cfg config.Global) {
+	log := log.Log
 	if !cfg.Silent && !cfg.CSVOutput && !cfg.JSONOutput {
 		log.Warn(" %s", strings.ToUpper(f.Description))
 		log.Info("  SignatureID..........: %s", f.SignatureID)

internal/core/gh_worker.go

@@ -12,7 +12,8 @@ const retrievedRepo string = " Retrieved repository %s"
 
 // GetAllRepositoriesForOwner will find all repositories for an owner (user or org)
 // and populate the repositories state list (s.State.Repositories)
-func GetAllRepositoriesForOwner(ctx context.Context, login string, kind string, tid int, sess *Session, log *log.Logger) {
+func GetAllRepositoriesForOwner(ctx context.Context, login string, kind string, tid int, sess *Session) {
+	log := log.Log
 	// Retrieve all the repos in an org regardless of public/private
 	repos, err := sess.Client.GetRepositoriesFromOwner(ctx, _coreapi.Owner{
 		Login: util.StringToPointer(login),

internal/core/github.go

@@ -7,12 +7,13 @@ import (
 	"strings"
 
 	_coreapi "github.com/rumenvasilev/rvsecret/internal/core/api"
+	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/util"
 )
 
 // GatherUsers will generate a list of users from github.com that can then be filtered down to a specific target range
 func (s *Session) GatherUserOrOrg(targetList []string) {
-	log := s.Out
+	log := log.Log
 	log.Important("Gathering targets...")
 	ctx := context.Background()
 	for _, o := range targetList {
@@ -66,7 +67,7 @@ func (s *Session) ValidateUserInput() error {
 // GatherOrgsMembers will gather all orgs members
 // and position them in Targets
 func GatherOrgsMembers(sess *Session) {
-	log := sess.Out
+	log := log.Log
 	log.Important("Gathering users from orgs...")
 	ctx := context.Background()
 
@@ -87,6 +88,6 @@ func GatherOrgsMembers(sess *Session) {
 // GetAllRepositoriesForTargets will iterate all targets and assemble a repository list with sess.AddRepository()
 func (s *Session) GetAllRepositoriesForTargets(ctx context.Context) {
 	for _, t := range s.State.Targets {
-		GetAllRepositoriesForOwner(ctx, *t.Login, *t.Kind, 0, s, s.Out)
+		GetAllRepositoriesForOwner(ctx, *t.Login, *t.Kind, 0, s)
 	}
 }

internal/core/gitlab.go

@@ -5,14 +5,16 @@ import (
 	"sync"
 
 	_coreapi "github.com/rumenvasilev/rvsecret/internal/core/api"
+	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/util"
 )
 
 // GatherTargets will enumerate git targets adding them to a running target list. This will set the targets based
 // on the scan type set within the cmd package.
 func GatherTargets(sess *Session) {
+	log := log.Log
 	sess.State.Stats.UpdateStatus(_coreapi.StatusGathering)
-	sess.Out.Important("Gathering targets...")
+	log.Important("Gathering targets...")
 	ctx := context.Background()
 
 	// var targets []string
@@ -40,30 +42,30 @@ func GatherTargets(sess *Session) {
 		//if sess.ScanType == "github" || sess.ScanType == "github-enterprise" {
 		//	target, err := sess.GithubClient.GetUserOrganization(loginOption)
 		//	if err != nil || target == nil {
-		//		sess.Out.Error(" Error retrieving information on %s: %s\n", loginOption, err)
+		//		log.Error(" Error retrieving information on %s: %s\n", loginOption, err)
 		//		continue
 		//	}
 		//} else {
 		target, err := sess.Client.GetUserOrganization(ctx, loginOption)
 		if err != nil || target == nil {
-			sess.Out.Error(" Error retrieving information on %s: %s", loginOption, err)
+			log.Error(" Error retrieving information on %s: %s", loginOption, err)
 			continue
 		}
 
-		sess.Out.Debug("%s (ID: %d) type: %s", *target.Login, *target.ID, *target.Type)
+		log.Debug("%s (ID: %d) type: %s", *target.Login, *target.ID, *target.Type)
 		sess.State.AddTarget(target)
 		// If forking is false AND the target type is an Organization as set above in GetUserOrganization
 		if sess.Config.Global.ExpandOrgs && *target.Type == _coreapi.TargetTypeOrganization {
-			sess.Out.Debug("Gathering members of %s (ID: %d)...", *target.Login, *target.ID)
+			log.Debug("Gathering members of %s (ID: %d)...", *target.Login, *target.ID)
 			members, err := sess.Client.GetOrganizationMembers(ctx, *target)
 			if err != nil {
-				sess.Out.Error(" Error retrieving members of %s: %s", *target.Login, err)
+				log.Error(" Error retrieving members of %s: %s", *target.Login, err)
 				continue
 			}
 			// Add organization members gathered above to the target list
 			// TODO Do we want to spider this out at some point to enumerate all members of an org?
 			for _, member := range members {
-				sess.Out.Debug("Adding organization member %s (ID: %d) to targets", *member.Login, *member.ID)
+				log.Debug("Adding organization member %s (ID: %d) to targets", *member.Login, *member.ID)
 				sess.State.AddTarget(member)
 			}
 		}
@@ -74,7 +76,7 @@ func GatherTargets(sess *Session) {
 // This is done using threads, whose count is set via commandline flag. Care much be taken to avoid rate
 // limiting associated with suspected DOS attacks.
 func GatherGitlabRepositories(sess *Session) {
-	log := sess.Out
+	log := log.Log
 	ctx := context.Background()
 	var ch = make(chan *_coreapi.Owner, len(sess.State.Targets))
 	log.Debug("Number of targets: %d", len(sess.State.Targets))