fixing all golint and adding some more tests

Makefile

@@ -11,8 +11,8 @@ test-cover:
 
 test:
 	go get golang.org/x/lint/golint
-	go fmt
-	golint
+	go fmt ./...
+	golint ./...
 	go test ./... --race $(PKG) -v
 
 test-integration:

audit/audit.go

@@ -7,6 +7,7 @@ import (
 	"path"
 )
 
+// Run accepts a manager and begins an audit based on the options/configs set in the manager.
 func Run(m *manager.Manager) error {
 	if m.Opts.OwnerPath != "" {
 		files, err := ioutil.ReadDir(m.Opts.OwnerPath)
@@ -17,7 +18,7 @@ func Run(m *manager.Manager) error {
 			if !f.IsDir() {
 				continue
 			}
-			m.Opts.RepoPath = fmt.Sprintf("%s/%s",m.Opts.OwnerPath, f.Name())
+			m.Opts.RepoPath = fmt.Sprintf("%s/%s", m.Opts.OwnerPath, f.Name())
 			if err := runHelper(NewRepo(m)); err != nil {
 				// TODO or send to errchan?
 				return err
@@ -46,10 +47,9 @@ func runHelper(r *Repo) error {
 			return nil
 		}
 	} else {
-		if err := r.Clone(); err != nil {
+		if err := r.Clone(nil); err != nil {
 			return err
 		}
 	}
 	return r.Audit()
 }
-

audit/audit_test.go

@@ -31,7 +31,6 @@ func TestAudit(t *testing.T) {
 			opts: options.Options{
 				RepoPath: "../test_data/test_repos/test_repo_1",
 				Report:   "../test_data/test_local_repo_one_aws_leak.json.got",
-
 			},
 			wantPath: "../test_data/test_local_repo_one_aws_leak.json",
 		},
@@ -123,8 +122,7 @@ func TestAudit(t *testing.T) {
 			description: "test owner path",
 			opts: options.Options{
 				OwnerPath: "../test_data/test_repos/",
-				Report:   "../test_data/test_local_owner_aws_leak.json.got",
-
+				Report:    "../test_data/test_local_owner_aws_leak.json.got",
 			},
 			wantPath: "../test_data/test_local_owner_aws_leak.json",
 		},
@@ -137,13 +135,21 @@ func TestAudit(t *testing.T) {
 			},
 			wantPath: "../test_data/test_entropy.json",
 		},
+		{
+			description: "test entropy and regex",
+			opts: options.Options{
+				RepoPath: "../test_data/test_repos/test_repo_1",
+				Report:   "../test_data/test_regex_entropy.json.got",
+				Config:   "../test_data/test_configs/regex_entropy.toml",
+			},
+			wantPath: "../test_data/test_regex_entropy.json",
+		},
 		{
 			description: "test local repo four entropy alternative config",
 			opts: options.Options{
-				RepoPath: "../test_data/test_repos/test_repo_4",
-				Report:   "../test_data/test_local_repo_four_alt_config_entropy.json.got",
+				RepoPath:   "../test_data/test_repos/test_repo_4",
+				Report:     "../test_data/test_local_repo_four_alt_config_entropy.json.got",
 				RepoConfig: true,
-
 			},
 			wantPath: "../test_data/test_local_repo_four_alt_config_entropy.json.got",
 		},
@@ -277,7 +283,6 @@ func TestAuditUncommited(t *testing.T) {
 			}
 		}
 	}
-
 }
 
 func fileCheck(wantPath, gotPath string) error {
@@ -294,11 +299,10 @@ func fileCheck(wantPath, gotPath string) error {
 	if strings.Trim(string(want), "\n") != strings.Trim(string(got), "\n") {
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(string(want), string(got), false)
-		return fmt.Errorf("does not equal: %s\n", dmp.DiffPrettyText(diffs))
-	} else {
-		if err := os.Remove(gotPath); err != nil {
-			return err
-		}
+		return fmt.Errorf("does not equal: %s", dmp.DiffPrettyText(diffs))
+	}
+	if err := os.Remove(gotPath); err != nil {
+		return err
 	}
 	return nil
 }
@@ -320,4 +324,3 @@ func moveDotGit(from, to string) error {
 	}
 	return nil
 }
-

audit/repo.go

@@ -27,13 +27,12 @@ import (
 type Repo struct {
 	*git.Repository
 
-	// AlternativeConfig is used when the --repo-config option is set.
+	// config is used when the --repo-config option is set.
 	// This allows users to load up configs specific to their repos.
 	// Imagine the scenario where you are doing an audit of a large organization
 	// and you want certain repos to look for specific rules. If those specific repos
 	// have a gitleaks.toml or .gitleaks.toml config then those configs will be used specifically
 	// for those repo audits.
-	AlternativeConfig config.Config
 	config config.Config
 
 	Name    string
@@ -44,21 +43,18 @@ type Repo struct {
 func NewRepo(m *manager.Manager) *Repo {
 	return &Repo{
 		Manager: m,
-		config: m.Config,
+		config:  m.Config,
 	}
 }
 
 // Clone will clone a repo and return a Repo struct which contains a go-git repo. The clone method
 // is determined by the clone options set in Manager.metadata.cloneOptions
-func (repo *Repo) Clone(cloneOptions ...*git.CloneOptions) error {
+func (repo *Repo) Clone(cloneOption *git.CloneOptions) error {
 	var (
-		repository  *git.Repository
-		err         error
-		cloneOption *git.CloneOptions
+		repository *git.Repository
+		err        error
 	)
-	if len(cloneOptions) != 0 {
-		cloneOption = cloneOptions[0]
-	} else {
+	if cloneOption == nil {
 		cloneOption = repo.Manager.CloneOptions
 	}
 
@@ -228,7 +224,6 @@ func (repo *Repo) Audit() error {
 			return nil
 		}
 
-		// TODO check whitelist Commit
 		if isCommitWhiteListed(c.Hash.String(), repo.config.Whitelist.Commits) {
 			return nil
 		}
@@ -304,4 +299,3 @@ func (repo *Repo) loadRepoConfig() (config.Config, error) {
 	_, err = toml.DecodeReader(f, &tomlLoader)
 	return tomlLoader.Parse()
 }
-

audit/util.go

@@ -99,18 +99,28 @@ func trippedEntropy(line string, rule config.Rule) bool {
 	return false
 }
 
+func ruleContainRegex(rule config.Rule) bool {
+	if rule.Regex == nil {
+		return false
+	}
+	if rule.Regex.String() == "" {
+		return false
+	}
+	return true
+}
+
 // InspectString accepts a string, commit object, repo, and filename. This function iterates over
 // all the rules set by the gitleaks config. If the rule contains entropy checks then entropy will be checked first.
 // Next, if the rule contains a regular expression then that will be checked.
 func InspectString(content string, c *object.Commit, repo *Repo, filename string) {
 	for _, rule := range repo.config.Rules {
 		// check entropy
 		if len(rule.Entropy) != 0 {
-			// TODO
 			// an optimization would be to switch the regex from FindAllIndex to FindString
 			// since we are iterating on the lines if entropy rules exist...
 			for _, line := range strings.Split(content, "\n") {
-				if trippedEntropy(line, rule) {
+				entropyTripped := trippedEntropy(line, rule)
+				if entropyTripped && !ruleContainRegex(rule) {
 					_line := line
 					if len(_line) > maxLineLen {
 						_line = line[0 : maxLineLen-1]
@@ -128,8 +138,32 @@ func InspectString(content string, c *object.Commit, repo *Repo, filename string
 						Tags:     strings.Join(rule.Tags, ", "),
 						File:     filename,
 					})
+				} else if entropyTripped {
+					// entropy has been tripped which means if there is a regex specified in the same
+					// rule, we need to inspect the line for a regex match. In otherwords, the current rule has
+					// both entropy and regex set which work in combination. This helps narrow down false positives
+					// on searches for generic passwords in code.
+					match := rule.Regex.FindString(line)
+					if match != "" {
+						// both the regex and entropy in this rule have been tripped which means this line
+						// contains a leak
+						repo.Manager.SendLeaks(manager.Leak{
+							Line:     line,
+							Offender: match,
+							Commit:   c.Hash.String(),
+							Message:  c.Message,
+							Repo:     repo.Name,
+							Rule:     rule.Description,
+							Author:   c.Author.Name,
+							Email:    c.Author.Email,
+							Date:     c.Author.When,
+							Tags:     strings.Join(rule.Tags, ", "),
+							File:     filename,
+						})
+					}
 				}
 			}
+			return
 		}
 		if rule.Regex.String() == "" {
 			continue

config/config.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 )
 
-// WhiteList is struct containing items that if encountered will whitelist
+// Whitelist is struct containing items that if encountered will whitelist
 // a commit/line of code that would be considered a leak.
 type Whitelist struct {
 	Description string
@@ -121,9 +121,6 @@ func (tomlLoader TomlLoader) Parse() (Config, error) {
 			if err != nil {
 				return cfg, fmt.Errorf("problem loading config: %v", err)
 			}
-			if err != nil {
-				return cfg, fmt.Errorf("problem loading config: %v", err)
-			}
 			whitelists = append(whitelists, Whitelist{
 				Description: wl.Description,
 				File:        fileRe,

config/config_test.go

@@ -18,7 +18,7 @@ func TestParse(t *testing.T) {
 	}{
 		{
 			description: "default config",
-			opts: options.Options{},
+			opts:        options.Options{},
 		},
 		{
 			description: "test successful load",

config/default.go

@@ -1,5 +1,8 @@
 package config
 
+// DefaultConfig is the default gitleaks configuration. If --config={path-to-config} is set than the config located
+// at {path-to-config} will be used. Alternatively, if --repo-config is set then gitleaks will attempt to
+// use the config set in a gitleaks.toml or .gitleaks.toml file in the repo that is run with --repo-config set.
 const DefaultConfig = `
 title = "gitleaks config"
 
@@ -133,4 +136,3 @@ title = "gitleaks config"
 	description = "image whitelists"
 	file = '''(.*?)(jpg|gif|doc|pdf|bin)$'''
 `
-

go.sum

@@ -70,7 +70,6 @@ github.com/xanzy/go-gitlab v0.21.0 h1:Ru55sR4TBoDNsAKwCOpzeaGtbiWj7xTksVmzBJbLu6
 github.com/xanzy/go-gitlab v0.21.0/go.mod h1:t4Bmvnxj7k37S4Y17lfLx+nLqkf/oQwT2HagfWKv5Og=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
-github.com/zricethezav/gitleaks-ng v0.0.1 h1:4WgulZbJ5QanI6f5kbhVZjuMhUeyEE+R8y3YvHuf9rA=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=

hosts/github.go

@@ -2,7 +2,6 @@ package hosts
 
 import (
 	"context"
-	"fmt"
 	"github.com/google/go-github/github"
 	log "github.com/sirupsen/logrus"
 	"github.com/zricethezav/gitleaks/audit"
@@ -17,24 +16,15 @@ import (
 	"sync"
 )
 
-type GithubError struct {
-	Err    string
-	Repo   string
-	Commit string
-}
-
-func (githubError *GithubError) Error() string {
-	return fmt.Sprintf("repo: %s, err: %s",
-		githubError.Repo, githubError.Err)
-}
-
+// Github wraps a github client and manager. This struct implements what the Host interface defines.
 type Github struct {
 	client  *github.Client
-	errChan chan GithubError
 	manager manager.Manager
 	wg      sync.WaitGroup
 }
 
+// NewGithubClient accepts a manager struct and returns a Github host pointer which will be used to
+// perform a github audit on an organization, user, or PR.
 func NewGithubClient(m manager.Manager) *Github {
 	ctx := context.Background()
 	token := oauth2.StaticTokenSource(
@@ -44,7 +34,6 @@ func NewGithubClient(m manager.Manager) *Github {
 	return &Github{
 		manager: m,
 		client:  github.NewClient(oauth2.NewClient(ctx, token)),
-		errChan: make(chan GithubError),
 	}
 }
 
@@ -53,6 +42,7 @@ func (g *Github) Audit() {
 	ctx := context.Background()
 	listOptions := github.ListOptions{
 		PerPage: 100,
+		Page:    1,
 	}
 
 	var githubRepos []*github.Repository
@@ -70,12 +60,18 @@ func (g *Github) Audit() {
 			_githubRepos, resp, err = g.client.Repositories.ListByOrg(ctx, g.manager.Opts.Organization,
 				&github.RepositoryListByOrgOptions{ListOptions: listOptions})
 		}
-
 		githubRepos = append(githubRepos, _githubRepos...)
 
 		if resp == nil {
 			break
 		}
+
+		if resp.LastPage != 0 {
+			log.Infof("gathering github repos... progress: page %d of %d", listOptions.Page, resp.LastPage)
+		} else {
+			log.Infof("gathering github repos... progress: page %d of %d", listOptions.Page, listOptions.Page)
+		}
+
 		listOptions.Page = resp.NextPage
 		if err != nil || listOptions.Page == 0 {
 			break
@@ -87,11 +83,20 @@ func (g *Github) Audit() {
 		err := r.Clone(&git.CloneOptions{
 			URL: *repo.CloneURL,
 		})
-		r.Name = *repo.Name
 		if err != nil {
-			log.Warn(err)
+			log.Warn("unable to clone via https and access token, attempting with ssh now")
+			auth, err := options.SSHAuth(g.manager.Opts)
+			if err != nil {
+				log.Warnf("unable to get ssh auth, skipping clone and audit for repo %s: %+v\n", *repo.CloneURL, err)
+			}
+			err = r.Clone(&git.CloneOptions{
+				URL:  *repo.SSHURL,
+				Auth: auth,
+			})
+			if err != nil {
+				log.Warnf("err cloning %s, skipping clone and audit: %+v\n", *repo.SSHURL, err)
+			}
 		}
-
 		if err = r.Audit(); err != nil {
 			log.Warn(err)
 		}