more refactoring wrt sonarqube analisys

cmd/root.go

@@ -14,12 +14,17 @@ import (
 	"github.com/spf13/viper"
 )
 
+const (
+	signaturesFile string = "signatures-file"
+	signaturesPath string = "signatures-path"
+)
+
 var (
 	// rootCmd represents the base command when called without any subcommands
 	rootCmd = &cobra.Command{
 		Use:   "rvsecret",
 		Short: "A tool to scan for secrets in various digital hiding spots",
-		Long:  "A tool to scan for secrets in various digital hiding spots - v" + version.AppVersion(), // TODO write a better long description
+		Long:  "A tool to scan for secrets in various digital hiding spots - v" + version.AppVersion(),
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			config.SetConfig(cmd)
 		},
@@ -41,9 +46,9 @@ func init() {
 	viper.BindPFlag("global.debug", rootCmd.PersistentFlags().Lookup("debug")) //nolint:errcheck
 	rootCmd.PersistentFlags().String("config-file", config.DefaultConfig.Global.ConfigFile, "Config file location")
 	viper.BindPFlag("global.config-file", rootCmd.PersistentFlags().Lookup("config-file")) //nolint:errcheck
-	rootCmd.PersistentFlags().String("signatures-file", config.DefaultConfig.Signatures.File, "file(s) containing detection signatures.")
-	viper.BindPFlag("signatures.file", rootCmd.PersistentFlags().Lookup("signatures-file")) //nolint:errcheck
-	rootCmd.PersistentFlags().String("signatures-path", config.DefaultConfig.Signatures.Path, "path containing detection signatures.")
-	rootCmd.MarkFlagsMutuallyExclusive("signatures-file", "signatures-path")
-	viper.BindPFlag("signatures.path", rootCmd.PersistentFlags().Lookup("signatures-path")) //nolint:errcheck
+	rootCmd.PersistentFlags().String(signaturesFile, config.DefaultConfig.Signatures.File, "file(s) containing detection signatures.")
+	viper.BindPFlag("signatures.file", rootCmd.PersistentFlags().Lookup(signaturesFile)) //nolint:errcheck
+	rootCmd.PersistentFlags().String(signaturesPath, config.DefaultConfig.Signatures.Path, "path containing detection signatures.")
+	rootCmd.MarkFlagsMutuallyExclusive(signaturesFile, signaturesPath)
+	viper.BindPFlag("signatures.path", rootCmd.PersistentFlags().Lookup(signaturesPath)) //nolint:errcheck
 }

cmd/scan/scan-gitlab.go

@@ -29,7 +29,8 @@ var scanGitlabCmd = &cobra.Command{
 func init() {
 	ScanCmd.AddCommand(scanGitlabCmd)
 	// scanGitlabCmd.Flags().Bool("add-org-members", false, "Add members to targets when processing organizations")
-	scanGitlabCmd.Flags().StringP("gitlab-api-token", "t", "", "API token for access to gitlab, see doc for necessary scope")
-	scanGitlabCmd.Flags().StringSlice("gitlab-projects", config.DefaultConfig.Gitlab.GitlabTargets, "List of Gitlab projects or users to scan")
-	viper.BindPFlags(scanGitlabCmd.Flags()) //nolint:errcheck
+	scanGitlabCmd.Flags().StringP("api-token", "t", "", "API token for access to gitlab, see doc for necessary scope")
+	viper.BindPFlag("gitlab.api-token", scanGitlabCmd.Flags().Lookup("api-token")) //nolint:errcheck
+	scanGitlabCmd.Flags().StringSlice("projects", config.DefaultConfig.Gitlab.Targets, "List of Gitlab projects or users to scan")
+	viper.BindPFlag("gitlab.projects", scanGitlabCmd.Flags().Lookup("projects")) //nolint:errcheck
 }

cmd/updateRules.go

@@ -10,6 +10,11 @@ import (
 	"github.com/spf13/viper"
 )
 
+const (
+	userRepo string = "user-repo"
+	url      string = "url"
+)
+
 // updateSignaturesCmd represents the updateSignatures command
 var updateSignaturesCmd = &cobra.Command{
 	Use:   "updateSignatures",
@@ -29,11 +34,11 @@ func init() {
 	rootCmd.AddCommand(updateSignaturesCmd)
 	updateSignaturesCmd.Flags().StringP("api-token", "t", "", "API token for github access, see documentation for necessary scope")
 	viper.BindPFlag("signatures.api-token", updateSignaturesCmd.Flags().Lookup("api-token")) //nolint:errcheck
-	updateSignaturesCmd.Flags().String("user-repo", "", "user/repo where signatures can be found, example: rumenvasilev/rvsecret-signatures")
-	viper.BindPFlag("signatures.user-repo", updateSignaturesCmd.Flags().Lookup("user-repo")) //nolint:errcheck
-	updateSignaturesCmd.Flags().String("url", config.DefaultConfig.Signatures.URL, "url where the signatures can be found")
-	viper.BindPFlag("signatures.url", updateSignaturesCmd.Flags().Lookup("url")) //nolint:errcheck
-	updateSignaturesCmd.MarkFlagsMutuallyExclusive("user-repo", "url")
+	updateSignaturesCmd.Flags().String(userRepo, "", "user/repo where signatures can be found, example: rumenvasilev/rvsecret-signatures")
+	viper.BindPFlag("signatures.user-repo", updateSignaturesCmd.Flags().Lookup(userRepo)) //nolint:errcheck
+	updateSignaturesCmd.Flags().String(url, config.DefaultConfig.Signatures.URL, "url where the signatures can be found")
+	viper.BindPFlag("signatures.url", updateSignaturesCmd.Flags().Lookup(url)) //nolint:errcheck
+	updateSignaturesCmd.MarkFlagsMutuallyExclusive(userRepo, url)
 	updateSignaturesCmd.Flags().String("signatures-version", config.DefaultConfig.Signatures.Version, "specific version of the signatures to install (latest, v1.2.0)")
 	viper.BindPFlag("signatures.version", updateSignaturesCmd.Flags().Lookup("signatures-version")) //nolint:errcheck
 	updateSignaturesCmd.Flags().Bool("test", config.DefaultConfig.Signatures.Test, "run any tests associated with the signatures and display the output")

internal/config/config.go

@@ -31,7 +31,7 @@ type Config struct {
 	Local      Local      `mapstructure:"local" yaml:"local"`
 	Signatures Signatures `mapstructure:"signatures" yaml:"signatures"`
 
-	Gitlab `mapstructure:"gitlab" yaml:"gitlab"`
+	Gitlab Gitlab `mapstructure:"gitlab" yaml:"gitlab"`
 	Global Global `mapstructure:"global" yaml:"global"`
 }
 
@@ -83,9 +83,9 @@ type Github struct {
 }
 
 type Gitlab struct {
-	GitlabAccessToken string   `mapstructure:"gitlab-api-token"`
-	GitlabTargets     []string `mapstructure:"gitlab-targets"`
-	_                 [24]byte
+	APIToken string   `mapstructure:"api-token" yaml:"api-token"`
+	Targets  []string `mapstructure:"projects" yaml:"projects"`
+	_        [24]byte
 }
 
 type Local struct {
@@ -165,6 +165,10 @@ func Load(scanType api.ScanType) (*Config, error) {
 		if cfg.Signatures.APIToken == "" {
 			return nil, errors.New("APIToken for Github is not set")
 		}
+	case api.Gitlab:
+		if cfg.Gitlab.APIToken == "" {
+			return nil, errors.New("APIToken for Gitlab is not set")
+		}
 	}
 	return cfg, nil
 }

internal/config/default_config.go

@@ -17,7 +17,7 @@ var github = Github{
 }
 
 var gitlab = Gitlab{
-	GitlabTargets: nil,
+	Targets: nil,
 }
 
 var global = Global{

internal/core/analysis.go

@@ -168,7 +168,6 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito
 		fPath := _git.GetChangePath(change)
 
 		// TODO Add an example of this
-		// FIXME This is where I have tracked the in-mem-clone issue to
 		fullFilePath := path + "/" + fPath
 
 		// Break a file name up into its composite pieces including the extension and base name

internal/core/gh_worker.go

@@ -13,55 +13,65 @@ import (
 func ghWorker(sess *Session, tid int, wg *sync.WaitGroup, ch chan *github.Organization, log *log.Logger) {
 	ctx := context.Background()
 	for {
-		var repos []*_coreapi.Repository
-		var err error
 		org, ok := <-ch
 		if !ok {
 			wg.Done()
 			return
 		}
-		// Retrieve all the repos in an org regardless of public/private
-		repos, err = sess.Client.GetRepositoriesFromOwner(ctx, _coreapi.Owner{
-			Login: org.Login,
-			Kind:  util.StringToPointer(_coreapi.TargetTypeOrganization),
-		})
-		if err != nil {
-			// We might get partial result, which is why we only log the error as warning
-			// var ghErrResp *github.ErrorResponse
-			// if errors.As(err, &ghErrResp) {
-			// 	if ghErrResp.Response.StatusCode == 404 {
-			// 		log.Warn("[THREAD #%d]: %s", tid, ghErrResp.Error())
-			// 	}
-			// }
-			log.Debug("[THREAD #%d]: GetRepositoriesFromOwner: %s", tid, err.Error())
-		}
 
-		// In the case where all the repos are private
-		if len(repos) == 0 {
-			log.Debug("No repositories have been gathered for %s", *org.Login)
-			continue
-		}
+		processRequest(ctx, org, tid, sess, log)
+	}
+}
+
+func processRequest(ctx context.Context, org *github.Organization, tid int, sess *Session, log *log.Logger) {
+	// Retrieve all the repos in an org regardless of public/private
+	repos, err := sess.Client.GetRepositoriesFromOwner(ctx, _coreapi.Owner{
+		Login: org.Login,
+		Kind:  util.StringToPointer(_coreapi.TargetTypeOrganization),
+	})
+	if err != nil {
+		// We might get partial result, which is why we only log the error as warning
+		// var ghErrResp *github.ErrorResponse
+		// if errors.As(err, &ghErrResp) {
+		// 	if ghErrResp.Response.StatusCode == 404 {
+		// 		log.Warn("[THREAD #%d]: %s", tid, ghErrResp.Error())
+		// 	}
+		// }
+		log.Debug("[THREAD #%d]: GetRepositoriesFromOwner: %s", tid, err.Error())
+	}
 
-		// If we re only looking for a subset of the repos in an org we do a comparison
-		// of the repos gathered for the org and the list pf repos that we care about.
-		for _, repo := range repos {
-			// Increment the total number of repos found even if we are not cloning them
-			sess.State.Stats.IncrementRepositoriesTotal()
+	// In the case where all the repos are private
+	if len(repos) == 0 {
+		log.Debug("No repositories have been gathered for %s", *org.Login)
+		return
+	}
 
-			if sess.GithubUserRepos != nil {
-				for _, r := range sess.GithubUserRepos {
-					if r == repo.Name {
-						log.Debug(" Retrieved repository %s", repo.FullName)
-						// Add the repo to the sess to be scanned
-						sess.AddRepository(repo)
-					}
-				}
-				continue
-			}
+	// If we re only looking for a subset of the repos in an org we do a comparison
+	// of the repos gathered for the org and the list pf repos that we care about.
+	for _, repo := range repos {
+		// Increment the total number of repos found even if we are not cloning them
+		sess.State.Stats.IncrementRepositoriesTotal()
+
+		// Only a subset of repos
+		if sess.GithubUserRepos != nil && isFilteredRepo(repo.Name, sess.GithubUserRepos) {
 			log.Debug(" Retrieved repository %s", repo.FullName)
-			// If we are not doing any filtering and simply grabbing all available repos we add the repos
-			// to the session to be scanned
+			// Add the repo to the sess to be scanned
 			sess.AddRepository(repo)
+			continue
+		}
+
+		log.Debug(" Retrieved repository %s", repo.FullName)
+		// If we are not doing any filtering and simply grabbing all available repos we add the repos
+		// to the session to be scanned
+		sess.AddRepository(repo)
+	}
+}
+
+func isFilteredRepo(name string, in []string) bool {
+	for _, r := range in {
+		if name == r {
+			return true
 		}
 	}
+	return false
 }

internal/core/git-clone.go

@@ -80,7 +80,7 @@ func cloneRepositoryWrapper(cfg *config.Config, repo _coreapi.Repository) (*git.
 			// Token:      , // TODO Is this need since we already have a client?
 		}
 		auth.Username = "oauth2"
-		auth.Password = cfg.GitlabAccessToken
+		auth.Password = cfg.Gitlab.APIToken
 	case api.LocalGit:
 		cloneConfig = CloneConfiguration{
 			URL:        repo.CloneURL,

internal/core/github.go

@@ -18,7 +18,8 @@ import (
 
 const retrievedRepoFromUser string = " Retrieved repository %s from user %s"
 
-// addUser will add a new user to the sess for further scanning and analyzing
+// addUser will add a new user to the sess for further scanning and analyzing.
+// It will also bump the stats.
 func (s *Session) addUser(user *_coreapi.Owner) {
 	s.State.Lock()
 	defer s.State.Unlock()
@@ -38,6 +39,9 @@ func (s *Session) addUser(user *_coreapi.Owner) {
 		}
 	}
 	s.GithubUsers = append(s.GithubUsers, user)
+
+	// bump stats
+	s.State.Stats.IncrementUsers()
 }
 
 // GatherUsers will generate a list of users from github.com that can then be filtered down to a specific target range
@@ -54,7 +58,6 @@ func GatherUsers(sess *Session) error {
 
 		// Add the user to the session and increment the user count
 		sess.addUser(owner)
-		sess.State.Stats.IncrementUsers()
 		log.Debug("Added user %s", *owner.Login)
 	}
 	if len(sess.GithubUsers) == 0 {
@@ -133,15 +136,10 @@ func GatherGithubRepositoriesFromOwner(sess *Session) error {
 	for _, repo := range allRepos {
 		// Increment the total number of repos found, regardless if we are cloning them
 		sess.State.Stats.IncrementRepositoriesTotal()
-		if sess.GithubUserRepos != nil {
-			for _, r := range sess.GithubUserRepos {
-				log.Debug("current repo: %s, comparing to: %s", r, repo.Name)
-				if r == repo.Name {
-					log.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
-					// Add the repo to the sess to be scanned
-					sess.AddRepository(repo)
-				}
-			}
+		if sess.GithubUserRepos != nil && isFilteredRepo(repo.Name, sess.GithubUserRepos) {
+			log.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
+			// Add the repo to the sess to be scanned
+			sess.AddRepository(repo)
 			continue
 		}
 		log.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
@@ -211,7 +209,6 @@ func GatherOrgs(sess *Session, log *log.Logger) error {
 	// Add the orgs to the list for later enumeration of repos
 	for _, org := range orgList {
 		sess.addOrganization(org)
-		sess.State.Stats.IncrementOrgs()
 		log.Debug("Added org %s", *org.Login)
 	}
 	return nil
@@ -240,7 +237,8 @@ func ownerToOrg(owner *_coreapi.Owner) *github.Organization {
 	}
 }
 
-// addOrganization will add a new organization to the session for further scanning and analyzing
+// addOrganization will add a new organization to the session for further scanning and analyzing.
+// It will also bump the stats.
 func (s *Session) addOrganization(organization *github.Organization) {
 	s.State.Lock()
 	defer s.State.Unlock()
@@ -260,6 +258,8 @@ func (s *Session) addOrganization(organization *github.Organization) {
 		}
 	}
 	s.Organizations = append(s.Organizations, organization)
+
+	s.State.Stats.IncrementOrgs()
 }
 
 // GatherGithubOrgRepositories will gather all the repositories for a given org.
@@ -302,24 +302,20 @@ func GatherOrgsMembersRepositories(sess *Session) {
 	var allRepos []*_coreapi.Repository
 	log := sess.Out
 
-	sess.Out.Important("Gathering users from orgs...")
+	log.Important("Gathering users from orgs...")
 	ctx := context.Background()
 
-	// optMember := &github.ListMembersOptions{}
-	// optRepo := &github.RepositoryListOptions{}
-
 	// TODO multi thread this
 	for _, o := range sess.Organizations {
 		members, err := sess.Client.GetOrganizationMembers(ctx, _coreapi.Owner{
 			Login: o.Login,
 		})
 		// Log error and continue with the next org
 		if err != nil {
-			sess.Out.Warn("%v", err)
+			log.Warn("%v", err)
 		}
 		for _, v := range members {
 			sess.addUser(v)
-			sess.State.Stats.IncrementUsers()
 			log.Debug("Added user %s", *v.Login)
 			// find all repositories
 			allRepos, err = sess.Client.GetRepositoriesFromOwner(ctx, *v)
@@ -338,13 +334,13 @@ func GatherOrgsMembersRepositories(sess *Session) {
 		if sess.GithubUserRepos != nil {
 			for _, r := range sess.GithubUserRepos {
 				if r == repo.Name {
-					sess.Out.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
+					log.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
 					sess.AddRepository(repo)
 				}
 			}
 			continue
 		} else {
-			sess.Out.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
+			log.Debug(retrievedRepoFromUser, repo.FullName, repo.Owner)
 			sess.AddRepository(repo)
 		}
 	}

internal/core/gitlab.go

@@ -23,7 +23,7 @@ func GatherTargets(sess *Session) {
 	//case "github":
 	//	targets = sess.GithubTargets
 	//case "gitlab":
-	targets := sess.Config.GitlabTargets
+	targets := sess.Config.Gitlab.Targets
 	//}
 
 	//var target *Owner

internal/core/provider/client.go

@@ -24,7 +24,7 @@ func InitGitClient(cfg *config.Config, log *log.Logger) (api.IClient, error) {
 	case scanAPI.Gitlab:
 		// TODO need to add in the bits to parse the url here as well
 		// TODO set this to some sort of consistent client, look to github for ideas
-		return gitlab.NewClient(cfg.GitlabAccessToken, log)
+		return gitlab.NewClient(cfg.Gitlab.APIToken, log)
 	case scanAPI.UpdateSignatures:
 		return github.NewClient(cfg.Signatures.APIToken, "", log)
 	default:

internal/pkg/scan/gitlab/gitlab.go

@@ -10,7 +10,6 @@ import (
 	"github.com/rumenvasilev/rvsecret/internal/log"
 	"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
 	"github.com/rumenvasilev/rvsecret/internal/webserver"
-	"github.com/spf13/viper"
 )
 
 type Gitlab struct {
@@ -37,8 +36,6 @@ func (g Gitlab) Do() error {
 	// during a silent run which is the default when using this in an automated fashion.
 	banner.HeaderInfo(cfg.Global, sess.State.Stats.StartedAt.Format(time.RFC3339), log)
 
-	cfg.GitlabAccessToken = viper.GetString("gitlab-api-token")
-
 	sess.Client, err = provider.InitGitClient(sess.Config, log)
 	if err != nil {
 		return err