Skip to content

Commit

Permalink
fix local scan commands
Browse files Browse the repository at this point in the history
  • Loading branch information
@rumenvasilev
rumenvasilev committed Sep 19, 2023
1 parent 7f0003e commit e570f9b
Show file tree
Hide file tree
Showing 6 changed files with 112 additions and 94 deletions.
5 changes: 2 additions & 3 deletions cmd/scan/scan-local-git-repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ var scanLocalGitRepoCmd = &cobra.Command{

func init() {
ScanCmd.AddCommand(scanLocalGitRepoCmd)
// scanLocalGitRepoCmd.Flags().Float64("commit-depth", -1, "Set the commit depth to scan")
scanLocalGitRepoCmd.Flags().StringSlice("local-repos", nil, "List of local git repos to scan")
viper.BindPFlags(scanLocalGitRepoCmd.Flags()) //nolint:errcheck
scanLocalGitRepoCmd.Flags().StringSliceP("paths", "p", nil, "List of local git repos to scan")
viper.BindPFlag("local.repos", scanLocalGitRepoCmd.Flags().Lookup("paths")) //nolint:errcheck
}
2 changes: 1 addition & 1 deletion cmd/scan/scan-localpath.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,5 @@ var scanLocalPathCmd = &cobra.Command{
func init() {
ScanCmd.AddCommand(scanLocalPathCmd)
scanLocalPathCmd.Flags().StringSliceP("paths", "p", nil, "List of local paths to scan")
viper.BindPFlags(scanLocalPathCmd.Flags()) //nolint:errcheck
viper.BindPFlag("local.paths", scanLocalPathCmd.Flags().Lookup("paths")) //nolint:errcheck
}
50 changes: 28 additions & 22 deletions internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,31 +31,32 @@ type Config struct {
Github Github `mapstructure:"github" yaml:"github"`
Gitlab `mapstructure:"gitlab" yaml:"gitlab"`
Global Global `mapstructure:"global" yaml:"global"`
Local Local `mapstructure:"local" yaml:"local"`
}

type Global struct {
AppVersion string `yaml:"-"`
BindAddress string `mapstructure:"bind-address" yaml:"bind-address"`
ConfigFile string `mapstructure:"config-file" yaml:"-"`
ScanType api.ScanType `mapstructure:"scan-type" yaml:"-"`
LocalPaths []string `mapstructure:"paths"`
SkippableExt []string `mapstructure:"ignore-extension" yaml:"ignore-extension"`
SkippablePath []string `mapstructure:"ignore-path" yaml:"ignore-path"`
BindPort int `mapstructure:"bind-port" yaml:"bind-port"`
CommitDepth int `mapstructure:"commit-depth" yaml:"commit-depth"`
ConfidenceLevel int `mapstructure:"confidence-level" yaml:"confidence-level"`
MaxFileSize int64 `mapstructure:"max-file-size" yaml:"max-file-size"`
Threads int `mapstructure:"num-threads" yaml:"num-threads"`
CSVOutput bool `mapstructure:"csv"`
Debug bool `mapstructure:"debug"`
ExpandOrgs bool `mapstructure:"expand-orgs" yaml:"expand-orgs"`
HideSecrets bool `mapstructure:"hide-secrets" yaml:"hide-secrets"`
InMemClone bool `mapstructure:"in-mem-clone" yaml:"in-mem-clone"`
JSONOutput bool `mapstructure:"json"`
ScanFork bool `mapstructure:"scan-forks" yaml:"scan-forks"`
ScanTests bool `mapstructure:"scan-tests" yaml:"scan-tests"`
Silent bool `mapstructure:"silent"`
WebServer bool `mapstructure:"web-server" yaml:"web-server"`
AppVersion string `yaml:"-"`
BindAddress string `mapstructure:"bind-address" yaml:"bind-address"`
ConfigFile string `mapstructure:"config-file" yaml:"-"`
ScanType api.ScanType `mapstructure:"scan-type" yaml:"-"`
// LocalPaths []string `mapstructure:"paths" yaml:"paths"`
SkippableExt []string `mapstructure:"ignore-extension" yaml:"ignore-extension"`
SkippablePath []string `mapstructure:"ignore-path" yaml:"ignore-path"`
BindPort int `mapstructure:"bind-port" yaml:"bind-port"`
CommitDepth int `mapstructure:"commit-depth" yaml:"commit-depth"`
ConfidenceLevel int `mapstructure:"confidence-level" yaml:"confidence-level"`
MaxFileSize int64 `mapstructure:"max-file-size" yaml:"max-file-size"`
Threads int `mapstructure:"num-threads" yaml:"num-threads"`
CSVOutput bool `mapstructure:"csv"`
Debug bool `mapstructure:"debug"`
ExpandOrgs bool `mapstructure:"expand-orgs" yaml:"expand-orgs"`
HideSecrets bool `mapstructure:"hide-secrets" yaml:"hide-secrets"`
InMemClone bool `mapstructure:"in-mem-clone" yaml:"in-mem-clone"`
JSONOutput bool `mapstructure:"json"`
ScanFork bool `mapstructure:"scan-forks" yaml:"scan-forks"`
ScanTests bool `mapstructure:"scan-tests" yaml:"scan-tests"`
Silent bool `mapstructure:"silent"`
WebServer bool `mapstructure:"web-server" yaml:"web-server"`
_ [6]byte
}

Expand Down Expand Up @@ -87,6 +88,11 @@ type Gitlab struct {
_ [24]byte
}

type Local struct {
Paths []string `mapstructure:"paths"`
Repos []string `mapstructure:"repos"`
}

// SetConfig will set the defaults, and load a config file and environment variables if they are present
func SetConfig(cmd *cobra.Command) {
// Set defaults
Expand Down
138 changes: 72 additions & 66 deletions internal/core/localRepo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ func GatherLocalRepositories(sess *Session) error {
// This is the number of targets as we don't do forks or anything else.
// It will contain directories, that will then be added to the repo count
// if they contain a .git directory
sess.State.Stats.Targets = len(sess.Config.Global.LocalPaths)
sess.State.Stats.Targets = len(sess.Config.Local.Repos)
sess.State.Stats.UpdateStatus(_coreapi.StatusGathering)
sess.Out.Important("Gathering Local Repositories...")

for _, pth := range sess.Config.Global.LocalPaths {
for _, pth := range sess.Config.Local.Repos {

if !util.PathExists(pth, log) {
return fmt.Errorf("[*] <%s> does not exist! Quitting.", pth)
return fmt.Errorf("[*] <%s> does not exist! Quitting", pth)
}

// Gather all paths in the tree
Expand All @@ -38,70 +38,76 @@ func GatherLocalRepositories(sess *Session) error {
return nil
}

// If it is a directory then move forward
if f.IsDir() {

// If there is a .git directory then we have a repo
if filepath.Ext(path) == ".git" { // TODO Should we reverse this to ! to make the code cleaner
// Set the url to the relative path of the repo based on the execution path of rvsecret
repoURL, _ := filepath.Split(path)
// This is used to id the owner, fullname, and description of the repo. It is ugly but effective. It is the relative path to the repo, for example ../foo
gitProjName, _ := filepath.Split(repoURL)

openRepo, err1 := git.PlainOpen(repoURL)
if err1 != nil {
return nil
}

ref, err1 := openRepo.Head()
if err1 != nil {
sess.Out.Error("Failed to open the repo HEAD: %s", err1.Error())
return nil
}

// Get the name of the branch we are working on
// s := ref.Strings()
// branchPath := fmt.Sprintf("%s", s[0])
branchPathParts := strings.Split(ref.Strings()[0], string("refs/heads/"))
branchName := branchPathParts[len(branchPathParts)-1]

commit, _ := openRepo.CommitObject(ref.Hash())
var commitHash = commit.Hash[:]

// TODO make this a generic function at some point
// Generate a uid for the repo
h := sha1.New()
repoID := fmt.Sprintf("%x", h.Sum(commitHash))

intRepoID, _ := strconv.ParseInt(repoID, 10, 64)
// var pRepoID *int64
// pRepoID = &intRepoID

// Set the url to the relative path of the repo based on the execution path of rvsecret
// pRepoURL := &parent

// pGitProjName := &gitProjName

// The project name is simply the parent directory in the case of a local scan with all other path bits removed for example ../foo -> foo.
projectPathParts := strings.Split(gitProjName, string(os.PathSeparator))
projectName := projectPathParts[len(projectPathParts)-2]

sessR := _coreapi.Repository{
Owner: gitProjName,
ID: intRepoID,
Name: projectName,
FullName: gitProjName,
CloneURL: repoURL,
URL: repoURL,
DefaultBranch: branchName,
Description: gitProjName,
Homepage: repoURL,
}

// Add the repo to the sess to be cloned and scanned
sess.AddRepository(&sessR)
}
// If it is not a directory, exit
if !f.IsDir() {
return nil
}

// If there is a .git directory then we have a repo
if filepath.Ext(path) != ".git" {
return nil
}

// Set the url to the relative path of the repo based on the execution path of rvsecret
repoURL, _ := filepath.Split(path)
// This is used to id the owner, fullname, and description of the repo. It is ugly but effective. It is the relative path to the repo, for example ../foo
gitProjName, _ := filepath.Split(repoURL)

openRepo, err1 := git.PlainOpen(repoURL)
if err1 != nil {
// if err1 == git.ErrRepositoryNotExists {
sess.Out.Error(err1.Error())
// }
return nil
}

ref, err1 := openRepo.Head()
if err1 != nil {
sess.Out.Error("Failed to open the repo HEAD: %s", err1.Error())
return nil
}

// Get the name of the branch we are working on
// s := ref.Strings()
// branchPath := fmt.Sprintf("%s", s[0])
branchPathParts := strings.Split(ref.Strings()[0], string("refs/heads/"))
branchName := branchPathParts[len(branchPathParts)-1]

commit, _ := openRepo.CommitObject(ref.Hash())
var commitHash = commit.Hash[:]

// TODO make this a generic function at some point
// Generate a uid for the repo
h := sha1.New()
repoID := fmt.Sprintf("%x", h.Sum(commitHash))

intRepoID, _ := strconv.ParseInt(repoID, 10, 64)
// var pRepoID *int64
// pRepoID = &intRepoID

// Set the url to the relative path of the repo based on the execution path of rvsecret
// pRepoURL := &parent

// pGitProjName := &gitProjName

// The project name is simply the parent directory in the case of a local scan with all other path bits removed for example ../foo -> foo.
projectPathParts := strings.Split(gitProjName, string(os.PathSeparator))
projectName := projectPathParts[len(projectPathParts)-2]

sessR := _coreapi.Repository{
Owner: gitProjName,
ID: intRepoID,
Name: projectName,
FullName: gitProjName,
CloneURL: repoURL,
URL: repoURL,
DefaultBranch: branchName,
Description: gitProjName,
Homepage: repoURL,
}

// Add the repo to the sess to be cloned and scanned
sess.AddRepository(&sessR)
return nil
})
if err != nil {
Expand Down
7 changes: 7 additions & 0 deletions internal/pkg/localgit/localgit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@ func Scan(cfg *config.Config, log *log.Logger) error {
ws := webserver.New(*cfg, sess.State, log)
go ws.Start()
}

log.Debug("We have these paths: %s", cfg.Local.Repos)

if cfg.Global.Debug {
log.Debug(config.PrintDebug(sess.SignatureVersion))
}

// By default we display a header to the user giving basic info about application. This will not be displayed
// during a silent run which is the default when using this in an automated fashion.
core.HeaderInfo(*cfg, sess.State.Stats.StartedAt.Format(time.RFC3339), log)
Expand Down
4 changes: 2 additions & 2 deletions internal/pkg/localpath/localpath.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func Scan(cfg *config.Config, log *log.Logger) error {
go ws.Start()
}

log.Debug("We have these paths: %s", cfg.Global.LocalPaths)
log.Debug("We have these paths: %s", cfg.Local.Paths)

if cfg.Global.Debug {
log.Debug(config.PrintDebug(sess.SignatureVersion))
Expand All @@ -36,7 +36,7 @@ func Scan(cfg *config.Config, log *log.Logger) error {
// during a silent run which is the default when using this in an automated fashion.
core.HeaderInfo(*cfg, sess.State.Stats.StartedAt.Format(time.RFC3339), log)

for _, p := range cfg.Global.LocalPaths {
for _, p := range cfg.Local.Paths {
if util.PathExists(p, log) {
last := p[len(p)-1:]
if last == "/" {
Expand Down

0 comments on commit e570f9b

Please sign in to comment.