Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history 
commit 59af97a
Author: Rumen Vasilev <[email protected]>
Date:   Wed Sep 20 13:23:15 2023 +0200

    fix linting errors

commit 849cbf1
Author: Rumen Vasilev <[email protected]>
Date:   Wed Sep 20 13:19:16 2023 +0200

    bigfix: didn't produce the expected results

commit d38bf77
Author: Rumen Vasilev <[email protected]>
Date:   Wed Sep 20 13:18:55 2023 +0200

    signatures, findings, banner, git, provider core packages

commit 17cdd77
Author: Rumen Vasilev <[email protected]>
Date:   Wed Sep 20 07:53:19 2023 +0200

    replace pkg.Scan() with interface

commit 67af06d
Author: Rumen Vasilev <[email protected]>
Date:   Wed Sep 20 01:46:45 2023 +0200

    simplify a bit

commit 72facd7
Author: Rumen Vasilev <[email protected]>
Date:   Wed Sep 20 00:53:01 2023 +0200

    add authn
  • Loading branch information
@rumenvasilev
rumenvasilev committed Sep 20, 2023
1 parent da5f716 commit 4fbb95d
Show file tree
Hide file tree
Showing 41 changed files with 1,189 additions and 1,061 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ A hard fork of [wraith](https://github.com/N0MoreSecr3ts/wraith) (which itself i

![build status](https://github.com/rumenvasilev/rvsecret/actions/workflows/on-push.yaml/badge.svg)
[![codecov](https://codecov.io/gh/rumenvasilev/rvsecret/graph/badge.svg?token=X2BXUU5H0S)](https://codecov.io/gh/rumenvasilev/rvsecret)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=rumenvasilev_rvsecret&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=rumenvasilev_rvsecret)

rvsecret detects secrets in git repositories (or localpath). The big differentiator is it is tightly integrated with Github and Gitlab APIs and has no extra dependencies. It's just one binary you can use anywhere (even in scratch containers).

Expand Down
18 changes: 16 additions & 2 deletions assets/static/javascripts/application.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ let Finding = Backbone.Model.extend({
fileContents: function (callback, error) {
$.ajax({
url: this.fileContentsUrl(),
settings: {
beforeSend: sendAuthentication
},
success: callback,
error: error
});
Expand All @@ -72,6 +75,13 @@ let Findings = Backbone.Collection.extend({

window.findings = new Findings();

let sendAuthentication = function (xhr) {
var user = "unknown";
var pass = "wh@tev3$#@FDS";
var token = user.concat(":", pass);
xhr.setRequestHeader('Authorization', ("Bearer ".concat(btoa(token))));
}

let StatsView = Backbone.View.extend({
id: "stats_container",
model: stats,
Expand Down Expand Up @@ -109,7 +119,9 @@ let StatsView = Backbone.View.extend({
},
startPolling: function () {
this.pollingTicker = setInterval(function () {
statsView.model.fetch();
statsView.model.fetch({
beforeSend: sendAuthentication
});
}, this.pollingInterval);
},
stopPolling: function () {
Expand Down Expand Up @@ -265,7 +277,9 @@ let FindingsView = Backbone.View.extend({
});
},
update: function () {
this.collection.fetch();
this.collection.fetch({
beforeSend: sendAuthentication
});
},
renderFinding: function (finding) {
var findingEl = new FindingView({model: finding}).render().el;
Expand Down
6 changes: 3 additions & 3 deletions cmd/scan/scan-github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ package scan
import (
"github.com/rumenvasilev/rvsecret/internal/config"
"github.com/rumenvasilev/rvsecret/internal/log"
"github.com/rumenvasilev/rvsecret/internal/pkg"
"github.com/rumenvasilev/rvsecret/internal/pkg/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
Expand All @@ -26,7 +26,7 @@ var scanGithubCmd = &cobra.Command{
return err
}
log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
return pkg.Scan(cfg, log)
return scan.New(cfg, log).Do()
},
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/scan/scan-gitlab.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ package scan
import (
"github.com/rumenvasilev/rvsecret/internal/config"
"github.com/rumenvasilev/rvsecret/internal/log"
"github.com/rumenvasilev/rvsecret/internal/pkg"
"github.com/rumenvasilev/rvsecret/internal/pkg/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
Expand All @@ -22,7 +22,7 @@ var scanGitlabCmd = &cobra.Command{
return err
}
log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
return pkg.Scan(cfg, log)
return scan.New(cfg, log).Do()
},
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/scan/scan-local-git-repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ package scan
import (
"github.com/rumenvasilev/rvsecret/internal/config"
"github.com/rumenvasilev/rvsecret/internal/log"
"github.com/rumenvasilev/rvsecret/internal/pkg"
"github.com/rumenvasilev/rvsecret/internal/pkg/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
Expand All @@ -21,7 +21,7 @@ var scanLocalGitRepoCmd = &cobra.Command{
return err
}
log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
return pkg.Scan(cfg, log)
return scan.New(cfg, log).Do()
},
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/scan/scan-localpath.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ package scan
import (
"github.com/rumenvasilev/rvsecret/internal/config"
"github.com/rumenvasilev/rvsecret/internal/log"
"github.com/rumenvasilev/rvsecret/internal/pkg"
"github.com/rumenvasilev/rvsecret/internal/pkg/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
Expand All @@ -22,7 +22,7 @@ var scanLocalPathCmd = &cobra.Command{
return err
}
log := log.NewLogger(cfg.Global.Debug, cfg.Global.Silent)
return pkg.Scan(cfg, log)
return scan.New(cfg, log).Do()
},
}

Expand Down
2 changes: 1 addition & 1 deletion cmd/updateRules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package cmd
import (
"github.com/rumenvasilev/rvsecret/internal/config"
"github.com/rumenvasilev/rvsecret/internal/log"
"github.com/rumenvasilev/rvsecret/internal/pkg/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/signatures"

"github.com/spf13/cobra"
Expand Down
56 changes: 28 additions & 28 deletions internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (

"github.com/fatih/structs"
"github.com/mitchellh/go-homedir"
"github.com/rumenvasilev/rvsecret/internal/pkg/api"
"github.com/rumenvasilev/rvsecret/internal/pkg/scan/api"
"github.com/rumenvasilev/rvsecret/internal/util"
"github.com/rumenvasilev/rvsecret/version"
"github.com/spf13/cobra"
Expand All @@ -21,42 +21,42 @@ var cfg *Config

// defaultIgnoreExtensions is an array of extensions that if they match a file that file will be excluded
var defaultIgnoreExtensions = []string{"jpg", "jpeg", "png", "gif", "bmp", "tiff",
"tif", "psd", "xcf"}
"tif", "psd", "xcf", "pdf"}

// defaultIgnorePaths is an array of directories that will be excluded from all types of scans.
var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cache", "/proc/"}

type Config struct {
Signatures Signatures `mapstructure:"signatures" yaml:"signatures"`
Github Github `mapstructure:"github" yaml:"github"`
Gitlab `mapstructure:"gitlab" yaml:"gitlab"`
Global Global `mapstructure:"global" yaml:"global"`
Local Local `mapstructure:"local" yaml:"local"`
Local Local `mapstructure:"local" yaml:"local"`
Signatures Signatures `mapstructure:"signatures" yaml:"signatures"`

Gitlab `mapstructure:"gitlab" yaml:"gitlab"`
Global Global `mapstructure:"global" yaml:"global"`
}

type Global struct {
AppVersion string `yaml:"-"`
BindAddress string `mapstructure:"bind-address" yaml:"bind-address"`
ConfigFile string `mapstructure:"config-file" yaml:"-"`
ScanType api.ScanType `mapstructure:"scan-type" yaml:"-"`
// LocalPaths []string `mapstructure:"paths" yaml:"paths"`
SkippableExt []string `mapstructure:"ignore-extension" yaml:"ignore-extension"`
SkippablePath []string `mapstructure:"ignore-path" yaml:"ignore-path"`
BindPort int `mapstructure:"bind-port" yaml:"bind-port"`
CommitDepth int `mapstructure:"commit-depth" yaml:"commit-depth"`
ConfidenceLevel int `mapstructure:"confidence-level" yaml:"confidence-level"`
MaxFileSize int64 `mapstructure:"max-file-size" yaml:"max-file-size"`
Threads int `mapstructure:"num-threads" yaml:"num-threads"`
CSVOutput bool `mapstructure:"csv"`
Debug bool `mapstructure:"debug"`
ExpandOrgs bool `mapstructure:"expand-orgs" yaml:"expand-orgs"`
HideSecrets bool `mapstructure:"hide-secrets" yaml:"hide-secrets"`
InMemClone bool `mapstructure:"in-mem-clone" yaml:"in-mem-clone"`
JSONOutput bool `mapstructure:"json"`
ScanFork bool `mapstructure:"scan-forks" yaml:"scan-forks"`
ScanTests bool `mapstructure:"scan-tests" yaml:"scan-tests"`
Silent bool `mapstructure:"silent"`
WebServer bool `mapstructure:"web-server" yaml:"web-server"`
AppVersion string `yaml:"-"`
BindAddress string `mapstructure:"bind-address" yaml:"bind-address"`
ConfigFile string `mapstructure:"config-file" yaml:"-"`
ScanType api.ScanType `mapstructure:"scan-type" yaml:"-"`
SkippableExt []string `mapstructure:"ignore-extension" yaml:"ignore-extension"`
SkippablePath []string `mapstructure:"ignore-path" yaml:"ignore-path"`
BindPort int `mapstructure:"bind-port" yaml:"bind-port"`
CommitDepth int `mapstructure:"commit-depth" yaml:"commit-depth"`
ConfidenceLevel int `mapstructure:"confidence-level" yaml:"confidence-level"`
MaxFileSize int64 `mapstructure:"max-file-size" yaml:"max-file-size"`
Threads int `mapstructure:"num-threads" yaml:"num-threads"`
CSVOutput bool `mapstructure:"csv"`
Debug bool `mapstructure:"debug"`
ExpandOrgs bool `mapstructure:"expand-orgs" yaml:"expand-orgs"`
HideSecrets bool `mapstructure:"hide-secrets" yaml:"hide-secrets"`
InMemClone bool `mapstructure:"in-mem-clone" yaml:"in-mem-clone"`
JSONOutput bool `mapstructure:"json"`
ScanFork bool `mapstructure:"scan-forks" yaml:"scan-forks"`
ScanTests bool `mapstructure:"scan-tests" yaml:"scan-tests"`
Silent bool `mapstructure:"silent"`
WebServer bool `mapstructure:"web-server" yaml:"web-server"`
_ [6]byte
}

Expand Down
102 changes: 39 additions & 63 deletions internal/core/analysis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import (
"sync"

_coreapi "github.com/rumenvasilev/rvsecret/internal/core/api"
"github.com/rumenvasilev/rvsecret/internal/core/finding"
_git "github.com/rumenvasilev/rvsecret/internal/core/git"
"github.com/rumenvasilev/rvsecret/internal/core/signatures"
"github.com/rumenvasilev/rvsecret/internal/log"
"github.com/rumenvasilev/rvsecret/internal/matchfile"
"github.com/rumenvasilev/rvsecret/internal/stats"
Expand Down Expand Up @@ -77,7 +80,7 @@ func analyzeWorker(tid int, ch chan _coreapi.Repository, wg *sync.WaitGroup, ses
// The path variable is returning the path that the clone was done to. The repo is cloned directly
// there.
log.Debug("[THREAD #%d][%s] Cloning repository...", tid, repo.CloneURL)
clone, path, err := cloneRepository(sess, repo)
clone, path, err := cloneRepository(sess.Config, stats.IncrementRepositoriesCloned, repo)
if err != nil {
log.Error("%v", err)
cleanUpPath(path, log)
Expand Down Expand Up @@ -107,7 +110,7 @@ func (sess *Session) analyzeHistory(clone *git.Repository, tid int, path string,
log := sess.Out

// Get the full commit history for the repo
history, err := GetRepositoryHistory(clone)
history, err := _git.GetRepositoryHistory(clone)
if err != nil {
log.Error("[THREAD #%d][%s] Cannot get full commit history, error: %v", tid, repo.CloneURL, err)
if err := os.RemoveAll(path); err != nil {
Expand Down Expand Up @@ -151,18 +154,18 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito
// through the commit history of a given repo.
dirtyCommit := false

changes, _ := GetChanges(commit, clone)
changes, _ := _git.GetChanges(commit, clone)
log.Debug("[THREAD #%d][%s] %d changes in %s", tid, repo.CloneURL, len(changes), commit.Hash)

for _, change := range changes {
// The total number of files that were evaluated
stats.IncrementFilesTotal()

// TODO Is this need for the finding object, why are we saving this?
changeAction := GetChangeAction(change)
changeAction := _git.GetChangeAction(change)

// TODO Add an example of the output from this function
fPath := GetChangePath(change)
fPath := _git.GetChangePath(change)

// TODO Add an example of this
// FIXME This is where I have tracked the in-mem-clone issue to
Expand All @@ -185,50 +188,42 @@ func (sess *Session) isDirtyCommit(commit *object.Commit, repo _coreapi.Reposito
// We set this to a default of false and will be used at the end of matching to
// increment the file count. If we try and do this in the loop it will hit for every
// signature and give us a false count.
dirtyFile := false

// for each signature that is loaded scan the file as a whole and generate a map of
// the match and the line number the match was found on
for _, signature := range Signatures {
bMatched, matchMap := signature.ExtractMatch(mf, sess, change)
if bMatched {
dirtyFile = true
dirtyCommit = true

// content will hold the secret found within the target
var content string

// For every instance of the secret that matched the specific signatures
// create a new finding. This will produce dupes as the file may exist
// in multiple commits.
for k, v := range matchMap {
// Default to no content, only publish information if explicitly allowed to
content = ""
if matchMap != nil && !sess.Config.Global.HideSecrets {
// This sets the content for the finding, in this case the actual secret
// is the content. This can be removed and hidden via a commandline flag.
cleanK := strings.SplitAfterN(k, "_", 2)
content = cleanK[1]
}

// Create a new instance of a finding and set the necessary fields.
finding := createFinding(changeAction, content, commit, signature, fPath, sess, v, repo)
// Set the urls for the finding
finding.Initialize(sess)

// Add it to the session
sess.AddFinding(finding)
log.Debug("[THREAD #%d][%s] Done analyzing changes in %s", tid, repo.CloneURL, commit.Hash)

// Print realtime data to stdout
finding.RealtimeOutput(sess)
}
// dirtyFile := false

// call signaturesfunc
dirtyFile, dcommit, out := signatures.Discover(mf, change, sess.Config, log)
for _, v := range out {
fin := &finding.Finding{
Action: changeAction,
Content: v.Content,
CommitAuthor: commit.Author.String(),
CommitHash: commit.Hash.String(),
CommitMessage: strings.TrimSpace(commit.Message),
Description: v.Sig.Description(),
FilePath: fPath,
AppVersion: sess.Config.Global.AppVersion,
LineNumber: strconv.Itoa(v.LineNum),
RepositoryName: repo.Name,
RepositoryOwner: repo.Owner,
SignatureID: v.Sig.SignatureID(),
SignatureVersion: sess.SignatureVersion,
SecretID: util.GenerateID(),
}
_ = fin.Initialize(sess.Config.Global.ScanType, sess.Config.Github.GithubEnterpriseURL)
// Add it to the session
sess.AddFinding(fin)
log.Debug("[THREAD #%d][%s] Done analyzing changes in %s", tid, repo.CloneURL, commit.Hash)

// // Print realtime data to stdout
fin.RealtimeOutput(sess.Config.Global, log)
}

if dirtyFile {
log.Debug("this is the file getting added: %s ", fullFilePath)
stats.IncrementFilesDirty()
}
if dcommit {
dirtyCommit = dcommit
}
}
return dirtyCommit
}
Expand Down Expand Up @@ -265,22 +260,3 @@ func ignoredFile(cfgScanTests bool, cfgMaxFileSize int64, fullFilePath string, m
}
return false, ""
}

func createFinding(changeAction, content string, commit *object.Commit, sig Signature, fPath string, sess *Session, lineNum int, repo _coreapi.Repository) *Finding {
return &Finding{
Action: changeAction,
Content: content,
CommitAuthor: commit.Author.String(),
CommitHash: commit.Hash.String(),
CommitMessage: strings.TrimSpace(commit.Message),
Description: sig.Description(),
FilePath: fPath,
AppVersion: sess.Config.Global.AppVersion,
LineNumber: strconv.Itoa(lineNum),
RepositoryName: repo.Name,
RepositoryOwner: repo.Owner,
SignatureID: sig.SignatureID(),
SignatureVersion: sess.SignatureVersion,
SecretID: util.GenerateID(),
}
}
Loading

0 comments on commit 4fbb95d

Please sign in to comment.