diff --git a/cmd/updateRules.go b/cmd/updateRules.go index f78ebb3..db1b429 100644 --- a/cmd/updateRules.go +++ b/cmd/updateRules.go @@ -26,8 +26,11 @@ func init() { rootCmd.AddCommand(updateSignaturesCmd) updateSignaturesCmd.Flags().StringP("github-api-token", "t", "", "API token for github access, see documentation for necessary scope") updateSignaturesCmd.MarkFlagRequired("github-api-token") //nolint:errcheck + updateSignaturesCmd.Flags().String("signatures-user-repo", "", "user/repo where signatures can be found, example: rumenvasilev/rvsecret-signatures") updateSignaturesCmd.Flags().String("signatures-url", "https://github.com/rumenvasilev/rvsecret-signatures", "url where the signatures can be found") - updateSignaturesCmd.Flags().String("signatures-version", "latest", "specific version of the signatures to install (current, latest, v1.2.0)") + updateSignaturesCmd.MarkFlagsMutuallyExclusive("signatures-user-repo", "signatures-url") + updateSignaturesCmd.Flags().String("signatures-version", "latest", "specific version of the signatures to install (latest, v1.2.0)") updateSignaturesCmd.Flags().Bool("test-signatures", false, "run any tests associated with the signatures and display the output") + updateSignaturesCmd.Flags().Bool("debug", false, "Print available debugging information to stdout") viper.BindPFlags(updateSignaturesCmd.Flags()) //nolint:errcheck } diff --git a/internal/config/config.go b/internal/config/config.go index f1e8230..6925daf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,34 +24,34 @@ var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cach // DefaultValues is a map of all flag default values and other mutable variables var defaultValues = map[string]interface{}{ - "bind-address": "127.0.0.1", - "bind-port": 9393, - "commit-depth": -1, - "config-file": "$HOME/.rvsecret/config.yaml", - "csv": false, - "debug": false, - "ignore-extension": nil, - "ignore-path": nil, - "in-mem-clone": false, - "json": false, - "max-file-size": 10, - "num-threads": -1, - "local-paths": nil, - "scan-forks": false, - "scan-tests": false, - "scan-type": "", - "silent": false, - "confidence-level": 3, - "signatures-file": "$HOME/.rvsecret/signatures/default.yaml", - "signatures-path": "$HOME/.rvsecret/signatures/", - "signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures", - "signatures-version": "", - "scan-dir": nil, - "scan-file": nil, - "hide-secrets": false, - "rules-url": "", - "test-signatures": false, - "web-server": false, + "bind-address": "127.0.0.1", + "bind-port": 9393, + "commit-depth": -1, + "config-file": "$HOME/.rvsecret/config.yaml", + "csv": false, + "debug": false, + "ignore-extension": nil, + "ignore-path": nil, + "in-mem-clone": false, + "json": false, + "max-file-size": 10, + "num-threads": -1, + "local-paths": nil, + "scan-forks": false, + "scan-tests": false, + "scan-type": "", + "silent": false, + "confidence-level": 3, + "signatures-file": "$HOME/.rvsecret/signatures/default.yaml", + "signatures-path": "$HOME/.rvsecret/signatures/", + "signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures", + // "signatures-version": "", + "scan-dir": nil, + "scan-file": nil, + "hide-secrets": false, + "rules-url": "", + "test-signatures": false, + "web-server": false, // Github "add-org-members": false, "github-enterprise-url": "", @@ -88,13 +88,8 @@ func SetConfig() { viper.SetConfigType("yaml") } - if err := viper.ReadInConfig(); err != nil { - fmt.Println("Couldn't load Viper config.", err) - // os.Exit(1) - } - + viper.ReadInConfig() //nolint:errcheck viper.AutomaticEnv() - cfg = viper.GetViper() } @@ -126,7 +121,9 @@ type Config struct { SignaturesFile string SignaturesPath string SignaturesURL string + SignaturesUserRepo string SignaturesVersion string + SignaturesTest bool Threads int UserDirtyNames []string UserDirtyOrgs []string @@ -182,6 +179,8 @@ func Load(scanType api.ScanType) (*Config, error) { c.GithubAccessToken = cfg.GetString("github-api-token") c.SignaturesVersion = viper.GetString("signatures-version") c.SignaturesURL = viper.GetString("signatures-url") + c.SignaturesUserRepo = viper.GetString("signatures-user-repo") + c.SignaturesTest = viper.GetBool("test-signatures") } // Add the default directories to the sess if they don't already exist diff --git a/internal/core/git.go b/internal/core/git.go index 9410de3..258d7f7 100644 --- a/internal/core/git.go +++ b/internal/core/git.go @@ -39,6 +39,8 @@ type CloneConfiguration struct { Username string Token string Branch string + Tag bool + TagMode git.TagMode Depth int } @@ -240,18 +242,26 @@ func cloneRepositoryFunc(sess *Session, repo _coreapi.Repository) (*git.Reposito // cloneRepositoryGeneric will create either an in memory clone of a given repository or clone to a temp dir. func CloneRepositoryGeneric(config CloneConfiguration, auth *http.BasicAuth) (repo *git.Repository, dir string, err error) { + ref := plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", config.Branch)) + if config.Tag { + ref = plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", config.Branch)) + } cloneOptions := &git.CloneOptions{ URL: config.URL, Depth: config.Depth, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", config.Branch)), + ReferenceName: ref, SingleBranch: true, - Tags: git.NoTags, + Tags: config.TagMode, } if auth != nil { cloneOptions.Auth = auth } + if config.TagMode == git.InvalidTagMode { + cloneOptions.Tags = git.NoTags + } + if !config.InMemClone { dir, err = os.MkdirTemp("", "rvsecret") if err != nil { diff --git a/internal/core/provider/github/client.go b/internal/core/provider/github/client.go index ea05936..fbdf761 100644 --- a/internal/core/provider/github/client.go +++ b/internal/core/provider/github/client.go @@ -222,3 +222,21 @@ func (c *Client) GetOrganizationMembers(ctx context.Context, target _coreapi.Own return allMembers, nil } + +func (c *Client) GetLatestRelease(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) { + release, _, err := c.apiClient.Repositories.GetLatestRelease(ctx, owner, repo) + if err != nil { + return nil, err + } + + return release, nil +} + +func (c *Client) GetReleaseByTag(ctx context.Context, owner, repo, tag string) (*github.RepositoryRelease, error) { + release, _, err := c.apiClient.Repositories.GetReleaseByTag(ctx, owner, repo, tag) + if err != nil { + return nil, err + } + + return release, nil +} diff --git a/internal/pkg/signatures/fetch-git.go b/internal/pkg/signatures/fetch-git.go new file mode 100644 index 0000000..9449921 --- /dev/null +++ b/internal/pkg/signatures/fetch-git.go @@ -0,0 +1,60 @@ +package signatures + +import ( + "fmt" + "os" + + "github.com/rumenvasilev/rvsecret/internal/core" + "gopkg.in/src-d/go-git.v4" + githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" +) + +// fetchSignaturesWithGit will clone the signatures repository and return the local path to it +func fetchSignaturesWithGit(version string, sess *core.Session) (string, error) { + branch := version + tag := true + if version == "latest" { + branch = "stable" + tag = false + } + err := sess.InitGitClient() + if err != nil { + return "", err + } + + // build the URL + url := sess.Config.SignaturesURL + if sess.Config.SignaturesUserRepo != "" { + // TODO this address has to be a const perhaps? + url = fmt.Sprintf("https://github.com/%s", sess.Config.SignaturesUserRepo) + } + // sanitize checks + cURL, err := cleanInput(url) + if err != nil { + return "", err + } + + cloneCfg := core.CloneConfiguration{ + URL: cURL, + Branch: branch, + Depth: sess.Config.CommitDepth, + InMemClone: sess.Config.InMemClone, + Tag: tag, + // Should we? + TagMode: git.AllTags, + } + auth := &githttp.BasicAuth{ + Username: "egal", + Password: sess.Config.GithubAccessToken, + } + + // If we're gonna use git clone to get a specific tag, we need to pass git.AllTags as parameter here. + _, dir, err := core.CloneRepositoryGeneric(cloneCfg, auth) + if err != nil { + // cleanup dir + _ = os.RemoveAll(dir) + return "", err + } + + return dir, nil +} diff --git a/internal/pkg/signatures/fetch-restapi.go b/internal/pkg/signatures/fetch-restapi.go new file mode 100644 index 0000000..6d82c8c --- /dev/null +++ b/internal/pkg/signatures/fetch-restapi.go @@ -0,0 +1,110 @@ +package signatures + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + + "github.com/google/go-github/github" + "github.com/rumenvasilev/rvsecret/internal/core" + _github "github.com/rumenvasilev/rvsecret/internal/core/provider/github" + "github.com/rumenvasilev/rvsecret/version" +) + +// fetchSignaturesFromGithubAPI will only download a version of the signatures file from Github REST API +func fetchSignaturesFromGithubAPI(version string, sess *core.Session) (string, error) { + ctx := context.Background() + if sess.Config.SignaturesUserRepo == "" { + return "", fmt.Errorf("please provide -signatures-user-repo value") + } + + res := strings.Split(sess.Config.SignaturesUserRepo, "/") + if len(res) != 2 { + return "", fmt.Errorf("user/repo doesn't have matching format, %s", sess.Config.SignaturesUserRepo) + } + owner := res[0] + repo := res[1] + + client, err := _github.NewClient(sess.Config.GithubAccessToken, "", sess.Out) + if err != nil { + return "", fmt.Errorf("failed instantiation of Github client, %w", err) + } + + var assets *github.RepositoryRelease + if version == "latest" { + assets, err = client.GetLatestRelease(ctx, owner, repo) + } else { + assets, err = client.GetReleaseByTag(ctx, owner, repo, version) + } + if err != nil { + // TODO: handle 404 not found + return "", fmt.Errorf("error while fetching release information, %w", err) + } + + assetURL, err := getAssetURL(assets.Assets) + if err != nil { + return "", err + } + + return downloadAsset(assetURL, sess) +} + +func getAssetURL(assets []github.ReleaseAsset) (string, error) { + var download string + for _, v := range assets { + if v.GetName() == "default.yaml" { + download = v.GetURL() + break + } + } + if download == "" { + return "", fmt.Errorf("couldn't find the release asset default.yaml") + } + return download, nil +} + +func downloadAsset(url string, sess *core.Session) (string, error) { + // Create tmp dir + path, err := os.MkdirTemp("", "rvsecret") + if err != nil { + return "", err + } + + err = os.Mkdir(fmt.Sprintf("%s/signatures", path), 0700) + if err != nil { + return "", err + } + + // fetch from URL + req, _ := http.NewRequest("GET", url, nil) + req.Header.Add("Authorization", fmt.Sprintf("token %s", sess.Config.GithubAccessToken)) + req.Header.Add("User-Agent", version.UserAgent) + req.Header.Add("Accept", "application/octet-stream") + + // call github + c := http.Client{} + resp, err := c.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() //nolint:errcheck + + // store file + filename := fmt.Sprintf("%s/signatures/default.yaml", path) + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) + defer f.Close() //nolint:staticcheck + if err != nil { + return "", err + } + + b := make([]byte, 4096) + var i int + for err == nil { + i, err = resp.Body.Read(b) + f.Write(b[:i]) //nolint:errcheck + } + + return path, nil +} diff --git a/internal/pkg/signatures/signatures.go b/internal/pkg/signatures/signatures.go index b1c3797..beff54f 100644 --- a/internal/pkg/signatures/signatures.go +++ b/internal/pkg/signatures/signatures.go @@ -1,29 +1,24 @@ package signatures import ( + "errors" "fmt" + "net/http" "os" "path/filepath" "regexp" + "github.com/google/go-github/github" cp "github.com/otiai10/copy" "github.com/rumenvasilev/rvsecret/internal/config" "github.com/rumenvasilev/rvsecret/internal/core" "github.com/rumenvasilev/rvsecret/internal/log" "github.com/rumenvasilev/rvsecret/internal/pkg/api" "github.com/rumenvasilev/rvsecret/internal/util" - "github.com/spf13/viper" whilp "github.com/whilp/git-urls" - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) // https://raw.githubusercontent.com/N0MoreSecr3ts/wraith-signatures/develop/signatures/default.yaml -// latest release -> -// curl https://github.com/N0MoreSecr3ts/wraith-signatures/releases/latest -// 301 -> https://github.com/N0MoreSecr3ts/wraith-signatures/releases/tag/0.0.2 -// https://raw.githubusercontent.com/N0MoreSecr3ts/wraith-signatures/0.0.2/signatures/default.yaml // signature version options // `latest` => last release in github @@ -31,6 +26,8 @@ import ( // `semver` => specific version func Update(log *log.Logger) error { + var dir string + // load config cfg, err := config.Load(api.UpdateSignatures) if err != nil { @@ -43,121 +40,52 @@ func Update(log *log.Logger) error { return err } - // get the signatures version or if blank, set it to latest - semver := regexp.MustCompile(`^[0-2].[0-9]+.[0-9]+$`) - switch cfg.SignaturesVersion { case "latest": - // find latest released (tagged) version - case "master", "main": - // download branch version + log.Debug("Fetching latest release") default: - // get specific tag - if semver.MatchString(cfg.SignaturesVersion) { - // semver + log.Debug("Fetching a specific version: %q", cfg.SignaturesVersion) + semver := regexp.MustCompile(`^[0-2].[0-9]+.[0-9]+$`) + if !semver.MatchString(cfg.SignaturesVersion) { + return fmt.Errorf("something went wrong, %w", err) } } - // if viper.GetString("signatures-path") != "" { - // signatureVersion = viper.GetString("signatures-version") - // } - - // NEWCODE - err = sess.InitGitClient() + // try from Github REST API first + dir, err = fetchSignaturesFromGithubAPI(cfg.SignaturesVersion, sess) if err != nil { - return err - } - - url := viper.GetString("signatures-url") - // if err != nil - cURL, _ := cleanInput(url) - cloneCfg := core.CloneConfiguration{ - URL: cURL, - Branch: "main", - Depth: sess.Config.CommitDepth, - InMemClone: sess.Config.InMemClone, - } - auth := &http.BasicAuth{ - Username: "egal", - Password: cfg.GithubAccessToken, + if isCredentialsError(err) { + log.Debug(err.Error()) + return fmt.Errorf("github token is not authorized, please update its permissions or generate a new one") + } + log.Warn("Couldn't fetch the signatures from Github REST API, falling back to git method") + dir, err = fetchSignaturesWithGit(cfg.SignaturesVersion, sess) + if err != nil { + err = fmt.Errorf("couldn't fetch the signatures with git clone either, reason: %w", err) + } } - _, dir, err := core.CloneRepositoryGeneric(cloneCfg, auth) if err != nil { return err } - // fetch the signatures from the remote location - // git clone - // rRepo, err := fetchSignatures(signatureVersion, log) - // if err != nil { - // return err - // } - // install the signatures if updateSignatures(dir, sess, log) { - log.Info("The signatures have been successfully updated at: %s", viper.GetString("signatures-path")) + log.Info("The signatures have been successfully updated at: %s", cfg.SignaturesPath) } else { - log.Warn("The signatures were not updated") + return fmt.Errorf("the signatures were not updated") } return nil } -// fetchSignatures will download the signatures from a remote location to a temp location -func fetchSignatures(signatureVersion string, log *log.Logger) (string, error) { - - // TODO if this is not set then pull from the stock place, that should be the default url set in the session - rURL := viper.GetString("signatures-url") - - // set the remote url that we will fetch - // TODO need to look into this more - remoteURL, err := cleanInput(rURL) - if err != nil { - return "", err - } - - // TODO document this - dir, err := os.MkdirTemp("", "rvsecret") - if err != nil { - return "", err - } - - // for now we only pull from a given version at some point we can look at pulling the latest - // TODO be able to pass in a commit or version string - repo, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: remoteURL, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", "stable")), - SingleBranch: true, - Tags: git.AllTags, - }) - if err != nil { - defer os.RemoveAll(dir) - return "", fmt.Errorf("Failed to clone signatures repository, %w", err) - } - - // TODO give a valid error if the version is not REMOVE ME - if signatureVersion != "" { - // Get the working tree so we can change refs - // TODO figure this out REMOVE ME - tree, err := repo.Worktree() - if err != nil { - log.Error(err.Error()) - } - - // Set the tag to the signatures version that we want to use - // TODO fix this REMOVE ME - tagName := string(signatureVersion) - - // Checkout our tag - // TODO way are we using a tag here is we only checkout master - // TODO fix this - err = tree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.ReferenceName("refs/tags/" + tagName), - }) - if err != nil { - return "", fmt.Errorf("Requested version not available. Please enter a valid version") +func isCredentialsError(err error) bool { + var gherr *github.ErrorResponse + if errors.As(err, &gherr) { + if gherr.Response.StatusCode == http.StatusUnauthorized { + // log.Debug(err.Error()) + return true } } - return dir, nil + return false } // cleanInput will ensure that any user supplied git url is in the proper format @@ -176,21 +104,16 @@ func updateSignatures(rRepo string, sess *core.Session, log *log.Logger) bool { // TODO put this in /tmp via a real library tempSignaturesDir := rRepo + "/signatures" - // final resting place for the signatures - rPath := viper.GetString("signatures-path") - // ensure we have the proper home directory - var err error - rPath, err = util.SetHomeDir(rPath) + home, err := util.SetHomeDir(sess.Config.SignaturesPath) if err != nil { // TODO-RV: Do something more? log.Error(err.Error()) } // if the signatures path does not exist then we create it - if !util.PathExists(rPath, log) { - - err := os.MkdirAll(rPath, 0700) + if !util.PathExists(home, log) { + err := os.MkdirAll(home, 0700) if err != nil { log.Error(err.Error()) } @@ -198,19 +121,19 @@ func updateSignatures(rRepo string, sess *core.Session, log *log.Logger) bool { // if we want to test the signatures before we install them // TODO need to implement something here - if viper.GetBool("test-signatures") { + if sess.Config.SignaturesTest { // if the tests pass then we install the signatures if executeTests(rRepo) { // copy the files from the temp directory to the signatures directory - if err := cp.Copy(tempSignaturesDir, rPath); err != nil { + if err := cp.Copy(tempSignaturesDir, home); err != nil { log.Error(err.Error()) return false } // get all the files in the signatures directory - files, err := os.ReadDir(rPath) + files, err := os.ReadDir(home) if err != nil { log.Error(err.Error()) return false @@ -218,7 +141,7 @@ func updateSignatures(rRepo string, sess *core.Session, log *log.Logger) bool { // set them to the current user and the proper permissions for _, f := range files { - if err := os.Chmod(rPath+"/"+f.Name(), 0644); err != nil { + if err := os.Chmod(home+"/"+f.Name(), 0644); err != nil { log.Error(err.Error()) return false } @@ -239,13 +162,13 @@ func updateSignatures(rRepo string, sess *core.Session, log *log.Logger) bool { } // copy the files from the temp directory to the signatures directory - if err := cp.Copy(tempSignaturesDir, rPath); err != nil { + if err := cp.Copy(tempSignaturesDir, home); err != nil { log.Error(err.Error()) return false } // get all the files in the signatures directory - files, err := os.ReadDir(rPath) + files, err := os.ReadDir(home) if err != nil { log.Error(err.Error()) return false @@ -254,17 +177,16 @@ func updateSignatures(rRepo string, sess *core.Session, log *log.Logger) bool { // set them to the current user and the proper permissions // TODO ensure these are .yaml somehow for _, f := range files { - sFileExt := filepath.Ext(rPath + "/" + f.Name()) + sFileExt := filepath.Ext(home + "/" + f.Name()) if sFileExt == "yml" || sFileExt == "yaml" { - if err := os.Chmod(rPath+"/"+f.Name(), 0644); err != nil { + if err := os.Chmod(home+"/"+f.Name(), 0644); err != nil { log.Error(err.Error()) return false } } } - // TODO why is the commented out - // TODO Cleanup after ourselves and remove any temp garbage - // os.RemoveAll(tempSignaturesDir) + // Cleanup after ourselves and remove any temp garbage + _ = os.RemoveAll(rRepo) return true }