Skip to content

Commit

Permalink
rewrite signatures update
Browse files Browse the repository at this point in the history
  • Loading branch information
rumenvasilev committed Sep 15, 2023
1 parent eeb8594 commit 8ecc1f0
Show file tree
Hide file tree
Showing 7 changed files with 280 additions and 158 deletions.
5 changes: 4 additions & 1 deletion cmd/updateRules.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@ func init() {
rootCmd.AddCommand(updateSignaturesCmd)
updateSignaturesCmd.Flags().StringP("github-api-token", "t", "", "API token for github access, see documentation for necessary scope")
updateSignaturesCmd.MarkFlagRequired("github-api-token") //nolint:errcheck
updateSignaturesCmd.Flags().String("signatures-user-repo", "", "user/repo where signatures can be found, example: rumenvasilev/rvsecret-signatures")
updateSignaturesCmd.Flags().String("signatures-url", "https://github.com/rumenvasilev/rvsecret-signatures", "url where the signatures can be found")
updateSignaturesCmd.Flags().String("signatures-version", "latest", "specific version of the signatures to install (current, latest, v1.2.0)")
updateSignaturesCmd.MarkFlagsMutuallyExclusive("signatures-user-repo", "signatures-url")
updateSignaturesCmd.Flags().String("signatures-version", "latest", "specific version of the signatures to install (latest, v1.2.0)")
updateSignaturesCmd.Flags().Bool("test-signatures", false, "run any tests associated with the signatures and display the output")
updateSignaturesCmd.Flags().Bool("debug", false, "Print available debugging information to stdout")
viper.BindPFlags(updateSignaturesCmd.Flags()) //nolint:errcheck
}
67 changes: 33 additions & 34 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,34 +24,34 @@ var defaultIgnorePaths = []string{"node_modules/", "vendor/bundle", "vendor/cach

// DefaultValues is a map of all flag default values and other mutable variables
var defaultValues = map[string]interface{}{
"bind-address": "127.0.0.1",
"bind-port": 9393,
"commit-depth": -1,
"config-file": "$HOME/.rvsecret/config.yaml",
"csv": false,
"debug": false,
"ignore-extension": nil,
"ignore-path": nil,
"in-mem-clone": false,
"json": false,
"max-file-size": 10,
"num-threads": -1,
"local-paths": nil,
"scan-forks": false,
"scan-tests": false,
"scan-type": "",
"silent": false,
"confidence-level": 3,
"signatures-file": "$HOME/.rvsecret/signatures/default.yaml",
"signatures-path": "$HOME/.rvsecret/signatures/",
"signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures",
"signatures-version": "",
"scan-dir": nil,
"scan-file": nil,
"hide-secrets": false,
"rules-url": "",
"test-signatures": false,
"web-server": false,
"bind-address": "127.0.0.1",
"bind-port": 9393,
"commit-depth": -1,
"config-file": "$HOME/.rvsecret/config.yaml",
"csv": false,
"debug": false,
"ignore-extension": nil,
"ignore-path": nil,
"in-mem-clone": false,
"json": false,
"max-file-size": 10,
"num-threads": -1,
"local-paths": nil,
"scan-forks": false,
"scan-tests": false,
"scan-type": "",
"silent": false,
"confidence-level": 3,
"signatures-file": "$HOME/.rvsecret/signatures/default.yaml",
"signatures-path": "$HOME/.rvsecret/signatures/",
"signatures-url": "https://github.com/rumenvasilev/rvsecret-signatures",
// "signatures-version": "",
"scan-dir": nil,
"scan-file": nil,
"hide-secrets": false,
"rules-url": "",
"test-signatures": false,
"web-server": false,
// Github
"add-org-members": false,
"github-enterprise-url": "",
Expand Down Expand Up @@ -88,13 +88,8 @@ func SetConfig() {
viper.SetConfigType("yaml")
}

if err := viper.ReadInConfig(); err != nil {
fmt.Println("Couldn't load Viper config.", err)
// os.Exit(1)
}

viper.ReadInConfig() //nolint:errcheck
viper.AutomaticEnv()

cfg = viper.GetViper()
}

Expand Down Expand Up @@ -126,7 +121,9 @@ type Config struct {
SignaturesFile string
SignaturesPath string
SignaturesURL string
SignaturesUserRepo string
SignaturesVersion string
SignaturesTest bool
Threads int
UserDirtyNames []string
UserDirtyOrgs []string
Expand Down Expand Up @@ -182,6 +179,8 @@ func Load(scanType api.ScanType) (*Config, error) {
c.GithubAccessToken = cfg.GetString("github-api-token")
c.SignaturesVersion = viper.GetString("signatures-version")
c.SignaturesURL = viper.GetString("signatures-url")
c.SignaturesUserRepo = viper.GetString("signatures-user-repo")
c.SignaturesTest = viper.GetBool("test-signatures")
}

// Add the default directories to the sess if they don't already exist
Expand Down
14 changes: 12 additions & 2 deletions internal/core/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ type CloneConfiguration struct {
Username string
Token string
Branch string
Tag bool
TagMode git.TagMode
Depth int
}

Expand Down Expand Up @@ -240,18 +242,26 @@ func cloneRepositoryFunc(sess *Session, repo _coreapi.Repository) (*git.Reposito

// cloneRepositoryGeneric will create either an in memory clone of a given repository or clone to a temp dir.
func CloneRepositoryGeneric(config CloneConfiguration, auth *http.BasicAuth) (repo *git.Repository, dir string, err error) {
ref := plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", config.Branch))
if config.Tag {
ref = plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", config.Branch))
}
cloneOptions := &git.CloneOptions{
URL: config.URL,
Depth: config.Depth,
ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", config.Branch)),
ReferenceName: ref,
SingleBranch: true,
Tags: git.NoTags,
Tags: config.TagMode,
}

if auth != nil {
cloneOptions.Auth = auth
}

if config.TagMode == git.InvalidTagMode {
cloneOptions.Tags = git.NoTags
}

if !config.InMemClone {
dir, err = os.MkdirTemp("", "rvsecret")
if err != nil {
Expand Down
18 changes: 18 additions & 0 deletions internal/core/provider/github/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,3 +222,21 @@ func (c *Client) GetOrganizationMembers(ctx context.Context, target _coreapi.Own

return allMembers, nil
}

func (c *Client) GetLatestRelease(ctx context.Context, owner, repo string) (*github.RepositoryRelease, error) {
release, _, err := c.apiClient.Repositories.GetLatestRelease(ctx, owner, repo)
if err != nil {
return nil, err
}

return release, nil
}

func (c *Client) GetReleaseByTag(ctx context.Context, owner, repo, tag string) (*github.RepositoryRelease, error) {
release, _, err := c.apiClient.Repositories.GetReleaseByTag(ctx, owner, repo, tag)
if err != nil {
return nil, err
}

return release, nil
}
60 changes: 60 additions & 0 deletions internal/pkg/signatures/fetch-git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package signatures

import (
"fmt"
"os"

"github.com/rumenvasilev/rvsecret/internal/core"
"gopkg.in/src-d/go-git.v4"
githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
)

// fetchSignaturesWithGit will clone the signatures repository and return the local path to it
func fetchSignaturesWithGit(version string, sess *core.Session) (string, error) {
branch := version
tag := true
if version == "latest" {
branch = "stable"
tag = false
}
err := sess.InitGitClient()
if err != nil {
return "", err
}

// build the URL
url := sess.Config.SignaturesURL
if sess.Config.SignaturesUserRepo != "" {
// TODO this address has to be a const perhaps?
url = fmt.Sprintf("https://github.com/%s", sess.Config.SignaturesUserRepo)
}
// sanitize checks
cURL, err := cleanInput(url)
if err != nil {
return "", err
}

cloneCfg := core.CloneConfiguration{
URL: cURL,
Branch: branch,
Depth: sess.Config.CommitDepth,
InMemClone: sess.Config.InMemClone,
Tag: tag,
// Should we?
TagMode: git.AllTags,
}
auth := &githttp.BasicAuth{
Username: "egal",
Password: sess.Config.GithubAccessToken,
}

// If we're gonna use git clone to get a specific tag, we need to pass git.AllTags as parameter here.
_, dir, err := core.CloneRepositoryGeneric(cloneCfg, auth)
if err != nil {
// cleanup dir
_ = os.RemoveAll(dir)
return "", err
}

return dir, nil
}
110 changes: 110 additions & 0 deletions internal/pkg/signatures/fetch-restapi.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package signatures

import (
"context"
"fmt"
"net/http"
"os"
"strings"

"github.com/google/go-github/github"
"github.com/rumenvasilev/rvsecret/internal/core"
_github "github.com/rumenvasilev/rvsecret/internal/core/provider/github"
"github.com/rumenvasilev/rvsecret/version"
)

// fetchSignaturesFromGithubAPI will only download a version of the signatures file from Github REST API
func fetchSignaturesFromGithubAPI(version string, sess *core.Session) (string, error) {
ctx := context.Background()
if sess.Config.SignaturesUserRepo == "" {
return "", fmt.Errorf("please provide -signatures-user-repo value")
}

res := strings.Split(sess.Config.SignaturesUserRepo, "/")
if len(res) != 2 {
return "", fmt.Errorf("user/repo doesn't have matching format, %s", sess.Config.SignaturesUserRepo)
}
owner := res[0]
repo := res[1]

client, err := _github.NewClient(sess.Config.GithubAccessToken, "", sess.Out)
if err != nil {
return "", fmt.Errorf("failed instantiation of Github client, %w", err)
}

var assets *github.RepositoryRelease
if version == "latest" {
assets, err = client.GetLatestRelease(ctx, owner, repo)
} else {
assets, err = client.GetReleaseByTag(ctx, owner, repo, version)
}
if err != nil {
// TODO: handle 404 not found
return "", fmt.Errorf("error while fetching release information, %w", err)
}

assetURL, err := getAssetURL(assets.Assets)
if err != nil {
return "", err
}

return downloadAsset(assetURL, sess)
}

func getAssetURL(assets []github.ReleaseAsset) (string, error) {
var download string
for _, v := range assets {
if v.GetName() == "default.yaml" {
download = v.GetURL()
break
}
}
if download == "" {
return "", fmt.Errorf("couldn't find the release asset default.yaml")
}
return download, nil
}

func downloadAsset(url string, sess *core.Session) (string, error) {
// Create tmp dir
path, err := os.MkdirTemp("", "rvsecret")
if err != nil {
return "", err
}

err = os.Mkdir(fmt.Sprintf("%s/signatures", path), 0700)
if err != nil {
return "", err
}

// fetch from URL
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", fmt.Sprintf("token %s", sess.Config.GithubAccessToken))
req.Header.Add("User-Agent", version.UserAgent)
req.Header.Add("Accept", "application/octet-stream")

// call github
c := http.Client{}
resp, err := c.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close() //nolint:errcheck

// store file
filename := fmt.Sprintf("%s/signatures/default.yaml", path)
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
defer f.Close() //nolint:staticcheck
if err != nil {
return "", err
}

b := make([]byte, 4096)
var i int
for err == nil {
i, err = resp.Body.Read(b)
f.Write(b[:i]) //nolint:errcheck
}

return path, nil
}
Loading

0 comments on commit 8ecc1f0

Please sign in to comment.