From cbe2f70b372b745e4ff90596a516927bfce19bce Mon Sep 17 00:00:00 2001 From: Michael Kedar Date: Tue, 15 Oct 2024 12:58:03 +1100 Subject: [PATCH] feat(guided remediation): support offline database in fix subcommand (#1306) #1295 Support the `--experimental-offline` and `--experimental-download-offline-databases` flags in `osv-scanner fix`. This is pretty inefficient since `local.MakeRequest()` reads and parses the zip file every time it's called. I didn't want to make too many changes to make it more efficient since we'll probably end up refactoring it in v2. Also fixed the links to headings on the offline docs page. --- cmd/osv-scanner/fix/main.go | 32 +++++- docs/guided-remediation.md | 6 + docs/offline-mode.md | 4 +- .../resolution/client/osv_offline_client.go | 105 ++++++++++++++++++ 4 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 internal/resolution/client/osv_offline_client.go diff --git a/cmd/osv-scanner/fix/main.go b/cmd/osv-scanner/fix/main.go index 01f33e9f66..b1e5236197 100644 --- a/cmd/osv-scanner/fix/main.go +++ b/cmd/osv-scanner/fix/main.go @@ -166,6 +166,20 @@ func Command(stdout, stderr io.Writer, r *reporter.Reporter) *cli.Command { Name: "maven-fix-management", Usage: "(pom.xml) also remediate vulnerabilities in dependencyManagement packages that do not appear in the resolved dependency graph", }, + // Offline database flags, copied from osv-scanner scan + &cli.BoolFlag{ + Name: "experimental-offline", + Usage: "checks for vulnerabilities using local databases that are already cached", + }, + &cli.BoolFlag{ + Name: "experimental-download-offline-databases", + Usage: "downloads vulnerability databases for offline comparison", + }, + &cli.StringFlag{ + Name: "experimental-local-db-path", + Usage: "sets the path that local databases should be stored", + Hidden: true, + }, }, Action: func(ctx *cli.Context) error { var err error @@ -252,13 +266,9 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro Manifest: ctx.String("manifest"), Lockfile: ctx.String("lockfile"), RelockCmd: ctx.String("relock-cmd"), - Client: client.ResolutionClient{ - VulnerabilityClient: client.NewOSVClient(), - }, } system := resolve.UnknownSystem - if opts.Lockfile != "" { rw, err := lockfile.GetReadWriter(opts.Lockfile) if err != nil { @@ -314,6 +324,20 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro } } + if ctx.Bool("experimental-offline") { + var err error + opts.Client.VulnerabilityClient, err = client.NewOSVOfflineClient( + r, + system, + ctx.Bool("experimental-download-offline-databases"), + ctx.String("experimental-local-db-path")) + if err != nil { + return nil, err + } + } else { + opts.Client.VulnerabilityClient = client.NewOSVClient() + } + if !ctx.Bool("non-interactive") { return nil, interactiveMode(ctx.Context, opts) } diff --git a/docs/guided-remediation.md b/docs/guided-remediation.md index 6dae1a4cf4..858c791ee6 100644 --- a/docs/guided-remediation.md +++ b/docs/guided-remediation.md @@ -262,6 +262,12 @@ If your project uses mirrored or private registries, you will need to use `--dat > > The native npm cache will store the addresses of private registries used, though not any authentication information. +### Offline Vulnerability Database + +The `fix` subcommand supports the `--experimental-offline` and `--experimental-download-offline-databases` flags. + +For more information, see [Offline Mode](./offline-mode.md). + ## Known issues - The subcommand does not use the `osv-scanner.toml` configuration. Use the `--ignore-vulns` flag instead. diff --git a/docs/offline-mode.md b/docs/offline-mode.md index 56bbc5b159..46efa171ec 100644 --- a/docs/offline-mode.md +++ b/docs/offline-mode.md @@ -49,7 +49,7 @@ If the `OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY` environment variable is _not_ set, 1. The location returned by [`os.UserCacheDir`](https://pkg.go.dev/os#UserCacheDir) 2. The location returned by [`os.TempDir`](https://pkg.go.dev/os#TempDir) -The database can be [downloaded manually](./experimental.md#manual-database-download) or by using the [`--experimental-download-offline-databases` flag](./experimental.md#download-databases-option). +The database can be [downloaded manually](#manual-database-download) or by using the [`--experimental-download-offline-databases` flag](#download-offline-databases-option). ## Offline option @@ -88,7 +88,7 @@ You can also download over HTTP via `https://osv-vulnerabilities.storage.googlea A list of all current ecosystems is available at [`gs://osv-vulnerabilities/ecosystems.txt`](https://osv-vulnerabilities.storage.googleapis.com/ecosystems.txt). -Set the location of your manually downloaded database by following the instructions [here](./experimental.md#specify-database-location). +Set the location of your manually downloaded database by following the instructions [here](#specify-database-location). ## Limitations diff --git a/internal/resolution/client/osv_offline_client.go b/internal/resolution/client/osv_offline_client.go new file mode 100644 index 0000000000..0779ac0c81 --- /dev/null +++ b/internal/resolution/client/osv_offline_client.go @@ -0,0 +1,105 @@ +package client + +import ( + "errors" + "fmt" + "strings" + + "deps.dev/util/resolve" + "github.com/google/osv-scanner/internal/local" + "github.com/google/osv-scanner/internal/resolution/util" + "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/pkg/osv" + "github.com/google/osv-scanner/pkg/reporter" +) + +type OSVOfflineClient struct { + // TODO: OSV-Scanner v2 plans to make vulnerability clients that can be used here. + localDBPath string +} + +func NewOSVOfflineClient(r reporter.Reporter, system resolve.System, downloadDBs bool, localDBPath string) (*OSVOfflineClient, error) { + if system == resolve.UnknownSystem { + return nil, errors.New("osv offline client created with unknown ecosystem") + } + // Make a dummy request to the local client to log and make sure the database is downloaded without error. + q := osv.BatchedQuery{Queries: []*osv.Query{{ + Package: osv.Package{ + Name: "foo", + Ecosystem: string(util.OSVEcosystem[system]), + }, + Version: "1.0.0", + }}} + _, err := local.MakeRequest(r, q, !downloadDBs, localDBPath) + if err != nil { + return nil, err + } + + if r.HasErrored() { + return nil, errors.New("error creating osv offline client") + } + + return &OSVOfflineClient{localDBPath: localDBPath}, nil +} + +func (c *OSVOfflineClient) FindVulns(g *resolve.Graph) ([]models.Vulnerabilities, error) { + var query osv.BatchedQuery + query.Queries = make([]*osv.Query, len(g.Nodes)-1) + for i, node := range g.Nodes[1:] { + query.Queries[i] = &osv.Query{ + Package: osv.Package{ + Name: node.Version.Name, + Ecosystem: string(util.OSVEcosystem[node.Version.System]), + }, + Version: node.Version.Version, + } + } + + // If local.MakeRequest logs an error, it's probably fatal for guided remediation. + // Set up a reporter to capture error logs and return the logs as an error. + r := &errorReporter{} + // DB should already be downloaded, set offline to true. + hydrated, err := local.MakeRequest(r, query, true, c.localDBPath) + + if err != nil { + return nil, err + } + + if r.HasErrored() { + return nil, r.GetError() + } + + nodeVulns := make([]models.Vulnerabilities, len(g.Nodes)) + for i, res := range hydrated.Results { + nodeVulns[i+1] = res.Vulns + } + + return nodeVulns, nil +} + +// errorReporter is a reporter.Reporter to capture error logs and pack them into an error. +type errorReporter struct { + s strings.Builder +} + +func (r *errorReporter) Errorf(format string, a ...any) { + fmt.Fprintf(&r.s, format, a...) +} + +func (r *errorReporter) HasErrored() bool { + return r.s.Len() > 0 +} + +func (r *errorReporter) GetError() error { + str := strings.TrimSpace(r.s.String()) + if str == "" { + return nil + } + + return errors.New(str) +} + +func (r *errorReporter) Warnf(string, ...any) {} +func (r *errorReporter) Infof(string, ...any) {} +func (r *errorReporter) Verbosef(string, ...any) {} +func (r *errorReporter) PrintResult(*models.VulnerabilityResults) error { return nil }