feat: deprecate axillary public packages in favor of private versions (…

…#1309)

As part of the upcoming v2 release we're planning on making as many of
our currently-public packages that we can private, in order to make it
easier for us to maintain them since most are "axillary" packages that
only expect to be used by the scanner, and that it's not a breaking
change to make particular packages public again in future if that does
prove useful.

While we cannot remove the existing public packages until v2 is
released, we can still create the private versions and switch over to
using them - this means that we can start making what'd be breaking
changes to their interfaces, and also gives people a chance to report
their usage of a particular package due to them now being marked as
deprecated.

Of our currently public packages, these are being made private and
marked as deprecated:
- `config`
- `depsdev`
- `grouper`
- `spdx`

Of the others:
- `lockfile` will be replaced with
https://github.com/google/osv-scalibr, but we won't be marking it as
deprecated until @another-rex has finished migrating the implementations
here over to that library
- `models` is going to be broken up, with at least the `constants` being
moved to a dedicated package in https://github.com/ossf/osv-schema and
at least some of the other content moved "elsewhere" (though the exact
fate of everything is yet to be determined)
- `osvscanner` is purposely public and will remain that way, which also
necessitates `reporter` being public since it's a parameter to the API
functions so it'd be burdensome for consumers to have it private
- `osv` is used by `osvscanner`, but technically might not need to
public; but that is still I think something we're interested in keeping
public at some point - I believe there are talks of oneday maybe housing
it in https://github.com/google/osv.dev, but for now it can remain
public too

Note this is focused on just "moving" the existing packages - there are
already a few functions in e.g. `config` that are marked as deprecated,
which I'll remove in a follow-up pull request to make it easier to
review.

---------

Co-authored-by: Rex P <[email protected]>

cmd/osv-scanner/fix/main.go

@@ -9,14 +9,14 @@ import (
 	"strings"
 
 	"deps.dev/util/resolve"
+	"github.com/google/osv-scanner/internal/depsdev"
 	"github.com/google/osv-scanner/internal/remediation"
 	"github.com/google/osv-scanner/internal/remediation/upgrade"
 	"github.com/google/osv-scanner/internal/resolution"
 	"github.com/google/osv-scanner/internal/resolution/client"
 	"github.com/google/osv-scanner/internal/resolution/datasource"
 	"github.com/google/osv-scanner/internal/resolution/lockfile"
 	"github.com/google/osv-scanner/internal/resolution/manifest"
-	"github.com/google/osv-scanner/pkg/depsdev"
 	"github.com/google/osv-scanner/pkg/reporter"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/term"

cmd/osv-scanner/scan/main.go

@@ -8,9 +8,9 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/google/osv-scanner/internal/spdx"
 	"github.com/google/osv-scanner/pkg/osvscanner"
 	"github.com/google/osv-scanner/pkg/reporter"
-	"github.com/google/osv-scanner/pkg/spdx"
 	"golang.org/x/term"
 
 	"github.com/urfave/cli/v2"

cmd/osv-scanner/update/main.go

@@ -6,10 +6,10 @@ import (
 	"io"
 	"os"
 
+	"github.com/google/osv-scanner/internal/depsdev"
 	"github.com/google/osv-scanner/internal/remediation/suggest"
 	"github.com/google/osv-scanner/internal/resolution/client"
 	"github.com/google/osv-scanner/internal/resolution/manifest"
-	"github.com/google/osv-scanner/pkg/depsdev"
 	"github.com/google/osv-scanner/pkg/lockfile"
 	"github.com/google/osv-scanner/pkg/reporter"
 	"github.com/urfave/cli/v2"

internal/ci/vulnerability_result_diff.go

@@ -1,8 +1,8 @@
 package ci
 
 import (
+	"github.com/google/osv-scanner/internal/grouper"
 	"github.com/google/osv-scanner/internal/output"
-	"github.com/google/osv-scanner/pkg/grouper"
 	"github.com/google/osv-scanner/pkg/models"
 )
 

internal/config/config.go

@@ -0,0 +1,251 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/google/osv-scanner/pkg/models"
+	"github.com/google/osv-scanner/pkg/reporter"
+)
+
+const osvScannerConfigName = "osv-scanner.toml"
+
+// Ignore stuttering as that would be a breaking change
+// TODO: V2 rename?
+//
+//nolint:revive
+type ConfigManager struct {
+	// Override to replace all other configs
+	OverrideConfig *Config
+	// Config to use if no config file is found alongside manifests
+	DefaultConfig Config
+	// Cache to store loaded configs
+	ConfigMap map[string]Config
+}
+
+type Config struct {
+	IgnoredVulns      []IgnoreEntry          `toml:"IgnoredVulns"`
+	PackageOverrides  []PackageOverrideEntry `toml:"PackageOverrides"`
+	GoVersionOverride string                 `toml:"GoVersionOverride"`
+	// The path to config file that this config was loaded from,
+	// set by the scanner after having successfully parsed the file
+	LoadPath string `toml:"-"`
+}
+
+type IgnoreEntry struct {
+	ID          string    `toml:"id"`
+	IgnoreUntil time.Time `toml:"ignoreUntil"`
+	Reason      string    `toml:"reason"`
+}
+
+type PackageOverrideEntry struct {
+	Name string `toml:"name"`
+	// If the version is empty, the entry applies to all versions.
+	Version        string        `toml:"version"`
+	Ecosystem      string        `toml:"ecosystem"`
+	Group          string        `toml:"group"`
+	Ignore         bool          `toml:"ignore"`
+	Vulnerability  Vulnerability `toml:"vulnerability"`
+	License        License       `toml:"license"`
+	EffectiveUntil time.Time     `toml:"effectiveUntil"`
+	Reason         string        `toml:"reason"`
+}
+
+func (e PackageOverrideEntry) matches(pkg models.PackageVulns) bool {
+	if e.Name != "" && e.Name != pkg.Package.Name {
+		return false
+	}
+	if e.Version != "" && e.Version != pkg.Package.Version {
+		return false
+	}
+	if e.Ecosystem != "" && e.Ecosystem != pkg.Package.Ecosystem {
+		return false
+	}
+	if e.Group != "" && !slices.Contains(pkg.DepGroups, e.Group) {
+		return false
+	}
+
+	return true
+}
+
+type Vulnerability struct {
+	Ignore bool `toml:"ignore"`
+}
+
+type License struct {
+	Override []string `toml:"override"`
+	Ignore   bool     `toml:"ignore"`
+}
+
+func (c *Config) ShouldIgnore(vulnID string) (bool, IgnoreEntry) {
+	index := slices.IndexFunc(c.IgnoredVulns, func(e IgnoreEntry) bool { return e.ID == vulnID })
+	if index == -1 {
+		return false, IgnoreEntry{}
+	}
+	ignoredLine := c.IgnoredVulns[index]
+
+	return shouldIgnoreTimestamp(ignoredLine.IgnoreUntil), ignoredLine
+}
+
+func (c *Config) filterPackageVersionEntries(pkg models.PackageVulns, condition func(PackageOverrideEntry) bool) (bool, PackageOverrideEntry) {
+	index := slices.IndexFunc(c.PackageOverrides, func(e PackageOverrideEntry) bool {
+		return e.matches(pkg) && condition(e)
+	})
+	if index == -1 {
+		return false, PackageOverrideEntry{}
+	}
+	ignoredLine := c.PackageOverrides[index]
+
+	return shouldIgnoreTimestamp(ignoredLine.EffectiveUntil), ignoredLine
+}
+
+// ShouldIgnorePackage determines if the given package should be ignored based on override entries in the config
+func (c *Config) ShouldIgnorePackage(pkg models.PackageVulns) (bool, PackageOverrideEntry) {
+	return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool {
+		return e.Ignore
+	})
+}
+
+// Deprecated: Use ShouldIgnorePackage instead
+func (c *Config) ShouldIgnorePackageVersion(name, version, ecosystem string) (bool, PackageOverrideEntry) {
+	return c.ShouldIgnorePackage(models.PackageVulns{
+		Package: models.PackageInfo{
+			Name:      name,
+			Version:   version,
+			Ecosystem: ecosystem,
+		},
+	})
+}
+
+// ShouldIgnorePackageVulnerabilities determines if the given package should have its vulnerabilities ignored based on override entries in the config
+func (c *Config) ShouldIgnorePackageVulnerabilities(pkg models.PackageVulns) bool {
+	overrides, _ := c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool {
+		return e.Vulnerability.Ignore
+	})
+
+	return overrides
+}
+
+// ShouldOverridePackageLicense determines if the given package should have its license ignored or changed based on override entries in the config
+func (c *Config) ShouldOverridePackageLicense(pkg models.PackageVulns) (bool, PackageOverrideEntry) {
+	return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool {
+		return e.License.Ignore || len(e.License.Override) > 0
+	})
+}
+
+// Deprecated: Use ShouldOverridePackageLicense instead
+func (c *Config) ShouldOverridePackageVersionLicense(name, version, ecosystem string) (bool, PackageOverrideEntry) {
+	return c.ShouldOverridePackageLicense(models.PackageVulns{
+		Package: models.PackageInfo{
+			Name:      name,
+			Version:   version,
+			Ecosystem: ecosystem,
+		},
+	})
+}
+
+func shouldIgnoreTimestamp(ignoreUntil time.Time) bool {
+	if ignoreUntil.IsZero() {
+		// If IgnoreUntil is not set, should ignore.
+		return true
+	}
+	// Should ignore if IgnoreUntil is still after current time
+	// Takes timezone offsets into account if it is specified. otherwise it's using local time
+	return ignoreUntil.After(time.Now())
+}
+
+// Sets the override config by reading the config file at configPath.
+// Will return an error if loading the config file fails
+func (c *ConfigManager) UseOverride(configPath string) error {
+	config, configErr := tryLoadConfig(configPath)
+	if configErr != nil {
+		return configErr
+	}
+	c.OverrideConfig = &config
+
+	return nil
+}
+
+// Attempts to get the config
+func (c *ConfigManager) Get(r reporter.Reporter, targetPath string) Config {
+	if c.OverrideConfig != nil {
+		return *c.OverrideConfig
+	}
+
+	configPath, err := normalizeConfigLoadPath(targetPath)
+	if err != nil {
+		// TODO: This can happen when target is not a file (e.g. Docker container, git hash...etc.)
+		// Figure out a more robust way to load config from non files
+		// r.PrintErrorf("Can't find config path: %s\n", err)
+		return Config{}
+	}
+
+	config, alreadyExists := c.ConfigMap[configPath]
+	if alreadyExists {
+		return config
+	}
+
+	config, configErr := tryLoadConfig(configPath)
+	if configErr == nil {
+		r.Infof("Loaded filter from: %s\n", config.LoadPath)
+	} else {
+		// anything other than the config file not existing is most likely due to an invalid config file
+		if !errors.Is(configErr, os.ErrNotExist) {
+			r.Errorf("Ignored invalid config file at: %s\n", configPath)
+			r.Verbosef("Config file %s is invalid because: %v\n", configPath, configErr)
+		}
+		// If config doesn't exist, use the default config
+		config = c.DefaultConfig
+	}
+	c.ConfigMap[configPath] = config
+
+	return config
+}
+
+// Finds the containing folder of `target`, then appends osvScannerConfigName
+func normalizeConfigLoadPath(target string) (string, error) {
+	stat, err := os.Stat(target)
+	if err != nil {
+		return "", fmt.Errorf("failed to stat target: %w", err)
+	}
+
+	var containingFolder string
+	if !stat.IsDir() {
+		containingFolder = filepath.Dir(target)
+	} else {
+		containingFolder = target
+	}
+	configPath := filepath.Join(containingFolder, osvScannerConfigName)
+
+	return configPath, nil
+}
+
+// tryLoadConfig attempts to parse the config file at the given path as TOML,
+// returning the Config object if successful or otherwise the error
+func tryLoadConfig(configPath string) (Config, error) {
+	config := Config{}
+	m, err := toml.DecodeFile(configPath, &config)
+	if err == nil {
+		unknownKeys := m.Undecoded()
+
+		if len(unknownKeys) > 0 {
+			keys := make([]string, 0, len(unknownKeys))
+
+			for _, key := range unknownKeys {
+				keys = append(keys, key.String())
+			}
+
+			return Config{}, fmt.Errorf("unknown keys in config file: %s", strings.Join(keys, ", "))
+		}
+
+		config.LoadPath = configPath
+	}
+
+	return config, err
+}