diff --git a/cmd/osv-scanner/fix/main.go b/cmd/osv-scanner/fix/main.go index b1e5236197..f22e6009b2 100644 --- a/cmd/osv-scanner/fix/main.go +++ b/cmd/osv-scanner/fix/main.go @@ -9,6 +9,7 @@ import ( "strings" "deps.dev/util/resolve" + "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/internal/remediation" "github.com/google/osv-scanner/internal/remediation/upgrade" "github.com/google/osv-scanner/internal/resolution" @@ -16,7 +17,6 @@ import ( "github.com/google/osv-scanner/internal/resolution/datasource" "github.com/google/osv-scanner/internal/resolution/lockfile" "github.com/google/osv-scanner/internal/resolution/manifest" - "github.com/google/osv-scanner/pkg/depsdev" "github.com/google/osv-scanner/pkg/reporter" "github.com/urfave/cli/v2" "golang.org/x/term" diff --git a/cmd/osv-scanner/scan/main.go b/cmd/osv-scanner/scan/main.go index fc2efa74b8..9fa3cc3a81 100644 --- a/cmd/osv-scanner/scan/main.go +++ b/cmd/osv-scanner/scan/main.go @@ -8,9 +8,9 @@ import ( "slices" "strings" + "github.com/google/osv-scanner/internal/spdx" "github.com/google/osv-scanner/pkg/osvscanner" "github.com/google/osv-scanner/pkg/reporter" - "github.com/google/osv-scanner/pkg/spdx" "golang.org/x/term" "github.com/urfave/cli/v2" diff --git a/cmd/osv-scanner/update/main.go b/cmd/osv-scanner/update/main.go index 28203b3998..4689f40eef 100644 --- a/cmd/osv-scanner/update/main.go +++ b/cmd/osv-scanner/update/main.go @@ -6,10 +6,10 @@ import ( "io" "os" + "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/internal/remediation/suggest" "github.com/google/osv-scanner/internal/resolution/client" "github.com/google/osv-scanner/internal/resolution/manifest" - "github.com/google/osv-scanner/pkg/depsdev" "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/reporter" "github.com/urfave/cli/v2" diff --git a/internal/ci/vulnerability_result_diff.go b/internal/ci/vulnerability_result_diff.go index bd260acde4..77436e400a 100644 --- a/internal/ci/vulnerability_result_diff.go +++ b/internal/ci/vulnerability_result_diff.go @@ -1,8 +1,8 @@ package ci import ( + "github.com/google/osv-scanner/internal/grouper" "github.com/google/osv-scanner/internal/output" - "github.com/google/osv-scanner/pkg/grouper" "github.com/google/osv-scanner/pkg/models" ) diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000000..d1b8b86b0c --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,251 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/BurntSushi/toml" + "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/pkg/reporter" +) + +const osvScannerConfigName = "osv-scanner.toml" + +// Ignore stuttering as that would be a breaking change +// TODO: V2 rename? +// +//nolint:revive +type ConfigManager struct { + // Override to replace all other configs + OverrideConfig *Config + // Config to use if no config file is found alongside manifests + DefaultConfig Config + // Cache to store loaded configs + ConfigMap map[string]Config +} + +type Config struct { + IgnoredVulns []IgnoreEntry `toml:"IgnoredVulns"` + PackageOverrides []PackageOverrideEntry `toml:"PackageOverrides"` + GoVersionOverride string `toml:"GoVersionOverride"` + // The path to config file that this config was loaded from, + // set by the scanner after having successfully parsed the file + LoadPath string `toml:"-"` +} + +type IgnoreEntry struct { + ID string `toml:"id"` + IgnoreUntil time.Time `toml:"ignoreUntil"` + Reason string `toml:"reason"` +} + +type PackageOverrideEntry struct { + Name string `toml:"name"` + // If the version is empty, the entry applies to all versions. + Version string `toml:"version"` + Ecosystem string `toml:"ecosystem"` + Group string `toml:"group"` + Ignore bool `toml:"ignore"` + Vulnerability Vulnerability `toml:"vulnerability"` + License License `toml:"license"` + EffectiveUntil time.Time `toml:"effectiveUntil"` + Reason string `toml:"reason"` +} + +func (e PackageOverrideEntry) matches(pkg models.PackageVulns) bool { + if e.Name != "" && e.Name != pkg.Package.Name { + return false + } + if e.Version != "" && e.Version != pkg.Package.Version { + return false + } + if e.Ecosystem != "" && e.Ecosystem != pkg.Package.Ecosystem { + return false + } + if e.Group != "" && !slices.Contains(pkg.DepGroups, e.Group) { + return false + } + + return true +} + +type Vulnerability struct { + Ignore bool `toml:"ignore"` +} + +type License struct { + Override []string `toml:"override"` + Ignore bool `toml:"ignore"` +} + +func (c *Config) ShouldIgnore(vulnID string) (bool, IgnoreEntry) { + index := slices.IndexFunc(c.IgnoredVulns, func(e IgnoreEntry) bool { return e.ID == vulnID }) + if index == -1 { + return false, IgnoreEntry{} + } + ignoredLine := c.IgnoredVulns[index] + + return shouldIgnoreTimestamp(ignoredLine.IgnoreUntil), ignoredLine +} + +func (c *Config) filterPackageVersionEntries(pkg models.PackageVulns, condition func(PackageOverrideEntry) bool) (bool, PackageOverrideEntry) { + index := slices.IndexFunc(c.PackageOverrides, func(e PackageOverrideEntry) bool { + return e.matches(pkg) && condition(e) + }) + if index == -1 { + return false, PackageOverrideEntry{} + } + ignoredLine := c.PackageOverrides[index] + + return shouldIgnoreTimestamp(ignoredLine.EffectiveUntil), ignoredLine +} + +// ShouldIgnorePackage determines if the given package should be ignored based on override entries in the config +func (c *Config) ShouldIgnorePackage(pkg models.PackageVulns) (bool, PackageOverrideEntry) { + return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { + return e.Ignore + }) +} + +// Deprecated: Use ShouldIgnorePackage instead +func (c *Config) ShouldIgnorePackageVersion(name, version, ecosystem string) (bool, PackageOverrideEntry) { + return c.ShouldIgnorePackage(models.PackageVulns{ + Package: models.PackageInfo{ + Name: name, + Version: version, + Ecosystem: ecosystem, + }, + }) +} + +// ShouldIgnorePackageVulnerabilities determines if the given package should have its vulnerabilities ignored based on override entries in the config +func (c *Config) ShouldIgnorePackageVulnerabilities(pkg models.PackageVulns) bool { + overrides, _ := c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { + return e.Vulnerability.Ignore + }) + + return overrides +} + +// ShouldOverridePackageLicense determines if the given package should have its license ignored or changed based on override entries in the config +func (c *Config) ShouldOverridePackageLicense(pkg models.PackageVulns) (bool, PackageOverrideEntry) { + return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { + return e.License.Ignore || len(e.License.Override) > 0 + }) +} + +// Deprecated: Use ShouldOverridePackageLicense instead +func (c *Config) ShouldOverridePackageVersionLicense(name, version, ecosystem string) (bool, PackageOverrideEntry) { + return c.ShouldOverridePackageLicense(models.PackageVulns{ + Package: models.PackageInfo{ + Name: name, + Version: version, + Ecosystem: ecosystem, + }, + }) +} + +func shouldIgnoreTimestamp(ignoreUntil time.Time) bool { + if ignoreUntil.IsZero() { + // If IgnoreUntil is not set, should ignore. + return true + } + // Should ignore if IgnoreUntil is still after current time + // Takes timezone offsets into account if it is specified. otherwise it's using local time + return ignoreUntil.After(time.Now()) +} + +// Sets the override config by reading the config file at configPath. +// Will return an error if loading the config file fails +func (c *ConfigManager) UseOverride(configPath string) error { + config, configErr := tryLoadConfig(configPath) + if configErr != nil { + return configErr + } + c.OverrideConfig = &config + + return nil +} + +// Attempts to get the config +func (c *ConfigManager) Get(r reporter.Reporter, targetPath string) Config { + if c.OverrideConfig != nil { + return *c.OverrideConfig + } + + configPath, err := normalizeConfigLoadPath(targetPath) + if err != nil { + // TODO: This can happen when target is not a file (e.g. Docker container, git hash...etc.) + // Figure out a more robust way to load config from non files + // r.PrintErrorf("Can't find config path: %s\n", err) + return Config{} + } + + config, alreadyExists := c.ConfigMap[configPath] + if alreadyExists { + return config + } + + config, configErr := tryLoadConfig(configPath) + if configErr == nil { + r.Infof("Loaded filter from: %s\n", config.LoadPath) + } else { + // anything other than the config file not existing is most likely due to an invalid config file + if !errors.Is(configErr, os.ErrNotExist) { + r.Errorf("Ignored invalid config file at: %s\n", configPath) + r.Verbosef("Config file %s is invalid because: %v\n", configPath, configErr) + } + // If config doesn't exist, use the default config + config = c.DefaultConfig + } + c.ConfigMap[configPath] = config + + return config +} + +// Finds the containing folder of `target`, then appends osvScannerConfigName +func normalizeConfigLoadPath(target string) (string, error) { + stat, err := os.Stat(target) + if err != nil { + return "", fmt.Errorf("failed to stat target: %w", err) + } + + var containingFolder string + if !stat.IsDir() { + containingFolder = filepath.Dir(target) + } else { + containingFolder = target + } + configPath := filepath.Join(containingFolder, osvScannerConfigName) + + return configPath, nil +} + +// tryLoadConfig attempts to parse the config file at the given path as TOML, +// returning the Config object if successful or otherwise the error +func tryLoadConfig(configPath string) (Config, error) { + config := Config{} + m, err := toml.DecodeFile(configPath, &config) + if err == nil { + unknownKeys := m.Undecoded() + + if len(unknownKeys) > 0 { + keys := make([]string, 0, len(unknownKeys)) + + for _, key := range unknownKeys { + keys = append(keys, key.String()) + } + + return Config{}, fmt.Errorf("unknown keys in config file: %s", strings.Join(keys, ", ")) + } + + config.LoadPath = configPath + } + + return config, err +} diff --git a/internal/config/config_internal_test.go b/internal/config/config_internal_test.go new file mode 100644 index 0000000000..2336c2ae23 --- /dev/null +++ b/internal/config/config_internal_test.go @@ -0,0 +1,1320 @@ +package config + +import ( + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/osv-scanner/pkg/models" +) + +// Attempts to normalize any file paths in the given `output` so that they can +// be compared reliably regardless of the file path separator being used. +// +// Namely, escaped forward slashes are replaced with backslashes. +func normalizeFilePaths(t *testing.T, output string) string { + t.Helper() + + return strings.ReplaceAll(strings.ReplaceAll(output, "\\\\", "/"), "\\", "/") +} + +func Test_normalizeConfigLoadPath(t *testing.T) { + t.Parallel() + + type args struct { + target string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "target does not exist", + args: args{ + target: "./fixtures/testdatainner/does-not-exist", + }, + want: "", + wantErr: true, + }, + { + name: "target is file in directory", + args: args{ + target: "./fixtures/testdatainner/innerFolder/test.yaml", + }, + want: "fixtures/testdatainner/innerFolder/osv-scanner.toml", + wantErr: false, + }, + { + name: "target is inner directory with trailing slash", + args: args{ + target: "./fixtures/testdatainner/innerFolder/", + }, + want: "fixtures/testdatainner/innerFolder/osv-scanner.toml", + wantErr: false, + }, + { + name: "target is inner directory without trailing slash", + args: args{ + target: "./fixtures/testdatainner/innerFolder", + }, + want: "fixtures/testdatainner/innerFolder/osv-scanner.toml", + wantErr: false, + }, + { + name: "target is directory with trailing slash", + args: args{ + target: "./fixtures/testdatainner/", + }, + want: "fixtures/testdatainner/osv-scanner.toml", + wantErr: false, + }, + { + name: "target is file in directory", + args: args{ + target: "./fixtures/testdatainner/some-manifest.yaml", + }, + want: "fixtures/testdatainner/osv-scanner.toml", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := normalizeConfigLoadPath(tt.args.target) + if (err != nil) != tt.wantErr { + t.Errorf("normalizeConfigLoadPath() error = %v, wantErr %v", err, tt.wantErr) + return + } + + got = normalizeFilePaths(t, got) + if got != tt.want { + t.Errorf("normalizeConfigLoadPath() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_tryLoadConfig(t *testing.T) { + t.Parallel() + + type args struct { + configPath string + } + tests := []struct { + name string + args args + want Config + wantErr bool + }{ + { + name: "config does not exist", + args: args{ + configPath: "./fixtures/testdatainner/does-not-exist", + }, + want: Config{}, + wantErr: true, + }, + { + name: "config has some ignored vulnerabilities and package overrides", + args: args{ + configPath: "./fixtures/testdatainner/osv-scanner.toml", + }, + want: Config{ + LoadPath: "./fixtures/testdatainner/osv-scanner.toml", + IgnoredVulns: []IgnoreEntry{ + { + ID: "GO-2022-0968", + }, + { + ID: "GO-2022-1059", + }, + }, + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib", + Version: "1.0.0", + Ecosystem: "Go", + Ignore: true, + Reason: "abc", + }, + { + Name: "my-pkg", + Version: "1.0.0", + Ecosystem: "Go", + Reason: "abc", + Ignore: true, + License: License{ + Override: []string{"MIT", "0BSD"}, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "load path cannot be overridden via config", + args: args{ + configPath: "./fixtures/testdatainner/osv-scanner-load-path.toml", + }, + want: Config{ + LoadPath: "", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := tryLoadConfig(tt.args.configPath) + if (err != nil) != tt.wantErr { + t.Errorf("tryLoadConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("tryLoadConfig() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestTryLoadConfig_UnknownKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + configPath string + unknownMsg string + }{ + { + configPath: "./fixtures/unknown-key-1.toml", + unknownMsg: "IgnoredVulns.ignoreUntilTime", + }, + { + configPath: "./fixtures/unknown-key-2.toml", + unknownMsg: "IgnoredVulns.ignoreUntiI", + }, + { + configPath: "./fixtures/unknown-key-3.toml", + unknownMsg: "IgnoredVulns.reasoning", + }, + { + configPath: "./fixtures/unknown-key-4.toml", + unknownMsg: "PackageOverrides.skip", + }, + { + configPath: "./fixtures/unknown-key-5.toml", + unknownMsg: "PackageOverrides.license.skip", + }, + { + configPath: "./fixtures/unknown-key-6.toml", + unknownMsg: "RustVersionOverride", + }, + { + configPath: "./fixtures/unknown-key-7.toml", + unknownMsg: "RustVersionOverride, PackageOverrides.skip", + }, + } + + for _, testData := range tests { + c, err := tryLoadConfig(testData.configPath) + + // we should always be returning an empty config on error + if diff := cmp.Diff(Config{}, c); diff != "" { + t.Errorf("tryLoadConfig() mismatch (-want +got):\n%s", diff) + } + if err == nil { + t.Fatal("tryLoadConfig() did not return an error") + } + + wantMsg := fmt.Sprintf("unknown keys in config file: %v", testData.unknownMsg) + + if err.Error() != wantMsg { + t.Errorf("tryLoadConfig() error = '%v', want '%s'", err, wantMsg) + } + } +} + +func TestConfig_ShouldIgnore(t *testing.T) { + t.Parallel() + + type args struct { + vulnID string + } + tests := []struct { + name string + config Config + args args + wantOk bool + wantEntry IgnoreEntry + }{ + // entry exists + { + name: "", + config: Config{ + IgnoredVulns: []IgnoreEntry{ + { + ID: "GHSA-123", + IgnoreUntil: time.Time{}, + Reason: "", + }, + }, + }, + args: args{ + vulnID: "GHSA-123", + }, + wantOk: true, + wantEntry: IgnoreEntry{ + ID: "GHSA-123", + IgnoreUntil: time.Time{}, + Reason: "", + }, + }, + // entry does not exist + { + name: "", + config: Config{ + IgnoredVulns: []IgnoreEntry{ + { + ID: "GHSA-123", + IgnoreUntil: time.Time{}, + Reason: "", + }, + }, + }, + args: args{ + vulnID: "nonexistent", + }, + wantOk: false, + wantEntry: IgnoreEntry{}, + }, + // ignored until a time in the past + { + name: "", + config: Config{ + IgnoredVulns: []IgnoreEntry{ + { + ID: "GHSA-123", + IgnoreUntil: time.Now().Add(-time.Hour).Round(time.Second), + Reason: "", + }, + }, + }, + args: args{ + vulnID: "GHSA-123", + }, + wantOk: false, + wantEntry: IgnoreEntry{ + ID: "GHSA-123", + IgnoreUntil: time.Now().Add(-time.Hour).Round(time.Second), + Reason: "", + }, + }, + // ignored until a time in the future + { + name: "", + config: Config{ + IgnoredVulns: []IgnoreEntry{ + { + ID: "GHSA-123", + IgnoreUntil: time.Now().Add(time.Hour).Round(time.Second), + Reason: "", + }, + }, + }, + args: args{ + vulnID: "GHSA-123", + }, + wantOk: true, + wantEntry: IgnoreEntry{ + ID: "GHSA-123", + IgnoreUntil: time.Now().Add(time.Hour).Round(time.Second), + Reason: "", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotOk, gotEntry := tt.config.ShouldIgnore(tt.args.vulnID) + if gotOk != tt.wantOk { + t.Errorf("ShouldIgnore() gotOk = %v, wantOk %v", gotOk, tt.wantOk) + } + if !reflect.DeepEqual(gotEntry, tt.wantEntry) { + t.Errorf("ShouldIgnore() gotEntry = %v, wantEntry %v", gotEntry, tt.wantEntry) + } + }) + } +} + +func TestConfig_ShouldIgnorePackage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config Config + args models.PackageVulns + wantOk bool + wantEntry PackageOverrideEntry + }{ + { + name: "Everything-level entry exists", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + // ------------------------------------------------------------------------- + { + name: "Ecosystem-level entry exists and does match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Ecosystem-level entry exists and does not match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: "npm", + }, + DepGroups: []string{"dev"}, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + // ------------------------------------------------------------------------- + { + name: "Group-level entry exists and does match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Group-level entry exists and does not match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: "npm", + }, + DepGroups: []string{"optional"}, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + { + name: "Group-level entry exists and does not match when empty", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: "npm", + }, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + // ------------------------------------------------------------------------- + { + name: "Version-level entry exists and does match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Version: "1.0.0", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Version: "1.0.0", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Version-level entry exists and does not match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Version: "1.0.0", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + // ------------------------------------------------------------------------- + { + name: "Name-level entry exists and does match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Name-level entry exists and does not match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib2", + Version: "1.0.0", + Ecosystem: "npm", + }, + DepGroups: []string{"dev"}, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + // ------------------------------------------------------------------------- + { + name: "Name, Version, and Ecosystem entry exists", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Name and Ecosystem entry exists", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Name, Ecosystem, and Group entry exists and matches", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"dev"}, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ecosystem: "Go", + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Name, Ecosystem, and Group entry exists but does not match", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + Group: "dev", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + DepGroups: []string{"prod"}, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + { + name: "Entry doesn't exist", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "2.0.0", + Ecosystem: "Go", + Ignore: false, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + { + Name: "lib2", + Version: "2.0.0", + Ignore: true, + Ecosystem: "Go", + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "2.0.0", + Ecosystem: "Go", + }, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotOk, gotEntry := tt.config.ShouldIgnorePackage(tt.args) + if gotOk != tt.wantOk { + t.Errorf("ShouldIgnorePackage() gotOk = %v, wantOk %v", gotOk, tt.wantOk) + } + if !reflect.DeepEqual(gotEntry, tt.wantEntry) { + t.Errorf("ShouldIgnorePackage() gotEntry = %v, wantEntry %v", gotEntry, tt.wantEntry) + } + }) + } +} + +func TestConfig_ShouldIgnorePackageVersion(t *testing.T) { + t.Parallel() + + type args struct { + name string + version string + ecosystem string + } + tests := []struct { + name string + config Config + args args + wantOk bool + wantEntry PackageOverrideEntry + }{ + { + name: "Version-level entry exists", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: args{ + name: "lib1", + version: "1.0.0", + ecosystem: "Go", + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Package-level entry exists", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: args{ + name: "lib1", + version: "1.0.0", + ecosystem: "Go", + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ecosystem: "Go", + Ignore: true, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + { + name: "Entry doesn't exist", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "2.0.0", + Ecosystem: "Go", + Ignore: false, + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + { + Name: "lib2", + Version: "2.0.0", + Ignore: true, + Ecosystem: "Go", + EffectiveUntil: time.Time{}, + Reason: "abc", + }, + }, + }, + args: args{ + name: "lib1", + version: "2.0.0", + ecosystem: "Go", + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotOk, gotEntry := tt.config.ShouldIgnorePackageVersion(tt.args.name, tt.args.version, tt.args.ecosystem) + if gotOk != tt.wantOk { + t.Errorf("ShouldIgnorePackageVersion() gotOk = %v, wantOk %v", gotOk, tt.wantOk) + } + if !reflect.DeepEqual(gotEntry, tt.wantEntry) { + t.Errorf("ShouldIgnorePackageVersion() gotEntry = %v, wantEntry %v", gotEntry, tt.wantEntry) + } + }) + } +} + +func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config Config + args models.PackageVulns + wantOk bool + }{ + { + name: "Exact version entry exists with ignore", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + Vulnerability: Vulnerability{ + Ignore: true, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + }, + wantOk: true, + }, + { + name: "Version entry doesn't exist with ignore", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + Vulnerability: Vulnerability{ + Ignore: true, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + }, + wantOk: false, + }, + { + name: "Name matches with ignore", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + Vulnerability: Vulnerability{ + Ignore: true, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + }, + wantOk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotOk := tt.config.ShouldIgnorePackageVulnerabilities(tt.args) + if gotOk != tt.wantOk { + t.Errorf("ShouldIgnorePackageVulnerabilities() gotOk = %v, wantOk %v", gotOk, tt.wantOk) + } + }) + } +} + +func TestConfig_ShouldOverridePackageLicense(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + config Config + args models.PackageVulns + wantOk bool + wantEntry PackageOverrideEntry + }{ + { + name: "Exact version entry exists with override", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + { + name: "Exact version entry exists with ignore", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Ignore: true, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + }, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Ignore: true, + }, + Reason: "abc", + }, + }, + { + name: "Version entry doesn't exist with override", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + { + name: "Version entry doesn't exist with ignore", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Ignore: true, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + { + name: "Name matches with override", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + { + name: "Name matches with ignore", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + License: License{ + Ignore: true, + }, + Reason: "abc", + }, + }, + }, + args: models.PackageVulns{ + Package: models.PackageInfo{ + Name: "lib1", + Version: "1.0.1", + Ecosystem: "Go", + }, + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ecosystem: "Go", + License: License{ + Ignore: true, + }, + Reason: "abc", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotOk, gotEntry := tt.config.ShouldOverridePackageLicense(tt.args) + if gotOk != tt.wantOk { + t.Errorf("ShouldOverridePackageLicense() gotOk = %v, wantOk %v", gotOk, tt.wantOk) + } + if !reflect.DeepEqual(gotEntry, tt.wantEntry) { + t.Errorf("ShouldOverridePackageLicense() gotEntry = %v, wantEntry %v", gotEntry, tt.wantEntry) + } + }) + } +} + +func TestConfig_ShouldOverridePackageVersionLicense(t *testing.T) { + t.Parallel() + + type args struct { + name string + version string + ecosystem string + } + tests := []struct { + name string + config Config + args args + wantOk bool + wantEntry PackageOverrideEntry + }{ + { + name: "Exact version entry exists", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + }, + args: args{ + name: "lib1", + version: "1.0.0", + ecosystem: "Go", + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + { + name: "Version entry doesn't exist", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Version: "1.0.0", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + }, + args: args{ + name: "lib1", + version: "1.0.1", + ecosystem: "Go", + }, + wantOk: false, + wantEntry: PackageOverrideEntry{}, + }, + { + name: "Name matches", + config: Config{ + PackageOverrides: []PackageOverrideEntry{ + { + Name: "lib1", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + }, + args: args{ + name: "lib1", + version: "1.0.1", + ecosystem: "Go", + }, + wantOk: true, + wantEntry: PackageOverrideEntry{ + Name: "lib1", + Ecosystem: "Go", + License: License{ + Override: []string{"mit"}, + }, + Reason: "abc", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotOk, gotEntry := tt.config.ShouldOverridePackageVersionLicense(tt.args.name, tt.args.version, tt.args.ecosystem) + if gotOk != tt.wantOk { + t.Errorf("ShouldOverridePackageVersionLicense() gotOk = %v, wantOk %v", gotOk, tt.wantOk) + } + if !reflect.DeepEqual(gotEntry, tt.wantEntry) { + t.Errorf("ShouldOverridePackageVersionLicense() gotEntry = %v, wantEntry %v", gotEntry, tt.wantEntry) + } + }) + } +} diff --git a/internal/config/fixtures/testdatainner/innerFolder/test.yaml b/internal/config/fixtures/testdatainner/innerFolder/test.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/config/fixtures/testdatainner/osv-scanner-load-path.toml b/internal/config/fixtures/testdatainner/osv-scanner-load-path.toml new file mode 100644 index 0000000000..001548b76d --- /dev/null +++ b/internal/config/fixtures/testdatainner/osv-scanner-load-path.toml @@ -0,0 +1 @@ +LoadPath = "a/b/c" diff --git a/internal/config/fixtures/testdatainner/osv-scanner.toml b/internal/config/fixtures/testdatainner/osv-scanner.toml new file mode 100644 index 0000000000..f9be2c0f2e --- /dev/null +++ b/internal/config/fixtures/testdatainner/osv-scanner.toml @@ -0,0 +1,25 @@ +[[IgnoredVulns]] +id = "GO-2022-0968" +# ignoreUntil = 2022-11-09 +# reason = "" # Optional reason + +[[IgnoredVulns]] +id = "GO-2022-1059" +# ignoreUntil = 2022-11-09 # Optional exception expiry date +# reason = "" # Optional reason + +[[PackageOverrides]] +name = "lib" +version = "1.0.0" +ecosystem = "Go" +ignore = true +# effectiveUntil = 2022-11-09 # Optional exception expiry date +reason = "abc" + +[[PackageOverrides]] +name = "my-pkg" +version = "1.0.0" +ecosystem = "Go" +ignore = true +reason = "abc" +license.override = ["MIT", "0BSD"] diff --git a/internal/config/fixtures/testdatainner/some-manifest.yaml b/internal/config/fixtures/testdatainner/some-manifest.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/config/fixtures/unknown-key-1.toml b/internal/config/fixtures/unknown-key-1.toml new file mode 100644 index 0000000000..2c8538325b --- /dev/null +++ b/internal/config/fixtures/unknown-key-1.toml @@ -0,0 +1,4 @@ +[[IgnoredVulns]] +id = "GHSA-jgvc-jfgh-rjvv" +ignoreUntilTime = 2024-08-02 # whoops, should be "ignoreUntil" +reason = "..." diff --git a/internal/config/fixtures/unknown-key-2.toml b/internal/config/fixtures/unknown-key-2.toml new file mode 100644 index 0000000000..7b6d964f43 --- /dev/null +++ b/internal/config/fixtures/unknown-key-2.toml @@ -0,0 +1,4 @@ +[[IgnoredVulns]] +id = "GHSA-jgvc-jfgh-rjvv" +ignoreUntiI = 2024-08-02 # whoops, should be "ignoreUntil" +reason = "..." diff --git a/internal/config/fixtures/unknown-key-3.toml b/internal/config/fixtures/unknown-key-3.toml new file mode 100644 index 0000000000..bce7ed9a19 --- /dev/null +++ b/internal/config/fixtures/unknown-key-3.toml @@ -0,0 +1,4 @@ +[[IgnoredVulns]] +id = "GHSA-jgvc-jfgh-rjvv" +ignoreUntil = 2024-08-02 +reasoning = "..." # whoops, should be "reason" diff --git a/internal/config/fixtures/unknown-key-4.toml b/internal/config/fixtures/unknown-key-4.toml new file mode 100644 index 0000000000..f508c89dd1 --- /dev/null +++ b/internal/config/fixtures/unknown-key-4.toml @@ -0,0 +1,4 @@ +[[PackageOverrides]] +ecosystem = "npm" +skip = true # whoops, should be "ignore" +license.override = ["0BSD"] diff --git a/internal/config/fixtures/unknown-key-5.toml b/internal/config/fixtures/unknown-key-5.toml new file mode 100644 index 0000000000..d1d832aed0 --- /dev/null +++ b/internal/config/fixtures/unknown-key-5.toml @@ -0,0 +1,3 @@ +[[PackageOverrides]] +ecosystem = "npm" +license.skip = false # whoops, should be "license.ignore" diff --git a/internal/config/fixtures/unknown-key-6.toml b/internal/config/fixtures/unknown-key-6.toml new file mode 100644 index 0000000000..80f0b87eee --- /dev/null +++ b/internal/config/fixtures/unknown-key-6.toml @@ -0,0 +1 @@ +RustVersionOverride = "1.2.3" # whoops, not supported diff --git a/internal/config/fixtures/unknown-key-7.toml b/internal/config/fixtures/unknown-key-7.toml new file mode 100644 index 0000000000..044156ccec --- /dev/null +++ b/internal/config/fixtures/unknown-key-7.toml @@ -0,0 +1,5 @@ +RustVersionOverride = "1.2.3" # whoops, not supported + +[[PackageOverrides]] +ecosystem = "npm" +skip = true # whoops, should be "ignore" diff --git a/internal/depsdev/license.go b/internal/depsdev/license.go new file mode 100644 index 0000000000..aca27bacc0 --- /dev/null +++ b/internal/depsdev/license.go @@ -0,0 +1,114 @@ +package depsdev + +import ( + "context" + "crypto/x509" + "fmt" + + "github.com/google/osv-scanner/pkg/lockfile" + "github.com/google/osv-scanner/pkg/models" + "github.com/google/osv-scanner/pkg/osv" + + depsdevpb "deps.dev/api/v3" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" +) + +// DepsdevAPI is the URL to the deps.dev API. It is documented at +// docs.deps.dev/api. +const DepsdevAPI = "api.deps.dev:443" + +// System maps from a lockfile system to the depsdev API system. +var System = map[lockfile.Ecosystem]depsdevpb.System{ + lockfile.NpmEcosystem: depsdevpb.System_NPM, + lockfile.NuGetEcosystem: depsdevpb.System_NUGET, + lockfile.CargoEcosystem: depsdevpb.System_CARGO, + lockfile.GoEcosystem: depsdevpb.System_GO, + lockfile.MavenEcosystem: depsdevpb.System_MAVEN, + lockfile.PipEcosystem: depsdevpb.System_PYPI, +} + +// VersionQuery constructs a GetVersion request from the arguments. +func VersionQuery(system depsdevpb.System, name string, version string) *depsdevpb.GetVersionRequest { + if system == depsdevpb.System_GO { + version = "v" + version + } + + return &depsdevpb.GetVersionRequest{ + VersionKey: &depsdevpb.VersionKey{ + System: system, + Name: name, + Version: version, + }, + } +} + +// MakeVersionRequests wraps MakeVersionRequestsWithContext using context.Background. +func MakeVersionRequests(queries []*depsdevpb.GetVersionRequest) ([][]models.License, error) { + return MakeVersionRequestsWithContext(context.Background(), queries) +} + +// MakeVersionRequestsWithContext calls the deps.dev GetVersion gRPC API endpoint for each +// query. It makes these requests concurrently, sharing the single HTTP/2 +// connection. The order in which the requests are specified should correspond +// to the order of licenses returned by this function. +func MakeVersionRequestsWithContext(ctx context.Context, queries []*depsdevpb.GetVersionRequest) ([][]models.License, error) { + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("getting system cert pool: %w", err) + } + creds := credentials.NewClientTLSFromCert(certPool, "") + dialOpts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + + if osv.RequestUserAgent != "" { + dialOpts = append(dialOpts, grpc.WithUserAgent(osv.RequestUserAgent)) + } + + conn, err := grpc.NewClient(DepsdevAPI, dialOpts...) + if err != nil { + return nil, fmt.Errorf("dialing deps.dev gRPC API: %w", err) + } + client := depsdevpb.NewInsightsClient(conn) + + licenses := make([][]models.License, len(queries)) + g, ctx := errgroup.WithContext(ctx) + for i := range queries { + if queries[i] == nil { + // This may be a private package. + licenses[i] = []models.License{models.License("UNKNOWN")} + continue + } + g.Go(func() error { + resp, err := client.GetVersion(ctx, queries[i]) + if err != nil { + if status.Code(err) == codes.NotFound { + licenses[i] = append(licenses[i], "UNKNOWN") + return nil + } + + return err + } + ls := make([]models.License, len(resp.GetLicenses())) + for j, license := range resp.GetLicenses() { + ls[j] = models.License(license) + } + if len(ls) == 0 { + // The deps.dev API will return an + // empty slice if the license is + // unknown. + ls = []models.License{models.License("UNKNOWN")} + } + licenses[i] = ls + + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + + return licenses, nil +} diff --git a/internal/grouper/grouper.go b/internal/grouper/grouper.go new file mode 100644 index 0000000000..b21edaae76 --- /dev/null +++ b/internal/grouper/grouper.go @@ -0,0 +1,73 @@ +package grouper + +import ( + "slices" + "sort" + + "golang.org/x/exp/maps" + + "github.com/google/osv-scanner/internal/identifiers" + "github.com/google/osv-scanner/pkg/models" +) + +func hasAliasIntersection(v1, v2 IDAliases) bool { + // Check if any aliases intersect. + for _, alias := range v1.Aliases { + if slices.Contains(v2.Aliases, alias) { + return true + } + } + // Check if either IDs are in the others' aliases. + return slices.Contains(v1.Aliases, v2.ID) || slices.Contains(v2.Aliases, v1.ID) +} + +// Group groups vulnerabilities by aliases. +func Group(vulns []IDAliases) []models.GroupInfo { + // Mapping of `vulns` index to a group ID. A group ID is just another index in the `vulns` slice. + groups := make([]int, len(vulns)) + + // Initially make every vulnerability its own group. + for i := range vulns { + groups[i] = i + } + + // Do a pair-wise (n^2) comparison and merge all intersecting vulns. + for i := range vulns { + for j := i + 1; j < len(vulns); j++ { + if hasAliasIntersection(vulns[i], vulns[j]) { + // Merge the two groups. Use the smaller index as the representative ID. + groups[i] = min(groups[i], groups[j]) + groups[j] = groups[i] + } + } + } + + // Extract groups into the final result structure. + extractedGroups := map[int][]string{} + extractedAliases := map[int][]string{} + for i, gid := range groups { + extractedGroups[gid] = append(extractedGroups[gid], vulns[i].ID) + extractedAliases[gid] = append(extractedAliases[gid], vulns[i].Aliases...) + } + + // Sort by group ID to maintain stable order for tests. + sortedKeys := maps.Keys(extractedGroups) + sort.Ints(sortedKeys) + + result := make([]models.GroupInfo, 0, len(sortedKeys)) + for _, key := range sortedKeys { + // Sort the strings so they are always in the same order + slices.SortFunc(extractedGroups[key], identifiers.IDSortFunc) + + // Add IDs to aliases + extractedAliases[key] = append(extractedAliases[key], extractedGroups[key]...) + + // Dedup entries + sort.Strings(extractedAliases[key]) + extractedAliases[key] = slices.Compact(extractedAliases[key]) + + result = append(result, models.GroupInfo{IDs: extractedGroups[key], Aliases: extractedAliases[key]}) + } + + return result +} diff --git a/internal/grouper/grouper_models.go b/internal/grouper/grouper_models.go new file mode 100644 index 0000000000..b713aafaa1 --- /dev/null +++ b/internal/grouper/grouper_models.go @@ -0,0 +1,33 @@ +package grouper + +import ( + "strings" + + "github.com/google/osv-scanner/pkg/models" +) + +type IDAliases struct { + ID string + Aliases []string +} + +func ConvertVulnerabilityToIDAliases(c []models.Vulnerability) []IDAliases { + output := []IDAliases{} + for _, v := range c { + idAliases := IDAliases{ + ID: v.ID, + Aliases: v.Aliases, + } + + // For Debian Security Advisory data, + // all related CVEs should be bundled together, as they are part of this DSA. + // TODO(gongh@): Revisit and provide a universal way to handle all Linux distro advisories. + if strings.Split(v.ID, "-")[0] == "DSA" { + idAliases.Aliases = append(idAliases.Aliases, v.Related...) + } + + output = append(output, idAliases) + } + + return output +} diff --git a/internal/grouper/grouper_test.go b/internal/grouper/grouper_test.go new file mode 100644 index 0000000000..17d4b2ddb0 --- /dev/null +++ b/internal/grouper/grouper_test.go @@ -0,0 +1,154 @@ +package grouper_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/osv-scanner/internal/grouper" + "github.com/google/osv-scanner/pkg/models" +) + +func TestGroup(t *testing.T) { + t.Parallel() + + // Should be grouped by IDs appearing in alias. + v1 := grouper.IDAliases{ + ID: "CVE-1", + Aliases: []string{ + "FOO-1", + }, + } + v2 := grouper.IDAliases{ + ID: "FOO-1", + Aliases: []string{}, + } + v3 := grouper.IDAliases{ + ID: "FOO-2", + Aliases: []string{ + "FOO-1", + }, + } + + // Should be grouped by aliases intersecting. + v4 := grouper.IDAliases{ + ID: "BAR-1", + Aliases: []string{ + "CVE-2", + "CVE-3", + }, + } + v5 := grouper.IDAliases{ + ID: "BAR-2", + Aliases: []string{ + "CVE-3", + "CVE-4", + }, + } + v6 := grouper.IDAliases{ + ID: "BAR-3", + Aliases: []string{ + "CVE-4", + }, + } + + // Unrelated. + v7 := grouper.IDAliases{ + ID: "UNRELATED-1", + Aliases: []string{ + "BAR-1337", + }, + } + v8 := grouper.IDAliases{ + ID: "UNRELATED-2", + Aliases: []string{ + "BAR-1338", + }, + } + + // Unrelated, empty aliases + v9 := grouper.IDAliases{ + ID: "UNRELATED-3", + } + v10 := grouper.IDAliases{ + ID: "UNRELATED-4", + } + for _, tc := range []struct { + vulns []grouper.IDAliases + want []models.GroupInfo + }{ + { + vulns: []grouper.IDAliases{ + v1, v2, v3, v4, v5, v6, v7, v8, + }, + want: []models.GroupInfo{ + { + IDs: []string{v1.ID, v2.ID, v3.ID}, + Aliases: []string{v1.ID, v2.ID, v3.ID}, + }, + { + IDs: []string{v4.ID, v5.ID, v6.ID}, + Aliases: []string{v4.ID, v5.ID, v6.ID, v4.Aliases[0], v4.Aliases[1], v5.Aliases[1]}, + }, + { + IDs: []string{v7.ID}, + Aliases: []string{v7.Aliases[0], v7.ID}, + }, + { + IDs: []string{v8.ID}, + Aliases: []string{v8.Aliases[0], v8.ID}, + }, + }, + }, + { + vulns: []grouper.IDAliases{ + v8, v2, v1, v5, v7, v4, v6, v3, v9, v10, + }, + want: []models.GroupInfo{ + { + IDs: []string{v8.ID}, + Aliases: []string{v8.Aliases[0], v8.ID}, + }, + { + IDs: []string{v1.ID, v2.ID, v3.ID}, // Deterministic order + Aliases: []string{v1.ID, v2.ID, v3.ID}, // Deterministic order + }, + { + IDs: []string{v4.ID, v5.ID, v6.ID}, + Aliases: []string{v4.ID, v5.ID, v6.ID, v4.Aliases[0], v4.Aliases[1], v5.Aliases[1]}, + }, + { + IDs: []string{v7.ID}, + Aliases: []string{v7.Aliases[0], v7.ID}, + }, + { + IDs: []string{v9.ID}, + Aliases: []string{v9.ID}, + }, + { + IDs: []string{v10.ID}, + Aliases: []string{v10.ID}, + }, + }, + }, + { + vulns: []grouper.IDAliases{ + v9, v10, + }, + want: []models.GroupInfo{ + { + IDs: []string{v9.ID}, + Aliases: []string{v9.ID}, + }, + { + IDs: []string{v10.ID}, + Aliases: []string{v10.ID}, + }, + }, + }, + } { + grouped := grouper.Group(tc.vulns) + if diff := cmp.Diff(tc.want, grouped); diff != "" { + t.Errorf("GroupedVulns() returned an unexpected result (-want +got):\n%s", diff) + } + } +} diff --git a/internal/resolution/client/client.go b/internal/resolution/client/client.go index 965af188ea..35da8b86e6 100644 --- a/internal/resolution/client/client.go +++ b/internal/resolution/client/client.go @@ -6,7 +6,7 @@ import ( pb "deps.dev/api/v3" "deps.dev/util/resolve" - "github.com/google/osv-scanner/pkg/depsdev" + "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" "google.golang.org/grpc" diff --git a/internal/resolution/client/npm_registry_client.go b/internal/resolution/client/npm_registry_client.go index 99867f6f64..9ff9cc0d87 100644 --- a/internal/resolution/client/npm_registry_client.go +++ b/internal/resolution/client/npm_registry_client.go @@ -13,8 +13,8 @@ import ( "deps.dev/util/resolve" "deps.dev/util/resolve/dep" "deps.dev/util/semver" + "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/internal/resolution/datasource" - "github.com/google/osv-scanner/pkg/depsdev" "github.com/google/osv-scanner/pkg/osv" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/internal/spdx/gen.go b/internal/spdx/gen.go new file mode 100644 index 0000000000..27c85b1ee9 --- /dev/null +++ b/internal/spdx/gen.go @@ -0,0 +1,54 @@ +//go:build generate +// +build generate + +//go:generate go run gen.go + +package main + +import ( + "encoding/json" + "fmt" + "go/format" + "io/ioutil" + "net/http" + "strings" +) + +type License struct { + SPDXID string `json:"licenseId"` +} + +func main() { + resp, err := http.Get("https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json") + if err != nil { + panic(err) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + + var licenseList struct { + Licenses []License `json:"licenses"` + } + err = json.Unmarshal(body, &licenseList) + if err != nil { + panic(err) + } + + output := "// Code generated by gen.go. DO NOT EDIT.\n package spdx\nvar IDs = map[string]bool{\n" + for _, license := range licenseList.Licenses { + output += fmt.Sprintf("%q: true,\n", strings.ToLower(license.SPDXID)) + } + output += "}" + formatted, err := format.Source([]byte(output)) + if err != nil { + panic(err) + } + err = ioutil.WriteFile("licenses.go", formatted, 0644) + if err != nil { + panic(err) + } +} diff --git a/internal/spdx/licenses.go b/internal/spdx/licenses.go new file mode 100644 index 0000000000..07e0396c52 --- /dev/null +++ b/internal/spdx/licenses.go @@ -0,0 +1,630 @@ +// Code generated by gen.go. DO NOT EDIT. +package spdx + +var IDs = map[string]bool{ + "0bsd": true, + "aal": true, + "abstyles": true, + "adacore-doc": true, + "adobe-2006": true, + "adobe-display-postscript": true, + "adobe-glyph": true, + "adobe-utopia": true, + "adsl": true, + "afl-1.1": true, + "afl-1.2": true, + "afl-2.0": true, + "afl-2.1": true, + "afl-3.0": true, + "afmparse": true, + "agpl-1.0": true, + "agpl-1.0-only": true, + "agpl-1.0-or-later": true, + "agpl-3.0": true, + "agpl-3.0-only": true, + "agpl-3.0-or-later": true, + "aladdin": true, + "amdplpa": true, + "aml": true, + "aml-glslang": true, + "ampas": true, + "antlr-pd": true, + "antlr-pd-fallback": true, + "apache-1.0": true, + "apache-1.1": true, + "apache-2.0": true, + "apafml": true, + "apl-1.0": true, + "app-s2p": true, + "apsl-1.0": true, + "apsl-1.1": true, + "apsl-1.2": true, + "apsl-2.0": true, + "arphic-1999": true, + "artistic-1.0": true, + "artistic-1.0-cl8": true, + "artistic-1.0-perl": true, + "artistic-2.0": true, + "aswf-digital-assets-1.0": true, + "aswf-digital-assets-1.1": true, + "baekmuk": true, + "bahyph": true, + "barr": true, + "beerware": true, + "bitstream-charter": true, + "bitstream-vera": true, + "bittorrent-1.0": true, + "bittorrent-1.1": true, + "blessing": true, + "blueoak-1.0.0": true, + "boehm-gc": true, + "borceux": true, + "brian-gladman-3-clause": true, + "bsd-1-clause": true, + "bsd-2-clause": true, + "bsd-2-clause-darwin": true, + "bsd-2-clause-freebsd": true, + "bsd-2-clause-netbsd": true, + "bsd-2-clause-patent": true, + "bsd-2-clause-views": true, + "bsd-3-clause": true, + "bsd-3-clause-acpica": true, + "bsd-3-clause-attribution": true, + "bsd-3-clause-clear": true, + "bsd-3-clause-flex": true, + "bsd-3-clause-hp": true, + "bsd-3-clause-lbnl": true, + "bsd-3-clause-modification": true, + "bsd-3-clause-no-military-license": true, + "bsd-3-clause-no-nuclear-license": true, + "bsd-3-clause-no-nuclear-license-2014": true, + "bsd-3-clause-no-nuclear-warranty": true, + "bsd-3-clause-open-mpi": true, + "bsd-3-clause-sun": true, + "bsd-4-clause": true, + "bsd-4-clause-shortened": true, + "bsd-4-clause-uc": true, + "bsd-4.3reno": true, + "bsd-4.3tahoe": true, + "bsd-advertising-acknowledgement": true, + "bsd-attribution-hpnd-disclaimer": true, + "bsd-inferno-nettverk": true, + "bsd-protection": true, + "bsd-source-beginning-file": true, + "bsd-source-code": true, + "bsd-systemics": true, + "bsd-systemics-w3works": true, + "bsl-1.0": true, + "busl-1.1": true, + "bzip2-1.0.5": true, + "bzip2-1.0.6": true, + "c-uda-1.0": true, + "cal-1.0": true, + "cal-1.0-combined-work-exception": true, + "caldera": true, + "caldera-no-preamble": true, + "catosl-1.1": true, + "cc-by-1.0": true, + "cc-by-2.0": true, + "cc-by-2.5": true, + "cc-by-2.5-au": true, + "cc-by-3.0": true, + "cc-by-3.0-at": true, + "cc-by-3.0-au": true, + "cc-by-3.0-de": true, + "cc-by-3.0-igo": true, + "cc-by-3.0-nl": true, + "cc-by-3.0-us": true, + "cc-by-4.0": true, + "cc-by-nc-1.0": true, + "cc-by-nc-2.0": true, + "cc-by-nc-2.5": true, + "cc-by-nc-3.0": true, + "cc-by-nc-3.0-de": true, + "cc-by-nc-4.0": true, + "cc-by-nc-nd-1.0": true, + "cc-by-nc-nd-2.0": true, + "cc-by-nc-nd-2.5": true, + "cc-by-nc-nd-3.0": true, + "cc-by-nc-nd-3.0-de": true, + "cc-by-nc-nd-3.0-igo": true, + "cc-by-nc-nd-4.0": true, + "cc-by-nc-sa-1.0": true, + "cc-by-nc-sa-2.0": true, + "cc-by-nc-sa-2.0-de": true, + "cc-by-nc-sa-2.0-fr": true, + "cc-by-nc-sa-2.0-uk": true, + "cc-by-nc-sa-2.5": true, + "cc-by-nc-sa-3.0": true, + "cc-by-nc-sa-3.0-de": true, + "cc-by-nc-sa-3.0-igo": true, + "cc-by-nc-sa-4.0": true, + "cc-by-nd-1.0": true, + "cc-by-nd-2.0": true, + "cc-by-nd-2.5": true, + "cc-by-nd-3.0": true, + "cc-by-nd-3.0-de": true, + "cc-by-nd-4.0": true, + "cc-by-sa-1.0": true, + "cc-by-sa-2.0": true, + "cc-by-sa-2.0-uk": true, + "cc-by-sa-2.1-jp": true, + "cc-by-sa-2.5": true, + "cc-by-sa-3.0": true, + "cc-by-sa-3.0-at": true, + "cc-by-sa-3.0-de": true, + "cc-by-sa-3.0-igo": true, + "cc-by-sa-4.0": true, + "cc-pddc": true, + "cc0-1.0": true, + "cddl-1.0": true, + "cddl-1.1": true, + "cdl-1.0": true, + "cdla-permissive-1.0": true, + "cdla-permissive-2.0": true, + "cdla-sharing-1.0": true, + "cecill-1.0": true, + "cecill-1.1": true, + "cecill-2.0": true, + "cecill-2.1": true, + "cecill-b": true, + "cecill-c": true, + "cern-ohl-1.1": true, + "cern-ohl-1.2": true, + "cern-ohl-p-2.0": true, + "cern-ohl-s-2.0": true, + "cern-ohl-w-2.0": true, + "cfitsio": true, + "check-cvs": true, + "checkmk": true, + "clartistic": true, + "clips": true, + "cmu-mach": true, + "cnri-jython": true, + "cnri-python": true, + "cnri-python-gpl-compatible": true, + "coil-1.0": true, + "community-spec-1.0": true, + "condor-1.1": true, + "copyleft-next-0.3.0": true, + "copyleft-next-0.3.1": true, + "cornell-lossless-jpeg": true, + "cpal-1.0": true, + "cpl-1.0": true, + "cpol-1.02": true, + "cronyx": true, + "crossword": true, + "crystalstacker": true, + "cua-opl-1.0": true, + "cube": true, + "curl": true, + "d-fsl-1.0": true, + "dec-3-clause": true, + "diffmark": true, + "dl-de-by-2.0": true, + "dl-de-zero-2.0": true, + "doc": true, + "dotseqn": true, + "drl-1.0": true, + "drl-1.1": true, + "dsdp": true, + "dtoa": true, + "dvipdfm": true, + "ecl-1.0": true, + "ecl-2.0": true, + "ecos-2.0": true, + "efl-1.0": true, + "efl-2.0": true, + "egenix": true, + "elastic-2.0": true, + "entessa": true, + "epics": true, + "epl-1.0": true, + "epl-2.0": true, + "erlpl-1.1": true, + "etalab-2.0": true, + "eudatagrid": true, + "eupl-1.0": true, + "eupl-1.1": true, + "eupl-1.2": true, + "eurosym": true, + "fair": true, + "fbm": true, + "fdk-aac": true, + "ferguson-twofish": true, + "frameworx-1.0": true, + "freebsd-doc": true, + "freeimage": true, + "fsfap": true, + "fsfap-no-warranty-disclaimer": true, + "fsful": true, + "fsfullr": true, + "fsfullrwd": true, + "ftl": true, + "furuseth": true, + "fwlw": true, + "gcr-docs": true, + "gd": true, + "gfdl-1.1": true, + "gfdl-1.1-invariants-only": true, + "gfdl-1.1-invariants-or-later": true, + "gfdl-1.1-no-invariants-only": true, + "gfdl-1.1-no-invariants-or-later": true, + "gfdl-1.1-only": true, + "gfdl-1.1-or-later": true, + "gfdl-1.2": true, + "gfdl-1.2-invariants-only": true, + "gfdl-1.2-invariants-or-later": true, + "gfdl-1.2-no-invariants-only": true, + "gfdl-1.2-no-invariants-or-later": true, + "gfdl-1.2-only": true, + "gfdl-1.2-or-later": true, + "gfdl-1.3": true, + "gfdl-1.3-invariants-only": true, + "gfdl-1.3-invariants-or-later": true, + "gfdl-1.3-no-invariants-only": true, + "gfdl-1.3-no-invariants-or-later": true, + "gfdl-1.3-only": true, + "gfdl-1.3-or-later": true, + "giftware": true, + "gl2ps": true, + "glide": true, + "glulxe": true, + "glwtpl": true, + "gnuplot": true, + "gpl-1.0": true, + "gpl-1.0+": true, + "gpl-1.0-only": true, + "gpl-1.0-or-later": true, + "gpl-2.0": true, + "gpl-2.0+": true, + "gpl-2.0-only": true, + "gpl-2.0-or-later": true, + "gpl-2.0-with-autoconf-exception": true, + "gpl-2.0-with-bison-exception": true, + "gpl-2.0-with-classpath-exception": true, + "gpl-2.0-with-font-exception": true, + "gpl-2.0-with-gcc-exception": true, + "gpl-3.0": true, + "gpl-3.0+": true, + "gpl-3.0-only": true, + "gpl-3.0-or-later": true, + "gpl-3.0-with-autoconf-exception": true, + "gpl-3.0-with-gcc-exception": true, + "graphics-gems": true, + "gsoap-1.3b": true, + "haskellreport": true, + "hdparm": true, + "hippocratic-2.1": true, + "hp-1986": true, + "hp-1989": true, + "hpnd": true, + "hpnd-dec": true, + "hpnd-doc": true, + "hpnd-doc-sell": true, + "hpnd-export-us": true, + "hpnd-export-us-modify": true, + "hpnd-kevlin-henney": true, + "hpnd-markus-kuhn": true, + "hpnd-mit-disclaimer": true, + "hpnd-pbmplus": true, + "hpnd-sell-mit-disclaimer-xserver": true, + "hpnd-sell-regexpr": true, + "hpnd-sell-variant": true, + "hpnd-sell-variant-mit-disclaimer": true, + "hpnd-uc": true, + "htmltidy": true, + "ibm-pibs": true, + "icu": true, + "iec-code-components-eula": true, + "ijg": true, + "ijg-short": true, + "imagemagick": true, + "imatix": true, + "imlib2": true, + "info-zip": true, + "inner-net-2.0": true, + "intel": true, + "intel-acpi": true, + "interbase-1.0": true, + "ipa": true, + "ipl-1.0": true, + "isc": true, + "isc-veillard": true, + "jam": true, + "jasper-2.0": true, + "jpl-image": true, + "jpnic": true, + "json": true, + "kastrup": true, + "kazlib": true, + "knuth-ctan": true, + "lal-1.2": true, + "lal-1.3": true, + "latex2e": true, + "latex2e-translated-notice": true, + "leptonica": true, + "lgpl-2.0": true, + "lgpl-2.0+": true, + "lgpl-2.0-only": true, + "lgpl-2.0-or-later": true, + "lgpl-2.1": true, + "lgpl-2.1+": true, + "lgpl-2.1-only": true, + "lgpl-2.1-or-later": true, + "lgpl-3.0": true, + "lgpl-3.0+": true, + "lgpl-3.0-only": true, + "lgpl-3.0-or-later": true, + "lgpllr": true, + "libpng": true, + "libpng-2.0": true, + "libselinux-1.0": true, + "libtiff": true, + "libutil-david-nugent": true, + "liliq-p-1.1": true, + "liliq-r-1.1": true, + "liliq-rplus-1.1": true, + "linux-man-pages-1-para": true, + "linux-man-pages-copyleft": true, + "linux-man-pages-copyleft-2-para": true, + "linux-man-pages-copyleft-var": true, + "linux-openib": true, + "loop": true, + "lpd-document": true, + "lpl-1.0": true, + "lpl-1.02": true, + "lppl-1.0": true, + "lppl-1.1": true, + "lppl-1.2": true, + "lppl-1.3a": true, + "lppl-1.3c": true, + "lsof": true, + "lucida-bitmap-fonts": true, + "lzma-sdk-9.11-to-9.20": true, + "lzma-sdk-9.22": true, + "magaz": true, + "mailprio": true, + "makeindex": true, + "martin-birgmeier": true, + "mcphee-slideshow": true, + "metamail": true, + "minpack": true, + "miros": true, + "mit": true, + "mit-0": true, + "mit-advertising": true, + "mit-cmu": true, + "mit-enna": true, + "mit-feh": true, + "mit-festival": true, + "mit-modern-variant": true, + "mit-open-group": true, + "mit-testregex": true, + "mit-wu": true, + "mitnfa": true, + "mmixware": true, + "motosoto": true, + "mpeg-ssg": true, + "mpi-permissive": true, + "mpich2": true, + "mpl-1.0": true, + "mpl-1.1": true, + "mpl-2.0": true, + "mpl-2.0-no-copyleft-exception": true, + "mplus": true, + "ms-lpl": true, + "ms-pl": true, + "ms-rl": true, + "mtll": true, + "mulanpsl-1.0": true, + "mulanpsl-2.0": true, + "multics": true, + "mup": true, + "naist-2003": true, + "nasa-1.3": true, + "naumen": true, + "nbpl-1.0": true, + "ncgl-uk-2.0": true, + "ncsa": true, + "net-snmp": true, + "netcdf": true, + "newsletr": true, + "ngpl": true, + "nicta-1.0": true, + "nist-pd": true, + "nist-pd-fallback": true, + "nist-software": true, + "nlod-1.0": true, + "nlod-2.0": true, + "nlpl": true, + "nokia": true, + "nosl": true, + "noweb": true, + "npl-1.0": true, + "npl-1.1": true, + "nposl-3.0": true, + "nrl": true, + "ntp": true, + "ntp-0": true, + "nunit": true, + "o-uda-1.0": true, + "occt-pl": true, + "oclc-2.0": true, + "odbl-1.0": true, + "odc-by-1.0": true, + "offis": true, + "ofl-1.0": true, + "ofl-1.0-no-rfn": true, + "ofl-1.0-rfn": true, + "ofl-1.1": true, + "ofl-1.1-no-rfn": true, + "ofl-1.1-rfn": true, + "ogc-1.0": true, + "ogdl-taiwan-1.0": true, + "ogl-canada-2.0": true, + "ogl-uk-1.0": true, + "ogl-uk-2.0": true, + "ogl-uk-3.0": true, + "ogtsl": true, + "oldap-1.1": true, + "oldap-1.2": true, + "oldap-1.3": true, + "oldap-1.4": true, + "oldap-2.0": true, + "oldap-2.0.1": true, + "oldap-2.1": true, + "oldap-2.2": true, + "oldap-2.2.1": true, + "oldap-2.2.2": true, + "oldap-2.3": true, + "oldap-2.4": true, + "oldap-2.5": true, + "oldap-2.6": true, + "oldap-2.7": true, + "oldap-2.8": true, + "olfl-1.3": true, + "oml": true, + "openpbs-2.3": true, + "openssl": true, + "openssl-standalone": true, + "opl-1.0": true, + "opl-uk-3.0": true, + "opubl-1.0": true, + "oset-pl-2.1": true, + "osl-1.0": true, + "osl-1.1": true, + "osl-2.0": true, + "osl-2.1": true, + "osl-3.0": true, + "padl": true, + "parity-6.0.0": true, + "parity-7.0.0": true, + "pddl-1.0": true, + "php-3.0": true, + "php-3.01": true, + "pixar": true, + "plexus": true, + "pnmstitch": true, + "polyform-noncommercial-1.0.0": true, + "polyform-small-business-1.0.0": true, + "postgresql": true, + "psf-2.0": true, + "psfrag": true, + "psutils": true, + "python-2.0": true, + "python-2.0.1": true, + "python-ldap": true, + "qhull": true, + "qpl-1.0": true, + "qpl-1.0-inria-2004": true, + "radvd": true, + "rdisc": true, + "rhecos-1.1": true, + "rpl-1.1": true, + "rpl-1.5": true, + "rpsl-1.0": true, + "rsa-md": true, + "rscpl": true, + "ruby": true, + "sax-pd": true, + "sax-pd-2.0": true, + "saxpath": true, + "scea": true, + "schemereport": true, + "sendmail": true, + "sendmail-8.23": true, + "sgi-b-1.0": true, + "sgi-b-1.1": true, + "sgi-b-2.0": true, + "sgi-opengl": true, + "sgp4": true, + "shl-0.5": true, + "shl-0.51": true, + "simpl-2.0": true, + "sissl": true, + "sissl-1.2": true, + "sl": true, + "sleepycat": true, + "smlnj": true, + "smppl": true, + "snia": true, + "snprintf": true, + "soundex": true, + "spencer-86": true, + "spencer-94": true, + "spencer-99": true, + "spl-1.0": true, + "ssh-keyscan": true, + "ssh-openssh": true, + "ssh-short": true, + "ssleay-standalone": true, + "sspl-1.0": true, + "standardml-nj": true, + "sugarcrm-1.1.3": true, + "sunpro": true, + "swl": true, + "swrule": true, + "symlinks": true, + "tapr-ohl-1.0": true, + "tcl": true, + "tcp-wrappers": true, + "termreadkey": true, + "tgppl-1.0": true, + "tmate": true, + "torque-1.1": true, + "tosl": true, + "tpdl": true, + "tpl-1.0": true, + "ttwl": true, + "ttyp0": true, + "tu-berlin-1.0": true, + "tu-berlin-2.0": true, + "ucar": true, + "ucl-1.0": true, + "ulem": true, + "unicode-3.0": true, + "unicode-dfs-2015": true, + "unicode-dfs-2016": true, + "unicode-tou": true, + "unixcrypt": true, + "unlicense": true, + "upl-1.0": true, + "urt-rle": true, + "vim": true, + "vostrom": true, + "vsl-1.0": true, + "w3c": true, + "w3c-19980720": true, + "w3c-20150513": true, + "w3m": true, + "watcom-1.0": true, + "widget-workshop": true, + "wsuipa": true, + "wtfpl": true, + "wxwindows": true, + "x11": true, + "x11-distribute-modifications-variant": true, + "xdebug-1.03": true, + "xerox": true, + "xfig": true, + "xfree86-1.1": true, + "xinetd": true, + "xkeyboard-config-zinoviev": true, + "xlock": true, + "xnet": true, + "xpp": true, + "xskat": true, + "ypl-1.0": true, + "ypl-1.1": true, + "zed": true, + "zeeff": true, + "zend-2.0": true, + "zimbra-1.3": true, + "zimbra-1.4": true, + "zlib": true, + "zlib-acknowledgement": true, + "zpl-1.1": true, + "zpl-2.0": true, + "zpl-2.1": true, +} diff --git a/internal/spdx/verify.go b/internal/spdx/verify.go new file mode 100644 index 0000000000..80aa5b5244 --- /dev/null +++ b/internal/spdx/verify.go @@ -0,0 +1,16 @@ +package spdx + +import "strings" + +// Unrecognized filters licenses for non-spdx identifiers. The "unknown" string is +// also treated as a valid identifier. +func Unrecognized(licenses []string) (unrecognized []string) { + for _, license := range licenses { + l := strings.ToLower(license) + if !IDs[l] && l != "unknown" { + unrecognized = append(unrecognized, license) + } + } + + return unrecognized +} diff --git a/internal/spdx/verify_test.go b/internal/spdx/verify_test.go new file mode 100644 index 0000000000..f0e0cecd4b --- /dev/null +++ b/internal/spdx/verify_test.go @@ -0,0 +1,37 @@ +package spdx + +import ( + "reflect" + "testing" +) + +func Test_unrecognized(t *testing.T) { + t.Parallel() + tests := []struct { + name string + licenses []string + want []string + }{ + { + name: "all recognized licenses", + licenses: []string{"agpl-1.0", "MIT", "apache-1.0", "UNKNOWN"}, + want: nil, + }, { + name: "all unrecognized licenses", + licenses: []string{"agpl1.0", "unrecognized license", "apache1.0"}, + want: []string{"agpl1.0", "unrecognized license", "apache1.0"}, + }, { + name: "some recognized, some unrecognized licenses", + licenses: []string{"agpl-1.0", "unrecognized license", "apache-1.0"}, + want: []string{"unrecognized license"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := Unrecognized(tt.licenses); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Unrecognized() = %v,\nwant %v", got, tt.want) + } + }) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index d1b8b86b0c..3e0058276a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,3 +1,4 @@ +// Deprecated: this is now private and should not be used outside the scanner package config import ( @@ -19,6 +20,8 @@ const osvScannerConfigName = "osv-scanner.toml" // Ignore stuttering as that would be a breaking change // TODO: V2 rename? // +// Deprecated: this is now private and should not be used outside the scanner +// //nolint:revive type ConfigManager struct { // Override to replace all other configs @@ -29,6 +32,7 @@ type ConfigManager struct { ConfigMap map[string]Config } +// Deprecated: this is now private and should not be used outside the scanner type Config struct { IgnoredVulns []IgnoreEntry `toml:"IgnoredVulns"` PackageOverrides []PackageOverrideEntry `toml:"PackageOverrides"` @@ -38,12 +42,14 @@ type Config struct { LoadPath string `toml:"-"` } +// Deprecated: this is now private and should not be used outside the scanner type IgnoreEntry struct { ID string `toml:"id"` IgnoreUntil time.Time `toml:"ignoreUntil"` Reason string `toml:"reason"` } +// Deprecated: this is now private and should not be used outside the scanner type PackageOverrideEntry struct { Name string `toml:"name"` // If the version is empty, the entry applies to all versions. @@ -74,15 +80,18 @@ func (e PackageOverrideEntry) matches(pkg models.PackageVulns) bool { return true } +// Deprecated: this is now private and should not be used outside the scanner type Vulnerability struct { Ignore bool `toml:"ignore"` } +// Deprecated: this is now private and should not be used outside the scanner type License struct { Override []string `toml:"override"` Ignore bool `toml:"ignore"` } +// Deprecated: this is now private and should not be used outside the scanner func (c *Config) ShouldIgnore(vulnID string) (bool, IgnoreEntry) { index := slices.IndexFunc(c.IgnoredVulns, func(e IgnoreEntry) bool { return e.ID == vulnID }) if index == -1 { @@ -106,6 +115,8 @@ func (c *Config) filterPackageVersionEntries(pkg models.PackageVulns, condition } // ShouldIgnorePackage determines if the given package should be ignored based on override entries in the config +// +// Deprecated: this is now private and should not be used outside the scanner func (c *Config) ShouldIgnorePackage(pkg models.PackageVulns) (bool, PackageOverrideEntry) { return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { return e.Ignore @@ -124,6 +135,8 @@ func (c *Config) ShouldIgnorePackageVersion(name, version, ecosystem string) (bo } // ShouldIgnorePackageVulnerabilities determines if the given package should have its vulnerabilities ignored based on override entries in the config +// +// Deprecated: this is now private and should not be used outside the scanner func (c *Config) ShouldIgnorePackageVulnerabilities(pkg models.PackageVulns) bool { overrides, _ := c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { return e.Vulnerability.Ignore @@ -133,6 +146,8 @@ func (c *Config) ShouldIgnorePackageVulnerabilities(pkg models.PackageVulns) boo } // ShouldOverridePackageLicense determines if the given package should have its license ignored or changed based on override entries in the config +// +// Deprecated: this is now private and should not be used outside the scanner func (c *Config) ShouldOverridePackageLicense(pkg models.PackageVulns) (bool, PackageOverrideEntry) { return c.filterPackageVersionEntries(pkg, func(e PackageOverrideEntry) bool { return e.License.Ignore || len(e.License.Override) > 0 @@ -162,6 +177,8 @@ func shouldIgnoreTimestamp(ignoreUntil time.Time) bool { // Sets the override config by reading the config file at configPath. // Will return an error if loading the config file fails +// +// Deprecated: this is now private and should not be used outside the scanner func (c *ConfigManager) UseOverride(configPath string) error { config, configErr := tryLoadConfig(configPath) if configErr != nil { @@ -173,6 +190,8 @@ func (c *ConfigManager) UseOverride(configPath string) error { } // Attempts to get the config +// +// Deprecated: this is now private and should not be used outside the scanner func (c *ConfigManager) Get(r reporter.Reporter, targetPath string) Config { if c.OverrideConfig != nil { return *c.OverrideConfig diff --git a/pkg/depsdev/license.go b/pkg/depsdev/license.go index aca27bacc0..67fc3398e8 100644 --- a/pkg/depsdev/license.go +++ b/pkg/depsdev/license.go @@ -1,3 +1,4 @@ +// Deprecated: this is now private and should not be used outside the scanner package depsdev import ( @@ -19,9 +20,13 @@ import ( // DepsdevAPI is the URL to the deps.dev API. It is documented at // docs.deps.dev/api. +// +// Deprecated: this is now private and should not be used outside the scanner const DepsdevAPI = "api.deps.dev:443" // System maps from a lockfile system to the depsdev API system. +// +// Deprecated: this is now private and should not be used outside the scanner var System = map[lockfile.Ecosystem]depsdevpb.System{ lockfile.NpmEcosystem: depsdevpb.System_NPM, lockfile.NuGetEcosystem: depsdevpb.System_NUGET, @@ -32,6 +37,8 @@ var System = map[lockfile.Ecosystem]depsdevpb.System{ } // VersionQuery constructs a GetVersion request from the arguments. +// +// Deprecated: this is now private and should not be used outside the scanner func VersionQuery(system depsdevpb.System, name string, version string) *depsdevpb.GetVersionRequest { if system == depsdevpb.System_GO { version = "v" + version @@ -47,6 +54,8 @@ func VersionQuery(system depsdevpb.System, name string, version string) *depsdev } // MakeVersionRequests wraps MakeVersionRequestsWithContext using context.Background. +// +// Deprecated: this is now private and should not be used outside the scanner func MakeVersionRequests(queries []*depsdevpb.GetVersionRequest) ([][]models.License, error) { return MakeVersionRequestsWithContext(context.Background(), queries) } @@ -55,6 +64,8 @@ func MakeVersionRequests(queries []*depsdevpb.GetVersionRequest) ([][]models.Lic // query. It makes these requests concurrently, sharing the single HTTP/2 // connection. The order in which the requests are specified should correspond // to the order of licenses returned by this function. +// +// Deprecated: this is now private and should not be used outside the scanner func MakeVersionRequestsWithContext(ctx context.Context, queries []*depsdevpb.GetVersionRequest) ([][]models.License, error) { certPool, err := x509.SystemCertPool() if err != nil { diff --git a/pkg/grouper/grouper.go b/pkg/grouper/grouper.go index b21edaae76..c64399915a 100644 --- a/pkg/grouper/grouper.go +++ b/pkg/grouper/grouper.go @@ -1,3 +1,4 @@ +// Deprecated: this is now private and should not be used outside the scanner package grouper import ( @@ -22,6 +23,8 @@ func hasAliasIntersection(v1, v2 IDAliases) bool { } // Group groups vulnerabilities by aliases. +// +// Deprecated: this is now private and should not be used outside the scanner func Group(vulns []IDAliases) []models.GroupInfo { // Mapping of `vulns` index to a group ID. A group ID is just another index in the `vulns` slice. groups := make([]int, len(vulns)) diff --git a/pkg/grouper/grouper_models.go b/pkg/grouper/grouper_models.go index b713aafaa1..1b759e74e3 100644 --- a/pkg/grouper/grouper_models.go +++ b/pkg/grouper/grouper_models.go @@ -1,3 +1,4 @@ +// Deprecated: this is now private and should not be used outside the scanner package grouper import ( @@ -6,11 +7,13 @@ import ( "github.com/google/osv-scanner/pkg/models" ) +// Deprecated: this is now private and should not be used outside the scanner type IDAliases struct { ID string Aliases []string } +// Deprecated: this is now private and should not be used outside the scanner func ConvertVulnerabilityToIDAliases(c []models.Vulnerability) []IDAliases { output := []IDAliases{} for _, v := range c { diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index f48f7d3313..eb050408f6 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -14,7 +14,9 @@ import ( "sort" "strings" + "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/customgitignore" + "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/internal/image" "github.com/google/osv-scanner/internal/local" "github.com/google/osv-scanner/internal/manifest" @@ -24,8 +26,6 @@ import ( "github.com/google/osv-scanner/internal/sbom" "github.com/google/osv-scanner/internal/semantic" "github.com/google/osv-scanner/internal/version" - "github.com/google/osv-scanner/pkg/config" - "github.com/google/osv-scanner/pkg/depsdev" "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" diff --git a/pkg/osvscanner/osvscanner_internal_test.go b/pkg/osvscanner/osvscanner_internal_test.go index 1acab47369..8b53753e2d 100644 --- a/pkg/osvscanner/osvscanner_internal_test.go +++ b/pkg/osvscanner/osvscanner_internal_test.go @@ -6,8 +6,8 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/testutility" - "github.com/google/osv-scanner/pkg/config" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/reporter" ) diff --git a/pkg/osvscanner/vulnerability_result.go b/pkg/osvscanner/vulnerability_result.go index d41f5700e2..aa837ebb66 100644 --- a/pkg/osvscanner/vulnerability_result.go +++ b/pkg/osvscanner/vulnerability_result.go @@ -5,10 +5,10 @@ import ( "sort" "strings" + "github.com/google/osv-scanner/internal/config" + "github.com/google/osv-scanner/internal/grouper" "github.com/google/osv-scanner/internal/output" "github.com/google/osv-scanner/internal/sourceanalysis" - "github.com/google/osv-scanner/pkg/config" - "github.com/google/osv-scanner/pkg/grouper" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" "github.com/google/osv-scanner/pkg/reporter" diff --git a/pkg/osvscanner/vulnerability_result_internal_test.go b/pkg/osvscanner/vulnerability_result_internal_test.go index 2a3959e14b..7b1bcfa65f 100644 --- a/pkg/osvscanner/vulnerability_result_internal_test.go +++ b/pkg/osvscanner/vulnerability_result_internal_test.go @@ -3,8 +3,8 @@ package osvscanner import ( "testing" + "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/testutility" - "github.com/google/osv-scanner/pkg/config" "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" diff --git a/pkg/spdx/gen.go b/pkg/spdx/gen.go index 27c85b1ee9..8c7daefd31 100644 --- a/pkg/spdx/gen.go +++ b/pkg/spdx/gen.go @@ -38,7 +38,15 @@ func main() { panic(err) } - output := "// Code generated by gen.go. DO NOT EDIT.\n package spdx\nvar IDs = map[string]bool{\n" + output := strings.TrimLeft(` +// Code generated by gen.go. DO NOT EDIT. +// +// Deprecated: this is now private and should not be used outside the scanner +package spdx + +// Deprecated: this is now private and should not be used outside the scanner +var IDs = map[string]bool{ +`, "\n") for _, license := range licenseList.Licenses { output += fmt.Sprintf("%q: true,\n", strings.ToLower(license.SPDXID)) } diff --git a/pkg/spdx/licenses.go b/pkg/spdx/licenses.go index 9395d46f91..c389b69086 100644 --- a/pkg/spdx/licenses.go +++ b/pkg/spdx/licenses.go @@ -1,6 +1,9 @@ // Code generated by gen.go. DO NOT EDIT. +// +// Deprecated: this is now private and should not be used outside the scanner package spdx +// Deprecated: this is now private and should not be used outside the scanner var IDs = map[string]bool{ "0bsd": true, "3d-slicer-1.0": true, diff --git a/pkg/spdx/verify.go b/pkg/spdx/verify.go index 80aa5b5244..df36e621fc 100644 --- a/pkg/spdx/verify.go +++ b/pkg/spdx/verify.go @@ -1,9 +1,12 @@ +// Deprecated: this is now private and should not be used outside the scanner package spdx import "strings" // Unrecognized filters licenses for non-spdx identifiers. The "unknown" string is // also treated as a valid identifier. +// +// Deprecated: this is now private and should not be used outside the scanner func Unrecognized(licenses []string) (unrecognized []string) { for _, license := range licenses { l := strings.ToLower(license) diff --git a/scripts/generate_mock_resolution_universe/main.go b/scripts/generate_mock_resolution_universe/main.go index bb6191a1c1..ad1082db3a 100644 --- a/scripts/generate_mock_resolution_universe/main.go +++ b/scripts/generate_mock_resolution_universe/main.go @@ -22,6 +22,7 @@ import ( pb "deps.dev/api/v3" "deps.dev/util/resolve" "deps.dev/util/resolve/dep" + "github.com/google/osv-scanner/internal/depsdev" "github.com/google/osv-scanner/internal/remediation" "github.com/google/osv-scanner/internal/remediation/upgrade" "github.com/google/osv-scanner/internal/resolution" @@ -30,7 +31,6 @@ import ( "github.com/google/osv-scanner/internal/resolution/lockfile" "github.com/google/osv-scanner/internal/resolution/manifest" "github.com/google/osv-scanner/internal/resolution/util" - "github.com/google/osv-scanner/pkg/depsdev" lf "github.com/google/osv-scanner/pkg/lockfile" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv"