From 0942add07834b0466a1b801d2caf9ad56449da89 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 31 Jul 2024 00:58:23 -0400 Subject: [PATCH] chore: add grype version to application update check headers Signed-off-by: Keith Zantow --- cmd/grype/cli/commands/db_check.go | 5 +- grype/db/curator.go | 99 +++++++++++++++---- grype/db/curator_test.go | 2 +- grype/db/listing.go | 8 +- grype/db/metadata.go | 14 ++- grype/db/metadata_test.go | 17 +++- grype/db/status.go | 1 + .../metadata-updated/metadata.json | 7 ++ 8 files changed, 128 insertions(+), 25 deletions(-) create mode 100644 grype/db/test-fixtures/metadata-updated/metadata.json diff --git a/cmd/grype/cli/commands/db_check.go b/cmd/grype/cli/commands/db_check.go index 5f34b8312b6..5ba1ab2258e 100644 --- a/cmd/grype/cli/commands/db_check.go +++ b/cmd/grype/cli/commands/db_check.go @@ -35,12 +35,13 @@ func runDBCheck(opts options.Database) error { return err } - updateAvailable, currentDBMetadata, updateDBEntry, err := dbCurator.IsUpdateAvailable() + currentDBMetadata := dbCurator.GetMetadata() + updateDBEntry, err := dbCurator.GetUpdate(currentDBMetadata) if err != nil { return fmt.Errorf("unable to check for vulnerability database update: %+v", err) } - if !updateAvailable { + if updateDBEntry == nil { return stderrPrintLnf("No update available") } diff --git a/grype/db/curator.go b/grype/db/curator.go index 34d2cd92c40..252312dfaba 100644 --- a/grype/db/curator.go +++ b/grype/db/curator.go @@ -5,6 +5,7 @@ import ( "crypto/x509" "fmt" "net/http" + "net/url" "os" "path" "strconv" @@ -43,6 +44,7 @@ type Config struct { type Curator struct { fs afero.Fs + listingClient *http.Client listingDownloader file.Getter updateDownloader file.Getter targetSchema int @@ -73,6 +75,7 @@ func NewCurator(cfg Config) (Curator, error) { return Curator{ fs: fs, targetSchema: vulnerability.SchemaVersion, + listingClient: listingClient, listingDownloader: file.NewGetter(listingClient), updateDownloader: file.NewGetter(dbClient), dbDir: dbDir, @@ -114,6 +117,7 @@ func (c *Curator) Status() Status { return Status{ Built: metadata.Built, + Updated: metadata.Updated, SchemaVersion: metadata.Version, Location: c.dbDir, Checksum: metadata.Checksum, @@ -148,13 +152,14 @@ func (c *Curator) Update() (bool, error) { defer downloadProgress.SetCompleted() defer importProgress.SetCompleted() - updateAvailable, metadata, updateEntry, err := c.IsUpdateAvailable() + metadata := c.GetMetadata() + updateEntry, err := c.GetUpdate(metadata) if err != nil { // we want to continue if possible even if we can't check for an update log.Warnf("unable to check for vulnerability database update") log.Debugf("check for vulnerability update failed: %+v", err) } - if updateAvailable { + if updateEntry != nil { log.Infof("downloading new vulnerability DB") err = c.UpdateTo(updateEntry, downloadProgress, importProgress, stage) if err != nil { @@ -184,35 +189,82 @@ func (c *Curator) Update() (bool, error) { return false, nil } -// IsUpdateAvailable indicates if there is a new update available as a boolean, and returns the latest listing information -// available for this schema. -func (c *Curator) IsUpdateAvailable() (bool, *Metadata, *ListingEntry, error) { +// GetMetadata returns the current metadata or nil if unable to find or read metadata +func (c *Curator) GetMetadata() *Metadata { + metadata, err := NewMetadataFromDir(c.fs, c.dbDir) + if err != nil { + log.Debugf("current metadata corrupt: %w", err) + } + return metadata +} + +// UpdateMetadataTimestamp updates the metadata file with the current timestamp +func (c *Curator) updateMetadataTimestamp() error { + metadata, err := NewMetadataFromDir(c.fs, c.dbDir) + if err != nil || metadata == nil { + return err + } + // update the check time + metadata.Updated = time.Now() + return metadata.Write(metadataPath(c.dbDir)) +} + +// GetUpdate returns an available update if one is available or an error if an error occurred while checking +func (c *Curator) GetUpdate(current *Metadata) (*ListingEntry, error) { log.Debugf("checking for available database updates") - listing, err := c.ListingFromURL() + u, err := url.Parse(c.listingURL) if err != nil { - return false, nil, nil, err + return nil, fmt.Errorf("invalid URL: %v %v", c.listingURL, err) } - updateEntry := listing.BestUpdate(c.targetSchema) - if updateEntry == nil { - return false, nil, nil, fmt.Errorf("no db candidates with correct version available (maybe there is an application update available?)") + headers := http.Header{} + + s := c.Status() + if s.Err == nil { + // valid status, get the db update time + headers.Add("If-Modified-Since", s.Updated.UTC().Format(http.TimeFormat)) + } + + req := http.Request{ + Method: http.MethodGet, + URL: u, + Header: headers, + } + + resp, err := c.listingClient.Do(&req) + if err != nil { + return nil, fmt.Errorf("error attempting to check for update: %w", err) + } + defer func() { + err := resp.Body.Close() + if err != nil { + log.Debug(err) + } + }() + + if s.Err == nil && resp.StatusCode == http.StatusNotModified { + return nil, nil } - log.Debugf("found database update candidate: %s", updateEntry) - // compare created data to current db date - current, err := NewMetadataFromDir(c.fs, c.dbDir) + listing, err := NewListingFromReader(resp.Body) if err != nil { - return false, nil, nil, fmt.Errorf("current metadata corrupt: %w", err) + return nil, fmt.Errorf("unable to parse db listing: %v", err) } - if current.IsSupersededBy(updateEntry) { + updateEntry := listing.BestUpdate(c.targetSchema) + if updateEntry == nil { + return nil, fmt.Errorf("no db candidates with correct version available (maybe there is an application update available?)") + } + log.Debugf("found database update candidate: %s", updateEntry) + + if current == nil || current.IsSupersededBy(updateEntry) { log.Debugf("database update available: %s", updateEntry) - return true, current, updateEntry, nil + return updateEntry, nil } log.Debugf("no database update available") - return false, nil, nil, nil + return nil, nil } // UpdateTo updates the existing DB with the specific other version provided from a listing entry. @@ -369,7 +421,18 @@ func (c *Curator) activate(dbDirPath string) error { } // activate the new db cache - return file.CopyDir(c.fs, dbDirPath, c.dbDir) + err = file.CopyDir(c.fs, dbDirPath, c.dbDir) + if err != nil { + return err + } + + // update the timestamp indicating when this db was downloaded + err = c.updateMetadataTimestamp() + if err != nil { + log.Debugf("unable to update metadata: %v", err) + } + + return nil } // ListingFromURL loads a Listing from a URL. diff --git a/grype/db/curator_test.go b/grype/db/curator_test.go index c652b1ef78a..4da6b942a0a 100644 --- a/grype/db/curator_test.go +++ b/grype/db/curator_test.go @@ -415,7 +415,7 @@ func TestCuratorTimeoutBehavior(t *testing.T) { stage := progress.NewAtomicStage("some-stage") runTheTest := func(success chan struct{}, errs chan error) { - _, _, _, err = curator.IsUpdateAvailable() + _, err = curator.GetUpdate(curator.GetMetadata()) if err == nil { errs <- errors.New("expected timeout error but got nil") return diff --git a/grype/db/listing.go b/grype/db/listing.go index 3930a736846..2f1de3eef49 100644 --- a/grype/db/listing.go +++ b/grype/db/listing.go @@ -3,6 +3,7 @@ package db import ( "encoding/json" "fmt" + "io" "os" "sort" @@ -48,8 +49,13 @@ func NewListingFromFile(fs afero.Fs, path string) (Listing, error) { } defer f.Close() + return NewListingFromReader(f) +} + +// NewListingFromReader loads a Listing from a given filepath. +func NewListingFromReader(reader io.Reader) (Listing, error) { var l Listing - err = json.NewDecoder(f).Decode(&l) + err := json.NewDecoder(reader).Decode(&l) if err != nil { return Listing{}, fmt.Errorf("unable to parse DB listing: %w", err) } diff --git a/grype/db/metadata.go b/grype/db/metadata.go index 60fc6702583..327917159d3 100644 --- a/grype/db/metadata.go +++ b/grype/db/metadata.go @@ -19,13 +19,15 @@ const MetadataFileName = "metadata.json" // verify the contents (checksum). type Metadata struct { Built time.Time + Updated time.Time Version int Checksum string } // MetadataJSON is a helper struct for parsing and assembling Metadata objects to and from JSON. type MetadataJSON struct { - Built string `json:"built"` // RFC 3339 + Built string `json:"built"` // RFC 3339 + Updated string `json:"updated"` // RFC 3339 Version int `json:"version"` Checksum string `json:"checksum"` } @@ -37,8 +39,16 @@ func (m MetadataJSON) ToMetadata() (Metadata, error) { return Metadata{}, fmt.Errorf("cannot convert built time (%s): %+v", m.Built, err) } + updated, err := time.Parse(time.RFC3339, m.Updated) + if err != nil { + // database build + delay to update when last modified time occurs is ~1 hour, + // so we can use the build time as a reasonable default + updated = build.Add(1 * time.Hour) + } + metadata := Metadata{ Built: build.UTC(), + Updated: updated.UTC(), Version: m.Version, Checksum: m.Checksum, } @@ -71,6 +81,7 @@ func NewMetadataFromDir(fs afero.Fs, dir string) (*Metadata, error) { if err != nil { return nil, fmt.Errorf("unable to parse DB metadata (%s): %w", metadataFilePath, err) } + return &m, nil } @@ -120,6 +131,7 @@ func (m Metadata) String() string { func (m Metadata) Write(toPath string) error { metadata := MetadataJSON{ Built: m.Built.UTC().Format(time.RFC3339), + Updated: m.Updated.UTC().Format(time.RFC3339), Version: m.Version, Checksum: m.Checksum, } diff --git a/grype/db/metadata_test.go b/grype/db/metadata_test.go index bd94cb65681..e883caf4095 100644 --- a/grype/db/metadata_test.go +++ b/grype/db/metadata_test.go @@ -9,6 +9,8 @@ import ( ) func TestMetadataParse(t *testing.T) { + timeUTC := time.Date(2020, 06, 15, 14, 02, 36, 0, time.UTC) + timeEDT := time.Date(2020, 06, 15, 18, 02, 36, 0, time.UTC) tests := []struct { fixture string expected *Metadata @@ -17,7 +19,8 @@ func TestMetadataParse(t *testing.T) { { fixture: "test-fixtures/metadata-gocase", expected: &Metadata{ - Built: time.Date(2020, 06, 15, 14, 02, 36, 0, time.UTC), + Built: timeUTC, + Updated: timeUTC.Add(1 * time.Hour), Version: 2, Checksum: "sha256:dcd6a285c839a7c65939e20c251202912f64826be68609dfc6e48df7f853ddc8", }, @@ -25,7 +28,17 @@ func TestMetadataParse(t *testing.T) { { fixture: "test-fixtures/metadata-edt-timezone", expected: &Metadata{ - Built: time.Date(2020, 06, 15, 18, 02, 36, 0, time.UTC), + Built: timeEDT, + Updated: timeEDT, + Version: 2, + Checksum: "sha256:dcd6a285c839a7c65939e20c251202912f64826be68609dfc6e48df7f853ddc8", + }, + }, + { + fixture: "test-fixtures/metadata-updated", + expected: &Metadata{ + Built: timeEDT, + Updated: timeUTC, Version: 2, Checksum: "sha256:dcd6a285c839a7c65939e20c251202912f64826be68609dfc6e48df7f853ddc8", }, diff --git a/grype/db/status.go b/grype/db/status.go index 7f90ab98ccb..e0d4f24b0d1 100644 --- a/grype/db/status.go +++ b/grype/db/status.go @@ -4,6 +4,7 @@ import "time" type Status struct { Built time.Time `json:"built"` + Updated time.Time `json:"updated"` SchemaVersion int `json:"schemaVersion"` Location string `json:"location"` Checksum string `json:"checksum"` diff --git a/grype/db/test-fixtures/metadata-updated/metadata.json b/grype/db/test-fixtures/metadata-updated/metadata.json new file mode 100644 index 00000000000..c65016bd00e --- /dev/null +++ b/grype/db/test-fixtures/metadata-updated/metadata.json @@ -0,0 +1,7 @@ +{ + "built": "2020-06-15T14:02:36-04:00", + "updated": "2020-06-15T14:02:36Z", + "last-check": "2020-06-15T14:02:36-04:00", + "version": 2, + "checksum": "sha256:dcd6a285c839a7c65939e20c251202912f64826be68609dfc6e48df7f853ddc8" +}