Skip to content

Commit

Permalink
fix: ensure async flow waits for image analysis and handles anchore 404
Browse files Browse the repository at this point in the history 
Change the ordering of the async logic to ensure that scan is created
and image analysis is complete before starting the build of the vuln
report.

Add a lock via mutex to the result store to prevent concurrent
read/writes to the map.

Signed-off-by: Bradley Jones <[email protected]>
  • Loading branch information
@bradleyjones
bradleyjones committed Nov 3, 2023
1 parent 8101eb2 commit 2330f3b
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 76 deletions.
3 changes: 2 additions & 1 deletion .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"from": "${workspaceFolder}",
"to": "/go/src/github.com/anchore/harbor-scanner-adapter"
}
]
],
"program": "${workspaceFolder}/cmd/harbor-scanner-adapter/main.go"
}
]
}
60 changes: 40 additions & 20 deletions pkg/adapter/anchore/adapter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,12 +273,22 @@ func (s *HarborScannerAdapter) GetHarborVulnerabilityReport(
Info("no result found in store checking image is analyzed in Anchore Enterprise")
}

if !result.ScanCreated {
log.WithFields(log.Fields{"scanId": scanID, "scanCreated": result.ScanCreated, "resultIsComplete": result.IsComplete, "resultError": result.Error}).
Debug("scan not created yet")
if result.Error != nil {
return nil, result.Error
}
return nil, fmt.Errorf("create scan not ready")
}

if !result.AnalysisComplete {
log.WithFields(log.Fields{"scanId": scanID}).Debug("checking image analysis state in Anchore Enterprise")
imageAnalsisFn := func() (bool, error) {
return IsImageAnalysed(imageDigest, scanID, &s.Configuration.AnchoreClientConfig)
}
resultStore.RequestAnalysisStatus(scanID, imageAnalsisFn)
return nil, fmt.Errorf("result not ready")
}

if result.ReportBuildInProgress {
Expand All @@ -289,17 +299,10 @@ func (s *HarborScannerAdapter) GetHarborVulnerabilityReport(
}
return nil, fmt.Errorf("result not ready")
}
if !result.ScanCreated {
log.WithFields(log.Fields{"scanId": scanID, "scanCreated": result.ScanCreated, "resultIsComplete": result.IsComplete, "resultError": result.Error}).
Debug("scan not created yet")
if result.Error != nil {
return nil, result.Error
}
return nil, fmt.Errorf("create scan not ready")
}

fn := func() (*harbor.VulnerabilityReport, error) {
rep, err := BuildHarborVulnerabilityReport(
scanID,
imageRepository,
imageDigest,
includeDescriptions,
Expand All @@ -314,7 +317,7 @@ func (s *HarborScannerAdapter) GetHarborVulnerabilityReport(
return &rep, err
}

log.WithField("scanId", scanID).Info("no existing result store entry found for scanId, creating a new one")
log.WithField("scanId", scanID).Info("begin building vulnerability report")
requestResult := resultStore.RequestResult(scanID, fn)
if requestResult.Error != nil {
return nil, requestResult.Error
Expand Down Expand Up @@ -369,7 +372,7 @@ func GetImageState(imageDigest string, clientConfig *client.Config) (ImageState,
}
log.WithField("imageDigest", imageDigest).Debug("no report in cache, generating")

img, err := client.GetImage(clientConfig, imageDigest)
img, err := client.GetImage(clientConfig, imageDigest, 0)
if err != nil {
return NotFound, err
}
Expand All @@ -394,6 +397,7 @@ func GetImageState(imageDigest string, clientConfig *client.Config) (ImageState,

// BuildHarborVulnerabilityReport Construct the harbor-formatted vulnerability report from an analyzed image in Anchore
func BuildHarborVulnerabilityReport(
scanID string,
imageRepository string,
imageDigest string,
includeDescriptions bool,
Expand All @@ -407,7 +411,7 @@ func BuildHarborVulnerabilityReport(
Debug("getting harbor vulnerability report")

start := time.Now()
anchoreVulnResponse, err := GetAnchoreVulnReport(imageDigest, clientConfig, filterVendorIgnoredVulns)
anchoreVulnResponse, err := GetAnchoreVulnReport(scanID, imageDigest, clientConfig, filterVendorIgnoredVulns)
if err != nil {
log.WithFields(log.Fields{"repository": imageRepository, "imageDigest": imageDigest}).
Error("error from vulnerability report api call to Anchore")
Expand Down Expand Up @@ -480,14 +484,17 @@ func BuildHarborVulnerabilityReport(
}

func GetAnchoreVulnReport(
scanID string,
digest string,
clientConfig *client.Config,
filterVendorIgnoredVulns bool,
) (anchore.ImageVulnerabilityReport, error) {
report, err := client.GetImageVulnerabilities(clientConfig, digest, filterVendorIgnoredVulns)
report, err := client.GetImageVulnerabilities(clientConfig, digest, filterVendorIgnoredVulns, 0)
if err == nil {
log.WithField("imageDigest", digest).Debug("caching result report")
log.WithFields(log.Fields{"scanID": scanID, "imageDigest": digest}).Debug("caching result report")
ReportCache.Add(digest, report)
} else {
log.WithFields(log.Fields{"scanID": scanID, "imageDigest": digest, "error": err}).Debug("Error getting image vulnerabilities")
}

return report, err
Expand All @@ -504,22 +511,34 @@ func (s *HarborScannerAdapter) GetRawVulnerabilityReport(scanID string) (harbor.
return harbor.VulnerabilityReport{}, err
}

rawScanID := fmt.Sprintf("%s-raw", scanID) // Used to store just the raw report results in the result store
result, _ := resultStore.PopResult(rawScanID)
rawScanID := fmt.Sprintf("%s-raw", scanID) // Used to store just the raw report results in the rawResult store
rawResult, _ := resultStore.PopResult(rawScanID)
result := resultStore.GetResult(scanID)

if !result.AnalysisComplete {
// Check Scan has been created for the non-report report. This ensures the image is in Anchore Enterprise and submitted for analysis.
if !rawResult.ScanCreated && !result.ScanCreated {
log.WithFields(log.Fields{"scanId": rawScanID, "scanCreated": result.ScanCreated, "resultIsComplete": result.IsComplete, "resultError": result.Error}).
Debug("scan not created yet")
if rawResult.Error != nil {
return nil, result.Error
}
return nil, fmt.Errorf("create scan not ready")
}

if !rawResult.AnalysisComplete {
log.WithFields(log.Fields{"scanId": scanID}).Debug("checking image analysis state in Anchore Enterprise")
imageAnalsisFn := func() (bool, error) {
return IsImageAnalysed(digest, scanID, &s.Configuration.AnchoreClientConfig)
}
resultStore.RequestAnalysisStatus(rawScanID, imageAnalsisFn)
return nil, fmt.Errorf("result not ready")
}

if result.ReportBuildInProgress {
log.WithFields(log.Fields{"scanId": scanID, "resultIsComplete": result.IsComplete, "resultError": result.Error}).
if rawResult.ReportBuildInProgress {
log.WithFields(log.Fields{"scanId": scanID, "resultIsComplete": rawResult.IsComplete, "resultError": rawResult.Error}).
Debug("checked result store for scan Id result")
if result.IsComplete {
return result.RawResult, result.Error
if rawResult.IsComplete {
return rawResult.RawResult, rawResult.Error
}
return nil, fmt.Errorf("result not ready")
}
Expand All @@ -528,6 +547,7 @@ func (s *HarborScannerAdapter) GetRawVulnerabilityReport(scanID string) (harbor.
log.WithFields(log.Fields{"repository": repository, "imageDigest": digest, "scanId": scanID}).
Info("Getting raw Anchore-formatted vulnerability report")
rep, err := GetAnchoreVulnReport(
scanID,
digest,
&s.Configuration.AnchoreClientConfig,
s.Configuration.FullVulnerabilityDescriptions,
Expand Down
41 changes: 39 additions & 2 deletions pkg/adapter/anchore/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ func GetImageVulnerabilities(
clientConfiguration *Config,
digest string,
filterIgnored bool,
retryCount int,
) (anchore.ImageVulnerabilityReport, error) {
log.WithFields(log.Fields{"digest": digest, "filterIgnored": filterIgnored}).Debug("retrieving scan result for image")

Expand All @@ -329,6 +330,7 @@ func GetImageVulnerabilities(
if err != nil {
return anchore.ImageVulnerabilityReport(imageVulnerabilityReportV1), err
}
log.Debug("returning v1 image vulnerability report")
return anchore.ImageVulnerabilityReport(imageVulnerabilityReportV1), nil
}
err := json.Unmarshal(body, &imageVulnerabilityReport)
Expand All @@ -337,10 +339,27 @@ func GetImageVulnerabilities(
}
return imageVulnerabilityReport, nil
}
if resp.StatusCode == 404 {
log.WithFields(log.Fields{"digest": digest}).
Debug("Received 404 getting Image vulnerabilities from Anchore, image not found in Anchore")

// TODO Make the retry count configurable and backoff retries
// Anchore returns 404 if the image is not found, retry up to 5 times
// This is to handle the case where the image is still being submitted for analysis
// and the image is not yet available in the Anchore DB
if retryCount < 5 {
log.WithFields(log.Fields{"digest": digest, "retryCount": retryCount}).
Debug("retrying get image vulnerabilities")
time.Sleep(5 * time.Second)
return GetImageVulnerabilities(clientConfiguration, digest, filterIgnored, retryCount+1)
}

return imageVulnerabilityReport, fmt.Errorf("not found")
}
return imageVulnerabilityReport, fmt.Errorf("error response from anchore api")
}

func GetImage(clientConfiguration *Config, digest string) (anchore.Image, error) {
func GetImage(clientConfiguration *Config, digest string, retryCount int) (anchore.Image, error) {
log.WithFields(log.Fields{"digest": digest}).Debug("retrieving anchore state for image")

var image anchore.Image
Expand All @@ -353,12 +372,30 @@ func GetImage(clientConfiguration *Config, digest string) (anchore.Image, error)

log.WithFields(log.Fields{"method": "get", "url": reqURL}).Debug("sending request to anchore api")
// call API get the full report until "analysis_status" = "analyzed"
_, body, errs := sendRequest(clientConfiguration, request.Get(reqURL))
resp, body, errs := sendRequest(clientConfiguration, request.Get(reqURL))
if errs != nil {
log.Errorf("could not contact anchore api")
return image, errs[0]
}

if resp.StatusCode == 404 {
log.WithFields(log.Fields{"digest": digest}).
Debug("Received 404 getting Image from Anchore, image not found in Anchore")

// TODO Make the retry count configurable and backoff retries
// Anchore returns 404 if the image is not found, retry up to 5 times
// This is to handle the case where the image is still being submitted for analysis
// and the image is not yet available in the Anchore DB
if retryCount < 5 {
log.WithFields(log.Fields{"digest": digest, "retryCount": retryCount}).
Debug("retrying image status check")
time.Sleep(5 * time.Second)
return GetImage(clientConfiguration, digest, retryCount+1)
}

return image, fmt.Errorf("not found")
}

if apiVersion == "v1" {
var imageList anchore.ImageListV1
err = json.Unmarshal(body, &imageList)
Expand Down
Loading

0 comments on commit 2330f3b

Please sign in to comment.