feat: Add support for analysing and scanning docker and oci archives (#…

…505)

* Add support for analysing and scanning docker and oci archives

This commit adds support for scanning docker archives (such as those
obtained through docker save) and oci archives (such as those obtained
through skopeo copy).

Trivy, Syft SBOM analyzers as well as the Trivy and Grype vulnerability
scanners have been updated to support this new input type.

* refactor: switch to stereoscope for untaring and add support for oci-dir

* testing(e2e): add docker archive analyze and scan in e2e tests

* docs(e2e): fix comment in CLI docker archive test

* fix(e2e): remove uploaded sboms and add them to gitignore

.gitignore

@@ -11,4 +11,7 @@ site/
 e2e/dir.sbom
 e2e/kubeclarity-cli
 e2e/merged.sbom
+e2e/docker-archive.sbom
+e2e/missingmeta.sbom
+e2e/nocomponents.sbom
 fanal/

Makefile

@@ -262,4 +262,4 @@ gomod-tidy:
 .PHONY: e2e
 e2e:
 	@echo "Running e2e tests ..."
-	cd e2e && export DOCKER_TAG=${DOCKER_TAG} && go test -v .
+	cd e2e && export DOCKER_TAG=${DOCKER_TAG} && go test -timeout 20m -v .

cli/cmd/analyze.go

@@ -57,8 +57,8 @@ func init() {
 	analyzeCmd.Flags().StringP("output", "o", "",
 		"set output (default: stdout)")
 	analyzeCmd.Flags().StringP("input-type", "i", "",
-		fmt.Sprintf("set input type (input type can be %s,%s,%s default:%s)",
-			sharedutils.DIR, sharedutils.FILE, sharedutils.IMAGE, sharedutils.IMAGE))
+		fmt.Sprintf("set input type (input type can be %s,%s,%s,%s,%s,%s,%s default:%s)",
+			sharedutils.SBOM, sharedutils.DIR, sharedutils.FILE, sharedutils.IMAGE, sharedutils.DOCKERARCHIVE, sharedutils.OCIARCHIVE, sharedutils.OCIDIR, sharedutils.IMAGE))
 	analyzeCmd.Flags().String("application-id", "",
 		"ID of a defined application to associate the exported analysis")
 	analyzeCmd.Flags().BoolP("export", "e", false,

cli/cmd/scan.go

@@ -70,8 +70,8 @@ func init() {
 		"file to write the report output to (default is STDOUT)",
 	)
 	scanCmd.Flags().StringP("input-type", "i", "",
-		fmt.Sprintf("set input type (input type can be %s,%s,%s,%s default:%s)",
-			sharedutils.SBOM, sharedutils.DIR, sharedutils.FILE, sharedutils.IMAGE, sharedutils.IMAGE))
+		fmt.Sprintf("set input type (input type can be %s,%s,%s,%s,%s,%s,%s default:%s)",
+			sharedutils.SBOM, sharedutils.DIR, sharedutils.FILE, sharedutils.IMAGE, sharedutils.DOCKERARCHIVE, sharedutils.OCIARCHIVE, sharedutils.OCIDIR, sharedutils.IMAGE))
 	scanCmd.Flags().String("application-id", "",
 		"ID of a defined application to associate the exported vulnerability scan")
 	scanCmd.Flags().BoolP("export", "e", false,

cli/pkg/analyzer/export/export.go

@@ -81,7 +81,7 @@ func createPackagesContentAnalysis(m *analyzer.MergedResults) []*models.PackageC
 
 func getResourceType(m *analyzer.MergedResults) models.ResourceType {
 	switch m.Source {
-	case utils.IMAGE:
+	case utils.IMAGE, utils.DOCKERARCHIVE, utils.OCIARCHIVE, utils.OCIDIR:
 		return models.ResourceTypeIMAGE
 	case utils.DIR:
 		return models.ResourceTypeDIRECTORY

cli/pkg/utils/hash.go

@@ -35,7 +35,7 @@ func GenerateHash(inputType utils.SourceType, source string) (string, error) {
 		return "", fmt.Errorf("failed to get absolute path of the source %s: %v", source, err)
 	}
 	switch inputType {
-	case utils.IMAGE:
+	case utils.IMAGE, utils.DOCKERARCHIVE, utils.OCIARCHIVE, utils.OCIDIR:
 		log.Infof("Skip generating hash in the case of image")
 		return "", nil
 	case utils.DIR, utils.ROOTFS:

e2e/cli_scan_test.go

@@ -40,6 +40,9 @@ const (
 	TestImageName                  = "nginx:1.10"
 	ApplicationName                = "test-app"
 
+	DockerArchiveApplicationName   = "test-app-docker-archive"
+	DockerArchiveOutputSBOMFile    = "docker-archive.sbom"
+
 	TestImageWithMissingSyftMetadata      = "docker.io/weaveworksdemos/front-end:sha-14254f9"
 	MissingMetaImageAnalyzeOutputSBOMFile = "missingmeta.sbom"
 	MissingMetaApplicationName            = "test-app-missingm"
@@ -78,7 +81,7 @@ func TestCLIScan(t *testing.T) {
 
 			// analyze image with --merge-sbom directory sbom, and export to backend
 			t.Logf("analyze image...")
-			analyzeImage(t, DirectoryAnalyzeOutputSBOMFile, appID, TestImageName, ImageAnalyzeOutputSBOMFile)
+			analyzeImage(t, DirectoryAnalyzeOutputSBOMFile, appID, TestImageName, "image", ImageAnalyzeOutputSBOMFile)
 			time.Sleep(common.WaitForMaterializedViewRefreshSecond * time.Second)
 			validateAnalyzeImage(t, ImageAnalyzeOutputSBOMFile, appID)
 
@@ -90,9 +93,9 @@ func TestCLIScan(t *testing.T) {
 
 			// scan image
 			t.Logf("scan image...")
-			scanImage(t, TestImageName, appID)
+			scanImage(t, TestImageName, "image", appID)
 			time.Sleep(common.WaitForMaterializedViewRefreshSecond * time.Second)
-			validateScanImage(t, appID)
+			validateScanImage(t, appID, true)
 
 			return ctx
 		}).
@@ -103,7 +106,7 @@ func TestCLIScan(t *testing.T) {
 
 			// analyze "bad" image
 			t.Logf("analyze image...")
-			analyzeImage(t, "", appID, TestImageWithMissingSyftMetadata, MissingMetaImageAnalyzeOutputSBOMFile)
+			analyzeImage(t, "", appID, TestImageWithMissingSyftMetadata, "image", MissingMetaImageAnalyzeOutputSBOMFile)
 			time.Sleep(common.WaitForMaterializedViewRefreshSecond * time.Second)
 			validateAnalyzeImage(t, MissingMetaImageAnalyzeOutputSBOMFile, appID)
 
@@ -122,7 +125,7 @@ func TestCLIScan(t *testing.T) {
 
 			// analyze "bad" image
 			t.Logf("analyze image...")
-			analyzeImage(t, "", appID, TestImageWithNoComponents, NoComponentsImageAnalyzeOutputSBOMFile)
+			analyzeImage(t, "", appID, TestImageWithNoComponents, "image", NoComponentsImageAnalyzeOutputSBOMFile)
 			time.Sleep(common.WaitForMaterializedViewRefreshSecond * time.Second)
 
 			// check generated sbom is a valid cyclonedx even
@@ -142,6 +145,45 @@ func TestCLIScan(t *testing.T) {
 			vuls := common.GetVulnerabilities(t, kubeclarityAPI, appID)
 			assert.Assert(t, *vuls.Total == 0)
 
+			return ctx
+		}).
+		Assess("cli scan flow - docker archive image", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			// create application
+			t.Logf("create application...")
+			appID := createApplication(t, DockerArchiveApplicationName)
+
+			tmpDir, err := os.MkdirTemp("", "")
+			if err != nil {
+				t.Fatalf("unable to make temporary directory")
+			}
+			defer func() {
+				err := os.RemoveAll(tmpDir)
+				if err != nil {
+					t.Logf("unable to remove temp directory: %v", err)
+				}
+			}()
+
+			outputFile := filepath.Join(tmpDir, "image.tar")
+
+			command := fmt.Sprintf("docker pull %s && docker image save %s -o %s", TestImageName, TestImageName, outputFile)
+			cmd := exec.Command("/bin/sh", "-c", command)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				t.Fatalf("docker save failed: %v, %s", err, out)
+			}
+
+			// analyze image and export to backend
+			t.Logf("analyze image...")
+			analyzeImage(t, "", appID, outputFile, "docker-archive", DockerArchiveOutputSBOMFile)
+			time.Sleep(common.WaitForMaterializedViewRefreshSecond * time.Second)
+			validateAnalyzeImage(t, DockerArchiveOutputSBOMFile, appID)
+
+			// scan image
+			t.Logf("scan image...")
+			scanImage(t, outputFile, "docker-archive", appID)
+			time.Sleep(common.WaitForMaterializedViewRefreshSecond * time.Second)
+			validateScanImage(t, appID, false)
+
 			return ctx
 		}).Feature()
 
@@ -180,16 +222,18 @@ func validateAnalyzeImage(t *testing.T, sbomFile, appID string) {
 	assert.Assert(t, *appResources.Total > 0)
 }
 
-func validateScanImage(t *testing.T, appID string) {
+func validateScanImage(t *testing.T, appID string, cis bool) {
 	t.Helper()
 	vuls := common.GetVulnerabilities(t, kubeclarityAPI, appID)
 	assert.Assert(t, *vuls.Total > 0)
 
 	appResources := common.GetApplicationResources(t, kubeclarityAPI, appID)
 	assert.Assert(t, appResources.Items[0].ResourceType == models.ResourceTypeIMAGE)
 
-	cisDockerBenchmarkResults := common.GetCISDockerBenchmarkResults(t, kubeclarityAPI, appResources.Items[0].ID)
-	assert.Assert(t, *cisDockerBenchmarkResults.Total > 0)
+	if cis {
+		cisDockerBenchmarkResults := common.GetCISDockerBenchmarkResults(t, kubeclarityAPI, appResources.Items[0].ID)
+		assert.Assert(t, *cisDockerBenchmarkResults.Total > 0)
+	}
 }
 
 func validateScanSBOM(t *testing.T, appID string) {
@@ -229,12 +273,12 @@ func analyzeDir(t *testing.T) {
 var cliPath = filepath.Join(common.GetCurrentDir(), "kubeclarity-cli")
 
 // analyze test image, merge inputSbom and export to backend
-func analyzeImage(t *testing.T, inputSbom string, appID string, image string, outputfile string) {
+func analyzeImage(t *testing.T, inputSbom string, appID string, image string, inputType string, outputfile string) {
 	t.Helper()
 	assert.NilError(t, os.Setenv("BACKEND_HOST", "localhost:"+common.KubeClarityPortForwardHostPort))
 	assert.NilError(t, os.Setenv("BACKEND_DISABLE_TLS", "true"))
 
-	command := fmt.Sprintf("%v analyze %v --application-id %v --input-type image", cliPath, image, appID)
+	command := fmt.Sprintf("%v analyze %v --application-id %v --input-type %s", cliPath, image, appID, inputType)
 
 	if inputSbom != "" {
 		command = fmt.Sprintf("%s --merge-sbom %v", command, inputSbom)
@@ -265,12 +309,12 @@ func scanSBOM(t *testing.T, inputSbom string, appID string) {
 	}
 }
 
-func scanImage(t *testing.T, image string, appID string) {
+func scanImage(t *testing.T, image string, inputType string, appID string) {
 	t.Helper()
 	assert.NilError(t, os.Setenv("BACKEND_HOST", "localhost:"+common.KubeClarityPortForwardHostPort))
 	assert.NilError(t, os.Setenv("BACKEND_DISABLE_TLS", "true"))
 
-	command := fmt.Sprintf("%v scan %v --application-id %v --input-type image --cis-docker-benchmark-scan -e", cliPath, image, appID)
+	command := fmt.Sprintf("%v scan %v --application-id %v --input-type %s --cis-docker-benchmark-scan -e", cliPath, image, appID, inputType)
 
 	cmd := exec.Command("/bin/sh", "-c", command)
 

e2e/common/utils.go

@@ -113,7 +113,7 @@ func PortForwardToKubeClarity(stopCh chan struct{}) {
 	go func() {
 		err := portForward("service", KubeClarityNamespace, KubeClarityServiceName, KubeClarityPortForwardHostPort, KubeClarityPortForwardTargetPort, stopCh)
 		if err != nil {
-			println("port forward failed. %v", err)
+			fmt.Printf("port forward failed: %v\n", err)
 			return
 		}
 	}()

shared/pkg/analyzer/merge.go

@@ -413,7 +413,7 @@ func toBomDescriptorComponent(sourceType utils.SourceType, srcMetadata *cdx.Meta
 	metaDataComponent := srcMetadata.Component
 
 	switch sourceType {
-	case utils.IMAGE:
+	case utils.IMAGE, utils.DOCKERARCHIVE, utils.OCIARCHIVE, utils.OCIDIR:
 		metaDataComponent.Type = cdx.ComponentTypeContainer
 	case utils.DIR, utils.FILE, utils.ROOTFS:
 		metaDataComponent.Type = cdx.ComponentTypeFile

shared/pkg/analyzer/trivy/trivy.go

@@ -75,7 +75,7 @@ func (a *Analyzer) Run(sourceType utils.SourceType, userInput string) error {
 
 		// Skip this analyser for input types we don't support
 		switch sourceType {
-		case utils.IMAGE, utils.ROOTFS, utils.DIR, utils.FILE:
+		case utils.IMAGE, utils.ROOTFS, utils.DIR, utils.FILE, utils.DOCKERARCHIVE, utils.OCIARCHIVE, utils.OCIDIR:
 			// These are all supported for SBOM analysing so continue
 		case utils.SBOM:
 			fallthrough
@@ -120,6 +120,14 @@ func (a *Analyzer) Run(sourceType utils.SourceType, userInput string) error {
 			return
 		}
 
+		// Configure Trivy image options according to the source type and user input.
+		trivyOptions, cleanup, err := utilsTrivy.SetTrivyImageOptions(sourceType, userInput, trivyOptions)
+		defer cleanup(a.logger)
+		if err != nil {
+			a.setError(res, fmt.Errorf("failed to configure trivy image options: %w", err))
+			return
+		}
+
 		// Ensure we're configured for private registry if required
 		trivyOptions = utilsTrivy.SetTrivyRegistryConfigs(a.config.Registry, trivyOptions)
 

shared/pkg/scanner/trivy/scanner.go

@@ -165,6 +165,7 @@ func (a *Scanner) createTrivyOptions(output string, userInput string) (trivyFlag
 	return trivyOptions, nil
 }
 
+// nolint:cyclop
 func (a *Scanner) Run(sourceType utils.SourceType, userInput string) error {
 	a.logger.Infof("Called %s scanner on source %v %v", ScannerName, sourceType, userInput)
 
@@ -178,7 +179,7 @@ func (a *Scanner) Run(sourceType utils.SourceType, userInput string) error {
 
 		var hash string
 		switch sourceType {
-		case utils.IMAGE, utils.ROOTFS, utils.DIR, utils.FILE:
+		case utils.IMAGE, utils.ROOTFS, utils.DIR, utils.FILE, utils.DOCKERARCHIVE, utils.OCIARCHIVE, utils.OCIDIR:
 		case utils.SBOM:
 			var err error
 			_, hash, err = utilsSBOM.GetTargetNameAndHashFromSBOM(userInput)
@@ -198,6 +199,14 @@ func (a *Scanner) Run(sourceType utils.SourceType, userInput string) error {
 			return
 		}
 
+		// Configure Trivy image options according to the source type and user input.
+		trivyOptions, cleanup, err := utilsTrivy.SetTrivyImageOptions(sourceType, userInput, trivyOptions)
+		defer cleanup(a.logger)
+		if err != nil {
+			a.setError(fmt.Errorf("failed to configure trivy image options: %w", err))
+			return
+		}
+
 		// Convert the kubeclarity source to the trivy source type
 		trivySourceType, err := utilsTrivy.KubeclaritySourceToTrivySource(sourceType)
 		if err != nil {

shared/pkg/utils/input.go

@@ -20,11 +20,14 @@ import "fmt"
 type SourceType string
 
 const (
-	SBOM   SourceType = "sbom"
-	IMAGE  SourceType = "image"
-	DIR    SourceType = "dir"
-	ROOTFS SourceType = "rootfs"
-	FILE   SourceType = "file"
+	SBOM          SourceType = "sbom"
+	IMAGE         SourceType = "image"
+	DOCKERARCHIVE SourceType = "docker-archive"
+	OCIARCHIVE    SourceType = "oci-archive"
+	OCIDIR        SourceType = "oci-dir"
+	DIR           SourceType = "dir"
+	ROOTFS        SourceType = "rootfs"
+	FILE          SourceType = "file"
 )
 
 func ValidateInputType(inputType string) (SourceType, error) {
@@ -33,6 +36,12 @@ func ValidateInputType(inputType string) (SourceType, error) {
 		return SBOM, nil
 	case "image", "IMAGE", "":
 		return IMAGE, nil
+	case "docker-archive":
+		return DOCKERARCHIVE, nil
+	case "oci-archive":
+		return OCIARCHIVE, nil
+	case "oci-dir":
+		return OCIDIR, nil
 	case "dir", "DIR", "directory":
 		return DIR, nil
 	case "file", "FILE":
@@ -48,6 +57,12 @@ func CreateSource(sourceType SourceType, src string, localImage bool) string {
 	switch sourceType {
 	case IMAGE:
 		return setImageSource(localImage, src)
+	case DOCKERARCHIVE:
+		return fmt.Sprintf("docker-archive:%s", src)
+	case OCIARCHIVE:
+		return fmt.Sprintf("oci-archive:%s", src)
+	case OCIDIR:
+		return fmt.Sprintf("oci-dir:%s", src)
 	case ROOTFS, DIR:
 		return fmt.Sprintf("%s:%s", DIR, src)
 	case FILE, SBOM: