Merge pull request #298 from intelops/bom-new-struct

bom-data

agent/kubviz/k8smetrics_agent.go

@@ -63,24 +63,6 @@ var (
 	schedulingIntervalStr string = os.Getenv("SCHEDULING_INTERVAL")
 )
 
-func runTrivyScans(config *rest.Config, js nats.JetStreamContext) error {
-	err := RunTrivySbomScan(config, js)
-	if err != nil {
-		return err
-	}
-	err = RunTrivyImageScans(config, js)
-	if err != nil {
-		return err
-	}
-	err = RunTrivyK8sClusterScan(js)
-	if err != nil {
-		return err
-	}
-
-	return nil
-
-}
-
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	env := Production
@@ -128,7 +110,11 @@ func main() {
 		err = RakeesOutput(config, js)
 		LogErr(err)
 		// //getK8sEvents(clientset)
-		err = runTrivyScans(config, js)
+		err = RunTrivySbomScan(config, js)
+		LogErr(err)
+		err = RunTrivyImageScans(config, js)
+		LogErr(err)
+		err = RunTrivyK8sClusterScan(js)
 		LogErr(err)
 		err = RunKubeScore(clientset, js)
 		LogErr(err)

agent/kubviz/scheduler_watch.go

@@ -133,6 +133,10 @@ func (v *TrivyJob) CronSpec() string {
 
 func (j *TrivyJob) Run() {
 	// Call the Trivy function with the provided config and js
-	err := runTrivyScans(j.config, j.js)
+	err := RunTrivySbomScan(j.config, j.js)
+	LogErr(err)
+	err = RunTrivyImageScans(j.config, j.js)
+	LogErr(err)
+	err = RunTrivyK8sClusterScan(j.js)
 	LogErr(err)
 }

agent/kubviz/trivy_sbom.go

@@ -17,17 +17,26 @@ import (
 )
 
 func publishTrivySbomReport(report cyclonedx.BOM, js nats.JetStreamContext) error {
-	metrics := model.Sbom{
-		ID:     uuid.New().String(),
-		Report: report,
+	metrics := model.SbomData{
+		ID:               uuid.New().String(),
+		ComponentName:    report.CycloneDX.Metadata.Component.Name,
+		PackageUrl:       report.CycloneDX.Metadata.Component.PackageURL,
+		BomRef:           report.CycloneDX.Metadata.Component.BOMRef,
+		SerialNumber:     report.CycloneDX.SerialNumber,
+		CycloneDxVersion: report.CycloneDX.Version,
+		BomFormat:        report.CycloneDX.BOMFormat,
 	}
-	metricsJson, _ := json.Marshal(metrics)
-	_, err := js.Publish(constants.TRIVY_SBOM_SUBJECT, metricsJson)
+	metricsJson, err := json.Marshal(metrics)
+	if err!=nil {
+		log.Println("error occurred while marshalling sbom metrics in agent", err.Error())
+		return err
+	}
+	_, err = js.Publish(constants.TRIVY_SBOM_SUBJECT, metricsJson)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Trivy report with Id %v has been published\n", metrics.ID)
+	log.Printf("Trivy sbom report with Id %v has been published\n", metrics.ID)
 	return nil
 }
 
@@ -36,27 +45,22 @@ func executeCommandSbom(command string) ([]byte, error) {
 	var outc, errc bytes.Buffer
 	cmd.Stdout = &outc
 	cmd.Stderr = &errc
-
 	err := cmd.Run()
-
 	if err != nil {
 		log.Println("Execute SBOM Command Error", err.Error())
 	}
-
 	return outc.Bytes(), err
 }
 
 func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
+	log.Println("trivy sbom scan started...")
 	pvcMountPath := "/mnt/agent/kbz"
 	trivySbomCacheDir := fmt.Sprintf("%s/trivy-sbomcache", pvcMountPath)
 	err := os.MkdirAll(trivySbomCacheDir, 0755)
 	if err != nil {
 		log.Printf("Error creating Trivy cache directory: %v\n", err)
 		return err
 	}
-	// clearCacheCmd := "trivy image --clear-cache"
-
-	log.Println("trivy sbom run started")
 	images, err := ListImages(config)
 
 	if err != nil {
@@ -71,7 +75,10 @@ func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
 			log.Printf("Error executing Trivy for image sbom %s: %v", image.PullableImage, err)
 			continue // Move on to the next image in case of an error
 		}
-
+		if out == nil {
+			log.Printf("Trivy output is nil for image sbom %s", image.PullableImage)
+			continue
+		}
 		// Check if the output is empty or invalid JSON
 		if len(out) == 0 {
 			log.Printf("Trivy output is empty for image sbom %s", image.PullableImage)
@@ -84,13 +91,6 @@ func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
 			log.Printf("Error unmarshaling JSON data for image sbom %s: %v", image.PullableImage, err)
 			continue // Move on to the next image in case of an error
 		}
-		// log.Println("report", report)
-		// _, err = executeCommandTrivy(clearCacheCmd)
-		// if err != nil {
-		// 	log.Printf("Error executing command: %v\n", err)
-		// 	return err
-		// }
-		// Publish the report using the given function
 		publishTrivySbomReport(report, js)
 	}
 	return nil

client/pkg/clickhouse/db_client.go

@@ -33,7 +33,7 @@ type DBInterface interface {
 	InsertGitEvent(string)
 	InsertKubeScoreMetrics(model.KubeScoreRecommendations)
 	InsertTrivyImageMetrics(metrics model.TrivyImage)
-	InsertTrivySbomMetrics(metrics model.Sbom)
+	InsertTrivySbomMetrics(metrics model.SbomData)
 	InsertTrivyMetrics(metrics model.Trivy)
 	RetriveKetallEvent() ([]model.Resource, error)
 	RetriveOutdatedEvent() ([]model.CheckResultfinal, error)
@@ -685,11 +685,9 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) {
 
 	}
 }
-func (c *DBClient) InsertTrivySbomMetrics(metrics model.Sbom) {
+func (c *DBClient) InsertTrivySbomMetrics(metrics model.SbomData) {
 	log.Println("####started inserting value")
-	result := metrics.Report
 
-	if result.CycloneDX != nil {
 		tx, err := c.conn.Begin()
 		if err != nil {
 			log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
@@ -701,26 +699,19 @@ func (c *DBClient) InsertTrivySbomMetrics(metrics model.Sbom) {
 
 		if _, err := stmt.Exec(
 			metrics.ID,
-			result.CycloneDX.Metadata.Component.Name,
-			result.CycloneDX.Metadata.Component.PackageURL,
-			result.CycloneDX.Metadata.Component.BOMRef,
-			result.CycloneDX.SerialNumber,
-			int32(result.CycloneDX.Version),
-			result.CycloneDX.BOMFormat,
-			result.CycloneDX.Metadata.Component.Version,
-			result.CycloneDX.Metadata.Component.MIMEType,
+			metrics.ComponentName,
+			metrics.PackageUrl,
+			metrics.BomRef,
+			metrics.SerialNumber,
+			int32(metrics.CycloneDxVersion),
+			metrics.BomFormat,
 		); err != nil {
 			log.Fatal(err)
 		}
 		if err := tx.Commit(); err != nil {
 			log.Fatal(err)
 		}
 		stmt.Close()
-	} else {
-		log.Println("sbom payload not available for db insertion, skipping db insertion")
-
-	}
-
 }
 func (c *DBClient) Close() {
 	_ = c.conn.Close()

client/pkg/clickhouse/statements.go

@@ -210,9 +210,7 @@ const trivySbomTable DBStatement = `
 		bom_ref String,
 		serial_number  String,
 		version INTEGER,
-		bom_format String,
-		component_version String,
-		component_mime_type String
+		bom_format String
 	) engine=File(TabSeparated)
 	`
 
@@ -230,6 +228,6 @@ const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespac
 const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id,  vul_pkg_id, vul_pkg_name,  vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
 const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
 const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Size, SHAID, EventTime) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?)"
-const InsertTrivySbom string = "INSERT INTO trivysbom (id, image_name, package_url, bom_ref, serial_number, version, bom_format, component_version, component_mime_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
+const InsertTrivySbom string = "INSERT INTO trivysbom (id, image_name, package_url, bom_ref, serial_number, version, bom_format) VALUES (?, ?, ?, ?, ?, ?, ?)"
 const InsertQuayContainerPushEvent DBStatement = "INSERT INTO quaycontainerpush (name, repository, nameSpace, dockerURL, homePage, tag, Event, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
 const InsertJfrogContainerPushEvent DBStatement = "INSERT INTO jfrogcontainerpush (Domain, EventType, RegistryURL, RepositoryName, SHAID, Size, ImageName, Tag, Event, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"

client/pkg/clients/kubviz_client.go

@@ -118,10 +118,11 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) {
 			Consumer: constants.Trivy_Sbom_Consumer,
 			Handler: func(msg *nats.Msg) {
 				msg.Ack()
-				var metrics model.Sbom
+				var metrics model.SbomData
 				err := json.Unmarshal(msg.Data, &metrics)
 				if err != nil {
-					log.Println("failed to unmarshal in nats", err)
+					log.Println("failed to unmarshal from nats", err)
+					return
 				}
 				log.Printf("Trivy sbom Metrics Received: %#v,", metrics)
 				conn.InsertTrivySbomMetrics(metrics)

model/trivy_sbom.go

@@ -9,4 +9,14 @@ type Sbom struct {
 	Report cyclonedx.BOM
 }
 
+type SbomData struct {
+	ID               string
+	ComponentName    string
+	PackageUrl       string
+	BomRef           string
+	SerialNumber     string
+	CycloneDxVersion int
+	BomFormat        string
+}
+
 

sql/0000015_trivysbom.up.sql

@@ -6,8 +6,6 @@ CREATE TABLE IF NOT EXISTS trivysbom (
 	serial_number         String,
 	version 			  INTEGER,
 	bom_format 			  String,
-	component_version     String,
-	component_mime_type   String,
 	ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}
 ) ENGINE = MergeTree() 
 ORDER BY ExpiryDate