Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pvc #282

Merged
merged 5 commits into from
Dec 13, 2023
Merged

Pvc #282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0965859
pvc changes
vijeyash1 Nov 21, 2023
6d31b45
added pvc changes
vijeyash1 Nov 21, 2023
030407f
changed image to get shell access
vijeyash1 Nov 28, 2023
6815d55
Merge branch 'main' into pvc
an1l4 Dec 5, 2023
318eadb
error-handle-changes-added
an1l4 Dec 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions agent/kubviz/kubePreUpgrade.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,19 +79,20 @@ func publishK8sDepricated_Deleted_Api(result *model.Result, js nats.JetStreamCon
}

func KubePreUpgradeDetector(config *rest.Config, js nats.JetStreamContext) error {
swaggerdir, err := os.MkdirTemp("", "kubepug")
pvcMountPath := "/mnt/agent/kbz"
uniqueDir := fmt.Sprintf("%s/kubepug", pvcMountPath)
err := os.MkdirAll(uniqueDir, 0755)
if err != nil {
return err
}
filename := fmt.Sprintf("%s/swagger-%s.json", swaggerdir, k8sVersion)
filename := fmt.Sprintf("%s/swagger-%s.json", uniqueDir, k8sVersion)
url := fmt.Sprintf("%s/%s/%s", baseURL, k8sVersion, fileURL)
err = downloadFile(filename, url)
if err != nil {
return err
}
defer os.RemoveAll(swaggerdir)
swaggerfile := filename
kubernetesAPIs, err := PopulateKubeAPIMap(swaggerfile)
defer os.RemoveAll(filename)
kubernetesAPIs, err := PopulateKubeAPIMap(filename)
if err != nil {
return err
}
Expand Down
11 changes: 10 additions & 1 deletion agent/kubviz/trivy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"os"
exec "os/exec"
"strings"

Expand All @@ -29,8 +31,15 @@ func executeCommandTrivy(command string) ([]byte, error) {
return outc.Bytes(), err
}
func RunTrivyK8sClusterScan(js nats.JetStreamContext) error {
pvcMountPath := "/mnt/agent/kbz"
trivyCacheDir := fmt.Sprintf("%s/trivy-cache", pvcMountPath)
err := os.MkdirAll(trivyCacheDir, 0755)
if err != nil {
log.Printf("Error creating Trivy cache directory: %v\n", err)
return err
}
var report report.ConsolidatedReport
cmdString := "trivy k8s --report summary cluster --exclude-nodes kubernetes.io/arch:amd64 --timeout 60m -f json --cache-dir /tmp/.cache --debug"
cmdString := fmt.Sprintf("trivy k8s --report summary cluster --exclude-nodes kubernetes.io/arch:amd64 --timeout 60m -f json --cache-dir %s --debug", trivyCacheDir)
clearCacheCmd := "trivy k8s --clear-cache"
out, err := executeCommandTrivy(cmdString)
if err != nil {
Expand Down
20 changes: 10 additions & 10 deletions agent/kubviz/trivy_image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"encoding/json"
"fmt"
"log"
"os"
"strings"
Expand All @@ -15,6 +16,13 @@ import (
)

func RunTrivyImageScans(config *rest.Config, js nats.JetStreamContext) error {
pvcMountPath := "/mnt/agent/kbz"
trivyImageCacheDir := fmt.Sprintf("%s/trivy-imagecache", pvcMountPath)
err := os.MkdirAll(trivyImageCacheDir, 0755)
if err != nil {
log.Printf("Error creating Trivy Image cache directory: %v\n", err)
return err
}
clearCacheCmd := "trivy image --clear-cache"

images, err := ListImages(config)
Expand All @@ -25,7 +33,8 @@ func RunTrivyImageScans(config *rest.Config, js nats.JetStreamContext) error {

for _, image := range images {
var report types.Report
out, err := executeCommand("trivy image " + image.PullableImage + " --timeout 60m -f json -q --cache-dir /tmp/.cache")
scanCmd := fmt.Sprintf("trivy image %s --timeout 60m -f json -q --cache-dir %s", image.PullableImage, trivyImageCacheDir)
out, err := executeCommand(scanCmd)
if err != nil {
log.Printf("Error scanning image %s: %v", image.PullableImage, err)
continue // Move on to the next image in case of an error
Expand Down Expand Up @@ -74,12 +83,3 @@ func publishImageScanReports(report types.Report, js nats.JetStreamContext) erro
log.Printf("Trivy image report with ID:%s has been published\n", metrics.ID)
return nil
}

func cleanupCache(cacheDir string) {
err := os.RemoveAll(cacheDir)
if err != nil {
log.Printf("Failed to clean up cache directory %s: %v", cacheDir, err)
} else {
log.Printf("Cache directory %s cleaned up successfully", cacheDir)
}
}
12 changes: 10 additions & 2 deletions agent/kubviz/trivy_sbom.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"log"
"os"
"os/exec"

"github.com/aquasecurity/trivy/pkg/sbom/cyclonedx"
Expand Down Expand Up @@ -46,6 +47,13 @@ func executeCommandSbom(command string) ([]byte, error) {
}

func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
pvcMountPath := "/mnt/agent/kbz"
trivySbomCacheDir := fmt.Sprintf("%s/trivy-sbomcache", pvcMountPath)
err := os.MkdirAll(trivySbomCacheDir, 0755)
if err != nil {
log.Printf("Error creating Trivy cache directory: %v\n", err)
return err
}
clearCacheCmd := "trivy image --clear-cache"

log.Println("trivy sbom run started")
Expand All @@ -56,8 +64,8 @@ func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext) error {
}
for _, image := range images {

command := fmt.Sprintf("trivy image --format cyclonedx %s %s", image.PullableImage, "--cache-dir /tmp/.cache")
out, err := executeCommandSbom(command)
sbomcmd := fmt.Sprintf("trivy image --format cyclonedx %s --cache-dir %s", image.PullableImage, trivySbomCacheDir)
out, err := executeCommandSbom(sbomcmd)

if err != nil {
log.Printf("Error executing Trivy for image sbom %s: %v", image.PullableImage, err)
Expand Down
90 changes: 30 additions & 60 deletions client/pkg/clickhouse/db_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,12 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE

tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}

stmt, err := tx.Prepare(string(InsertAzureContainerPushEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -187,14 +185,12 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE
func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}

stmt, err := tx.Prepare(string(InsertQuayContainerPushEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -240,14 +236,12 @@ func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload
func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushEventPayload) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}

stmt, err := tx.Prepare(string(InsertJfrogContainerPushEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -293,13 +287,11 @@ func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushE
func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertRakees))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand All @@ -325,13 +317,11 @@ func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) {
func (c *DBClient) InsertKetallEvent(metrics model.Resource) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertKetall))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand All @@ -356,13 +346,11 @@ func (c *DBClient) InsertKetallEvent(metrics model.Resource) {
func (c *DBClient) InsertOutdatedEvent(metrics model.CheckResultfinal) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertOutdated))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand All @@ -389,13 +377,11 @@ func (c *DBClient) InsertOutdatedEvent(metrics model.CheckResultfinal) {
func (c *DBClient) InsertDeprecatedAPI(deprecatedAPI model.DeprecatedAPI) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertDepricatedApi))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -429,13 +415,11 @@ func (c *DBClient) InsertDeprecatedAPI(deprecatedAPI model.DeprecatedAPI) {
func (c *DBClient) InsertDeletedAPI(deletedAPI model.DeletedAPI) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertDeletedApi))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -470,13 +454,11 @@ func (c *DBClient) InsertDeletedAPI(deletedAPI model.DeletedAPI) {
func (c *DBClient) InsertKubvizEvent(metrics model.Metrics) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertKubvizEvent))
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -539,13 +521,11 @@ func (c *DBClient) InsertContainerEvent(event string) {
func (c *DBClient) InsertKubeScoreMetrics(metrics model.KubeScoreRecommendations) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertKubeScore)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}
defer stmt.Close()

Expand All @@ -571,13 +551,11 @@ func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) {
for _, vulnerability := range result.Vulnerabilities {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivyVul)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}
if _, err := stmt.Exec(
metrics.ID,
Expand Down Expand Up @@ -608,13 +586,11 @@ func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) {
for _, misconfiguration := range result.Misconfigurations {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivyMisconfig)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

defer stmt.Close()
Expand Down Expand Up @@ -654,13 +630,11 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) {
for _, vulnerability := range result.Vulnerabilities {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivyImage)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

if _, err := stmt.Exec(
Expand Down Expand Up @@ -700,13 +674,11 @@ func (c *DBClient) InsertTrivySbomMetrics(metrics model.Sbom) {
if result.CycloneDX != nil {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(InsertTrivySbom)
if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}

if _, err := stmt.Exec(
Expand Down Expand Up @@ -834,14 +806,12 @@ func (c *DBClient) RetrieveKubvizEvent() ([]model.DbEvent, error) {
func (c *DBClient) InsertContainerEventDockerHub(build model.DockerHubBuild) {
tx, err := c.conn.Begin()
if err != nil {
log.Printf("error beginning transaction: %v", err)
return
log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err)
}
stmt, err := tx.Prepare(string(InsertDockerHubBuild))

if err != nil {
log.Printf("error preparing statement: %v", err)
return
log.Fatalf("error preparing statement: %v", err)
}
defer stmt.Close()

Expand Down
2 changes: 1 addition & 1 deletion dockerfiles/client/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o k8smetri

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
FROM gcr.io/distroless/static:nonroot
FROM golang:alpine
WORKDIR /
COPY --from=builder /workspace/k8smetrics_client .
USER 65532:65532
Expand Down
Loading