From 6cfba5a473ed3ca423624ed0e1a115efea0e4ffd Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Mon, 12 Feb 2024 10:17:36 +0530 Subject: [PATCH 1/8] store the exiting secret --- client/pkg/application/application.go | 73 ++++++++++++- client/pkg/clickhouse/db_client.go | 34 +++--- client/pkg/config/config.go | 5 + client/pkg/storage/store.go | 150 ++++++++++++++++++++++++++ go.mod | 2 + go.sum | 18 ++++ sql/0000010_trivy_misconfig.up.sql | 1 + sql/0000011_trivyimage.up.sql | 1 + sql/0000012_dockerhubbuild.up.sql | 1 + sql/0000013_azurecontainerpush.up.sql | 1 + sql/0000014_quaycontainerpush.up.sql | 1 + sql/0000015_trivysbom.up.sql | 1 + sql/0000016_azure_devops.up.sql | 1 + sql/0000017_github.up.sql | 1 + sql/0000018_gitlab.up.sql | 1 + sql/0000019_bitbucket.up.sql | 1 + sql/000001_events.up.sql | 1 + sql/0000020_gitea.up.sql | 1 + sql/000002_rakkess.up.sql | 1 + sql/000003_DeprecatedAPIs.up.sql | 1 + sql/000004_DeletedAPIs.up.sql | 1 + sql/000005_jfrogcontainerpush.up.sql | 1 + sql/000006_getall_resources.up.sql | 1 + sql/000007_outdated_images.up.sql | 1 + sql/000008_kubescore.up.sql | 1 + sql/000009_trivy_vul.up.sql | 1 + 26 files changed, 282 insertions(+), 20 deletions(-) create mode 100644 client/pkg/storage/store.go diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 4b78b3d8..5f87e8cf 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -2,13 +2,19 @@ package application import ( "context" + "database/sql" "log" + "os" + "os/signal" + "syscall" "github.com/intelops/kubviz/client/pkg/clickhouse" "github.com/intelops/kubviz/client/pkg/clients" "github.com/intelops/kubviz/client/pkg/config" + "github.com/intelops/kubviz/client/pkg/storage" "github.com/intelops/kubviz/pkg/opentelemetry" "github.com/kelseyhightower/envconfig" + "github.com/robfig/cron/v3" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" ) @@ -19,20 +25,43 @@ type Application struct { dbClient clickhouse.DBInterface } +const ( + EventsTable = "events" + RakkessTable = "rakkess" + DeprecatedAPIsTable = "DeprecatedAPIs" + DeletedAPIsTable = "DeletedAPIs" + JfrogContainerPushTable = "jfrogcontainerpush" + GetAllResourcesTable = "getall_resources" + OutdatedImagesTable = "outdated_images" + KubeScoreTable = "kubescore" + TrivyVulTable = "trivy_vul" + TrivyMisconfigTable = "trivy_misconfig" + TrivyImageTable = "trivyimage" + DockerHubBuildTable = "dockerhubbuild" + AzureContainerPushTable = "azurecontainerpush" + QuayContainerPushTable = "quaycontainerpush" + TrivySBOMTable = "trivysbom" + AzureDevOpsTable = "azure_devops" + GitHubTable = "github" + GitLabTable = "gitlab" + BitbucketTable = "bitbucket" + GiteaTable = "gitea" +) + func Start() *Application { log.Println("Client Application started...") - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("kubviz-client") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "Start") span.SetAttributes(attribute.String("start-app-client", "application")) defer span.End() - + cfg := &config.Config{} if err := envconfig.Process("", cfg); err != nil { log.Fatalf("Could not parse env Config: %v", err) } - dbClient, _, err := clickhouse.NewDBClient(cfg) + dbClient, conn, err := clickhouse.NewDBClient(cfg) if err != nil { log.Fatal(err) } @@ -41,6 +70,28 @@ func Start() *Application { if err != nil { log.Fatal("Error establishing connection to NATS:", err) } + c := cron.New() + _, err = c.AddFunc("@daily", func() { + if err := exportDataForTables(conn); err != nil { + log.Println("Error exporting data:", err) + } + }) + if err != nil { + log.Fatal("Error adding cron job:", err) + } + + // Listen for interrupt signals to stop the program + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + + // Start the cron job scheduler + c.Start() + + // Wait for an interrupt signal to stop the program + <-interrupt + + // Stop the cron scheduler gracefully + c.Stop() return &Application{ Config: cfg, conn: natsContext, @@ -53,3 +104,19 @@ func (app *Application) Close() { app.conn.Close() app.dbClient.Close() } +func exportDataForTables(db *sql.DB) error { + pvcMountPath := "/mnt/client/kbz" + tables := []string{ + EventsTable, RakkessTable, DeprecatedAPIsTable, DeletedAPIsTable, JfrogContainerPushTable, GetAllResourcesTable, OutdatedImagesTable, KubeScoreTable, TrivyVulTable, TrivyMisconfigTable, TrivyImageTable, DockerHubBuildTable, AzureContainerPushTable, QuayContainerPushTable, TrivySBOMTable, AzureDevOpsTable, GitHubTable, GitLabTable, BitbucketTable, GiteaTable, + } + for _, tableName := range tables { + err := storage.ExportExpiredData(tableName, db, pvcMountPath) + if err != nil { + log.Printf("Error exporting data for table %s: %v", tableName, err) + } else { + log.Printf("Export completed successfully for table %s.\n", tableName) + } + } + + return nil +} diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index 8e8a6fe7..a50f4889 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -137,7 +137,7 @@ func NewDBClient(conf *config.Config) (DBInterface, *sql.DB, error) { func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushEventPayload) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-container-azure") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertContainerEventAzure") span.SetAttributes(attribute.String("container-azure-client", "insert")) @@ -193,7 +193,7 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-container-quay") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertContainerEventQuay") span.SetAttributes(attribute.String("container-quay-client", "insert")) @@ -251,7 +251,7 @@ func (c *DBClient) InsertContainerEventQuay(pushEvent model.QuayImagePushPayload func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushEventPayload) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-container-jfrog") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertContainerEventJfrog") span.SetAttributes(attribute.String("container-jfrog-client", "insert")) @@ -309,7 +309,7 @@ func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushE func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-rakees-metrics") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertRakeesMetrics") span.SetAttributes(attribute.String("rakees-client", "insert")) @@ -346,7 +346,7 @@ func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) { func (c *DBClient) InsertKetallEvent(metrics model.Resource) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-ketall-event") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertKetallEvent") span.SetAttributes(attribute.String("ketall-client", "insert")) @@ -382,7 +382,7 @@ func (c *DBClient) InsertKetallEvent(metrics model.Resource) { func (c *DBClient) InsertOutdatedEvent(metrics model.CheckResultfinal) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-outdated-event") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertOutdatedEvent") span.SetAttributes(attribute.String("outdated-client", "insert")) @@ -420,7 +420,7 @@ func (c *DBClient) InsertOutdatedEvent(metrics model.CheckResultfinal) { func (c *DBClient) InsertDeprecatedAPI(deprecatedAPI model.DeprecatedAPI) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-depricated-event") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertDeprecatedAPI") span.SetAttributes(attribute.String("depricated-client", "insert")) @@ -465,7 +465,7 @@ func (c *DBClient) InsertDeprecatedAPI(deprecatedAPI model.DeprecatedAPI) { func (c *DBClient) InsertDeletedAPI(deletedAPI model.DeletedAPI) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-deletedapi") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertDeletedAPI") span.SetAttributes(attribute.String("deletedapi-client", "insert")) @@ -511,7 +511,7 @@ func (c *DBClient) InsertDeletedAPI(deletedAPI model.DeletedAPI) { func (c *DBClient) InsertKubvizEvent(metrics model.Metrics) { - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("insert-kubviz-event") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "InsertKubvizEvent") span.SetAttributes(attribute.String("kubvizevent-client", "insert")) @@ -791,14 +791,14 @@ func (c *DBClient) InsertTrivySbomMetrics(metrics model.SbomData) { span.SetAttributes(attribute.String("trivy-sbom-client", "insert")) defer span.End() - tx, err := c.conn.Begin() - if err != nil { - log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err) - } - stmt, err := tx.Prepare(InsertTrivySbom) - if err != nil { - log.Fatalf("error preparing statement: %v", err) - } + tx, err := c.conn.Begin() + if err != nil { + log.Fatalf("error beginning transaction, clickhouse connection not available: %v", err) + } + stmt, err := tx.Prepare(InsertTrivySbom) + if err != nil { + log.Fatalf("error preparing statement: %v", err) + } if _, err := stmt.Exec( metrics.ID, diff --git a/client/pkg/config/config.go b/client/pkg/config/config.go index 7f030aa9..93fec150 100644 --- a/client/pkg/config/config.go +++ b/client/pkg/config/config.go @@ -17,4 +17,9 @@ type Config struct { TrivyConsumer string `envconfig:"TRIVY_CONSUMER" required:"true"` TrivyImageConsumer string `envconfig:"TRIVY_IMAGE_CONSUMER" required:"true"` TrivySbomConsumer string `envconfig:"TRIVY_SBOM_CONSUMER" required:"true"` + AWSRegion string `envconfig:"AWS_REGION"` + AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` + AWSSecretKey string `envconfig:"AWS_SECRET_KEY"` + S3BucketName string `envconfig:"S3_BUCKET_NAME"` + S3ObjectKey string `envconfig:"S3_OBJECT_KEY"` } diff --git a/client/pkg/storage/store.go b/client/pkg/storage/store.go new file mode 100644 index 00000000..b56fc839 --- /dev/null +++ b/client/pkg/storage/store.go @@ -0,0 +1,150 @@ +package storage + +import ( + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/intelops/kubviz/client/pkg/config" + "github.com/kelseyhightower/envconfig" +) + +// ExportExpiredData exports expired data from a specific table in ClickHouse to an external storage (S3 in this case). +func ExportExpiredData(tableName string, db *sql.DB, pvcMountPath string) error { // Create ClickHouse database client + + columns, err := getTableColumns(db, tableName) + if err != nil { + return fmt.Errorf("error getting columns for table %s: %v", tableName, err) + } + // Construct CSV file path based on the provided mount path + csvFilePath := filepath.Join(pvcMountPath, fmt.Sprintf("exported_data_%s.csv", tableName)) + + // Construct SQL query + query := fmt.Sprintf("SELECT * FROM %s WHERE ExportedAt IS NULL", tableName) + + // Query expired data + rows, err := db.Query(query) + if err != nil { + return fmt.Errorf("error querying ClickHouse: %v", err) + } + defer rows.Close() + + // Create a CSV file to store the exported data + csvFile, err := os.Create(fmt.Sprintf("exported_data_%s.csv", tableName)) + if err != nil { + return fmt.Errorf("error creating CSV file: %v", err) + } + defer csvFile.Close() + + // Write CSV header + csvFile.WriteString(fmt.Sprintf("%s\n", columns)) + + // Write rows to CSV + for rows.Next() { + // Assuming a dynamic structure, scan the columns into a slice of interface{} + columnValues := make([]interface{}, len(columns)) + for i := range columnValues { + columnValues[i] = new(interface{}) + } + + err := rows.Scan(columnValues...) + if err != nil { + return fmt.Errorf("error scanning ClickHouse row: %v", err) + } + + // Write the values to the CSV file + var rowData []string + for _, value := range columnValues { + // Dereference the pointer to get the interface{} value, then format it as a string + rowData = append(rowData, fmt.Sprintf("%v", *value.(*interface{}))) + } + csvline := strings.Join(rowData, ",") + "\n" + _, err = csvFile.WriteString(fmt.Sprintf("%s\n", csvline)) + if err != nil { + return fmt.Errorf("error writing into csv file: %v", err) + } + } + // Upload the CSV file to S3 with a custom object key + err = uploadToS3(csvFilePath, fmt.Sprintf("exported_data_%s.csv", tableName)) + if err != nil { + return fmt.Errorf("error uploading CSV to S3: %v", err) + } + // err = uploadToS3(fmt.Sprintf("exported_data_%s.csv", tableName)) + // if err != nil { + // return fmt.Errorf("error uploading CSV to S3: %v", err) + // } + // Update ExportedAt column with the current timestamp for exported rows + updateQuery := fmt.Sprintf("ALTER TABLE %s UPDATE ExportedAt = now() WHERE ExportedAt IS NULL", tableName) + _, err = db.Exec(updateQuery) + if err != nil { + return fmt.Errorf("error updating ExportedAt column: %v", err) + } + + return nil +} + +func getTableColumns(db *sql.DB, tableName string) (string, error) { + // Query to get column names + query := fmt.Sprintf("DESCRIBE TABLE %s", tableName) + rows, err := db.Query(query) + if err != nil { + return "", err + } + defer rows.Close() + + // Get column names + var columns []string + for rows.Next() { + var columnName string + rows.Scan(&columnName) + columns = append(columns, columnName) + } + + return strings.Join(columns, ","), nil +} + +// func uploadToS3(filePath string) error { +func uploadToS3(filePath, s3ObjectKey string) error { + cfg := &config.Config{} + if err := envconfig.Process("", cfg); err != nil { + log.Fatalf("Could not parse env Config: %v", err) + } + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("error opening file: %v", err) + } + defer file.Close() + + // Set up AWS S3 session + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(cfg.AWSRegion), + Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSSecretKey, ""), + }) + if err != nil { + return fmt.Errorf("error creating S3 session: %v", err) + } + + // Create an S3 service client + s3Client := s3.New(sess) + + // Upload the file to S3 + _, err = s3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(cfg.S3BucketName), + Key: aws.String((s3ObjectKey)), + Body: file, + }) + if err != nil { + return fmt.Errorf("error uploading file to S3: %v", err) + } + + fmt.Printf("File uploaded to S3: %s\n", filePath) + + return nil +} diff --git a/go.mod b/go.mod index 41cc1277..9d93c5ba 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/99designs/gqlgen v0.17.42 github.com/ClickHouse/clickhouse-go/v2 v2.10.1 github.com/aquasecurity/trivy v0.43.1 + github.com/aws/aws-sdk-go v1.44.245 github.com/blang/semver v3.5.1+incompatible github.com/corneliusweig/tabwriter v0.0.0-20190512204542-5f8a091e83b5 github.com/davecgh/go-spew v1.1.1 @@ -96,6 +97,7 @@ require ( github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/yaml v0.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect diff --git a/go.sum b/go.sum index 1098ebc7..04672d96 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,7 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/aws/aws-sdk-go v1.44.245 h1:KtY2s4q31/kn33AdV63R5t77mdxsI7rq3YT7Mgo805M= +github.com/aws/aws-sdk-go v1.44.245/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -283,6 +284,9 @@ github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -495,6 +499,7 @@ github.com/yashtewari/glob-intersection v0.1.0 h1:6gJvMYQlTDOL3dMsPF6J0+26vwX9MB github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0= github.com/zegl/kube-score v1.17.0 h1:vedzK0pm5yOb1ocm5gybMNYsJRG8iTAatbo3LFIWbUc= @@ -536,6 +541,7 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= @@ -547,6 +553,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -558,7 +565,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -570,6 +580,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -583,13 +594,18 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -597,6 +613,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -611,6 +628,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/sql/0000010_trivy_misconfig.up.sql b/sql/0000010_trivy_misconfig.up.sql index 89d758a5..a4368387 100644 --- a/sql/0000010_trivy_misconfig.up.sql +++ b/sql/0000010_trivy_misconfig.up.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS trivy_misconfig ( misconfig_status String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000011_trivyimage.up.sql b/sql/0000011_trivyimage.up.sql index 572bb2b4..694e0aa0 100644 --- a/sql/0000011_trivyimage.up.sql +++ b/sql/0000011_trivyimage.up.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS trivyimage ( vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000012_dockerhubbuild.up.sql b/sql/0000012_dockerhubbuild.up.sql index 0485e52b..91f91d2c 100644 --- a/sql/0000012_dockerhubbuild.up.sql +++ b/sql/0000012_dockerhubbuild.up.sql @@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS dockerhubbuild ( Event String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000013_azurecontainerpush.up.sql b/sql/0000013_azurecontainerpush.up.sql index a0f5916d..1f86303b 100644 --- a/sql/0000013_azurecontainerpush.up.sql +++ b/sql/0000013_azurecontainerpush.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS azurecontainerpush ( SHAID String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; \ No newline at end of file diff --git a/sql/0000014_quaycontainerpush.up.sql b/sql/0000014_quaycontainerpush.up.sql index 2c249b56..8da71985 100644 --- a/sql/0000014_quaycontainerpush.up.sql +++ b/sql/0000014_quaycontainerpush.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS quaycontainerpush ( Event String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000015_trivysbom.up.sql b/sql/0000015_trivysbom.up.sql index 0e3a9c2e..92fab695 100644 --- a/sql/0000015_trivysbom.up.sql +++ b/sql/0000015_trivysbom.up.sql @@ -9,6 +9,7 @@ CREATE TABLE IF NOT EXISTS trivysbom ( version INTEGER, bom_format String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000016_azure_devops.up.sql b/sql/0000016_azure_devops.up.sql index 06a32dd2..09fbe3a6 100644 --- a/sql/0000016_azure_devops.up.sql +++ b/sql/0000016_azure_devops.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS azure_devops ( TimeStamp DateTime('UTC'), Event String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000017_github.up.sql b/sql/0000017_github.up.sql index 96d9bf24..7cbc541a 100644 --- a/sql/0000017_github.up.sql +++ b/sql/0000017_github.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS github ( TimeStamp DateTime('UTC'), Event String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000018_gitlab.up.sql b/sql/0000018_gitlab.up.sql index 2403dff1..93d58dc3 100644 --- a/sql/0000018_gitlab.up.sql +++ b/sql/0000018_gitlab.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS gitlab ( TimeStamp DateTime('UTC'), Event String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000019_bitbucket.up.sql b/sql/0000019_bitbucket.up.sql index adf08956..225db703 100644 --- a/sql/0000019_bitbucket.up.sql +++ b/sql/0000019_bitbucket.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS bitbucket ( TimeStamp DateTime('UTC'), Event String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000001_events.up.sql b/sql/000001_events.up.sql index a410aaf5..720ed670 100644 --- a/sql/000001_events.up.sql +++ b/sql/000001_events.up.sql @@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS events ( FirstTime String, LastTime String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000020_gitea.up.sql b/sql/0000020_gitea.up.sql index 1b42c0e7..0a3cc097 100644 --- a/sql/0000020_gitea.up.sql +++ b/sql/0000020_gitea.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS gitea ( TimeStamp DateTime('UTC'), Event String, ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000002_rakkess.up.sql b/sql/000002_rakkess.up.sql index 3542f59b..c5139fae 100644 --- a/sql/000002_rakkess.up.sql +++ b/sql/000002_rakkess.up.sql @@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS rakkess ( Update String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000003_DeprecatedAPIs.up.sql b/sql/000003_DeprecatedAPIs.up.sql index a7ed1b6e..3756a8e2 100644 --- a/sql/000003_DeprecatedAPIs.up.sql +++ b/sql/000003_DeprecatedAPIs.up.sql @@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs ( Scope String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000004_DeletedAPIs.up.sql b/sql/000004_DeletedAPIs.up.sql index 931b93e7..d79d3393 100644 --- a/sql/000004_DeletedAPIs.up.sql +++ b/sql/000004_DeletedAPIs.up.sql @@ -9,6 +9,7 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs ( Scope String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000005_jfrogcontainerpush.up.sql b/sql/000005_jfrogcontainerpush.up.sql index 6bac6d8d..14028075 100644 --- a/sql/000005_jfrogcontainerpush.up.sql +++ b/sql/000005_jfrogcontainerpush.up.sql @@ -10,6 +10,7 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush ( Event String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000006_getall_resources.up.sql b/sql/000006_getall_resources.up.sql index 23991a21..678d4f4e 100644 --- a/sql/000006_getall_resources.up.sql +++ b/sql/000006_getall_resources.up.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS getall_resources ( Age String, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000007_outdated_images.up.sql b/sql/000007_outdated_images.up.sql index 479c0902..fc9d55e5 100644 --- a/sql/000007_outdated_images.up.sql +++ b/sql/000007_outdated_images.up.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS outdated_images ( VersionsBehind Int64, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000008_kubescore.up.sql b/sql/000008_kubescore.up.sql index b6ee3e3f..8c7587ed 100644 --- a/sql/000008_kubescore.up.sql +++ b/sql/000008_kubescore.up.sql @@ -14,6 +14,7 @@ CREATE TABLE IF NOT EXISTS kubescore ( file_row BIGINT, EventTime DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000009_trivy_vul.up.sql b/sql/000009_trivy_vul.up.sql index 8acb241d..0fbf46c3 100644 --- a/sql/000009_trivy_vul.up.sql +++ b/sql/000009_trivy_vul.up.sql @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS trivy_vul ( vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; From 3accd06b411bf96babea2a724f43cd66045c4124 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Mon, 12 Feb 2024 16:08:37 +0530 Subject: [PATCH 2/8] fix --- sql/0000010_trivy_misconfig.up.sql | 2 +- sql/0000011_trivyimage.up.sql | 2 +- sql/0000012_dockerhubbuild.up.sql | 2 +- sql/0000013_azurecontainerpush.up.sql | 2 +- sql/0000014_quaycontainerpush.up.sql | 2 +- sql/0000015_trivysbom.up.sql | 2 +- sql/0000016_azure_devops.up.sql | 2 +- sql/0000017_github.up.sql | 2 +- sql/0000018_gitlab.up.sql | 2 +- sql/0000019_bitbucket.up.sql | 2 +- sql/000001_events.up.sql | 2 +- sql/0000020_gitea.up.sql | 2 +- sql/000002_rakkess.up.sql | 2 +- sql/000003_DeprecatedAPIs.up.sql | 2 +- sql/000004_DeletedAPIs.up.sql | 2 +- sql/000005_jfrogcontainerpush.up.sql | 2 +- sql/000006_getall_resources.up.sql | 2 +- sql/000007_outdated_images.up.sql | 2 +- sql/000008_kubescore.up.sql | 2 +- sql/000009_trivy_vul.up.sql | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sql/0000010_trivy_misconfig.up.sql b/sql/0000010_trivy_misconfig.up.sql index a4368387..978a9ff8 100644 --- a/sql/0000010_trivy_misconfig.up.sql +++ b/sql/0000010_trivy_misconfig.up.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS trivy_misconfig ( misconfig_severity String, misconfig_status String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000011_trivyimage.up.sql b/sql/0000011_trivyimage.up.sql index 694e0aa0..3787fa59 100644 --- a/sql/0000011_trivyimage.up.sql +++ b/sql/0000011_trivyimage.up.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS trivyimage ( vul_severity String, vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000012_dockerhubbuild.up.sql b/sql/0000012_dockerhubbuild.up.sql index 91f91d2c..a448994e 100644 --- a/sql/0000012_dockerhubbuild.up.sql +++ b/sql/0000012_dockerhubbuild.up.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS dockerhubbuild ( Owner String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000013_azurecontainerpush.up.sql b/sql/0000013_azurecontainerpush.up.sql index 1f86303b..6dbb5aa2 100644 --- a/sql/0000013_azurecontainerpush.up.sql +++ b/sql/0000013_azurecontainerpush.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS azurecontainerpush ( Size Int32, SHAID String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000014_quaycontainerpush.up.sql b/sql/0000014_quaycontainerpush.up.sql index 8da71985..6304fcc7 100644 --- a/sql/0000014_quaycontainerpush.up.sql +++ b/sql/0000014_quaycontainerpush.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS quaycontainerpush ( tag String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000015_trivysbom.up.sql b/sql/0000015_trivysbom.up.sql index 92fab695..7bc749ac 100644 --- a/sql/0000015_trivysbom.up.sql +++ b/sql/0000015_trivysbom.up.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS trivysbom ( serial_number String, version INTEGER, bom_format String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000016_azure_devops.up.sql b/sql/0000016_azure_devops.up.sql index 09fbe3a6..bc7752a8 100644 --- a/sql/0000016_azure_devops.up.sql +++ b/sql/0000016_azure_devops.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS azure_devops ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000017_github.up.sql b/sql/0000017_github.up.sql index 7cbc541a..2f627dd5 100644 --- a/sql/0000017_github.up.sql +++ b/sql/0000017_github.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS github ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000018_gitlab.up.sql b/sql/0000018_gitlab.up.sql index 93d58dc3..d47dd988 100644 --- a/sql/0000018_gitlab.up.sql +++ b/sql/0000018_gitlab.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS gitlab ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000019_bitbucket.up.sql b/sql/0000019_bitbucket.up.sql index 225db703..778e2ea7 100644 --- a/sql/0000019_bitbucket.up.sql +++ b/sql/0000019_bitbucket.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS bitbucket ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000001_events.up.sql b/sql/000001_events.up.sql index 720ed670..1915420a 100644 --- a/sql/000001_events.up.sql +++ b/sql/000001_events.up.sql @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS events ( Event String, FirstTime String, LastTime String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/0000020_gitea.up.sql b/sql/0000020_gitea.up.sql index 0a3cc097..3232bc56 100644 --- a/sql/0000020_gitea.up.sql +++ b/sql/0000020_gitea.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS gitea ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000002_rakkess.up.sql b/sql/000002_rakkess.up.sql index c5139fae..0bbd400a 100644 --- a/sql/000002_rakkess.up.sql +++ b/sql/000002_rakkess.up.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS rakkess ( List String, Update String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000003_DeprecatedAPIs.up.sql b/sql/000003_DeprecatedAPIs.up.sql index 3756a8e2..46be7b93 100644 --- a/sql/000003_DeprecatedAPIs.up.sql +++ b/sql/000003_DeprecatedAPIs.up.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs ( Deprecated UInt8, Scope String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000004_DeletedAPIs.up.sql b/sql/000004_DeletedAPIs.up.sql index d79d3393..d333b3a4 100644 --- a/sql/000004_DeletedAPIs.up.sql +++ b/sql/000004_DeletedAPIs.up.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs ( Deleted UInt8, Scope String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000005_jfrogcontainerpush.up.sql b/sql/000005_jfrogcontainerpush.up.sql index 14028075..a3a98fa6 100644 --- a/sql/000005_jfrogcontainerpush.up.sql +++ b/sql/000005_jfrogcontainerpush.up.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush ( Tag String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000006_getall_resources.up.sql b/sql/000006_getall_resources.up.sql index 678d4f4e..32168213 100644 --- a/sql/000006_getall_resources.up.sql +++ b/sql/000006_getall_resources.up.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS getall_resources ( Resource String, Age String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000007_outdated_images.up.sql b/sql/000007_outdated_images.up.sql index fc9d55e5..9779aead 100644 --- a/sql/000007_outdated_images.up.sql +++ b/sql/000007_outdated_images.up.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS outdated_images ( LatestVersion String, VersionsBehind Int64, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000008_kubescore.up.sql b/sql/000008_kubescore.up.sql index 8c7587ed..2db441b1 100644 --- a/sql/000008_kubescore.up.sql +++ b/sql/000008_kubescore.up.sql @@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS kubescore ( file_name String, file_row BIGINT, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate diff --git a/sql/000009_trivy_vul.up.sql b/sql/000009_trivy_vul.up.sql index 0fbf46c3..2ebc670f 100644 --- a/sql/000009_trivy_vul.up.sql +++ b/sql/000009_trivy_vul.up.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS trivy_vul ( vul_severity String, vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate From afc63d6bef2233083a622a666479b1c2ec1a0072 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Mon, 12 Feb 2024 18:45:53 +0530 Subject: [PATCH 3/8] fix --- client/pkg/application/application.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 5f87e8cf..91681185 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -71,7 +71,7 @@ func Start() *Application { log.Fatal("Error establishing connection to NATS:", err) } c := cron.New() - _, err = c.AddFunc("@daily", func() { + _, err = c.AddFunc("*/5 * * * *", func() { if err := exportDataForTables(conn); err != nil { log.Println("Error exporting data:", err) } From 225382fc701e5b4945bcd484d8101979dc8e9a55 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Mon, 12 Feb 2024 20:22:08 +0530 Subject: [PATCH 4/8] fix --- client/pkg/application/application.go | 4 +- client/pkg/storage/store.go | 164 ++++++++++++++++++++------ 2 files changed, 127 insertions(+), 41 deletions(-) diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 91681185..862fb9d2 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -105,12 +105,12 @@ func (app *Application) Close() { app.dbClient.Close() } func exportDataForTables(db *sql.DB) error { - pvcMountPath := "/mnt/client/kbz" + //pvcMountPath := "/mnt/client/kbz" tables := []string{ EventsTable, RakkessTable, DeprecatedAPIsTable, DeletedAPIsTable, JfrogContainerPushTable, GetAllResourcesTable, OutdatedImagesTable, KubeScoreTable, TrivyVulTable, TrivyMisconfigTable, TrivyImageTable, DockerHubBuildTable, AzureContainerPushTable, QuayContainerPushTable, TrivySBOMTable, AzureDevOpsTable, GitHubTable, GitLabTable, BitbucketTable, GiteaTable, } for _, tableName := range tables { - err := storage.ExportExpiredData(tableName, db, pvcMountPath) + err := storage.ExportExpiredData(tableName, db) if err != nil { log.Printf("Error exporting data for table %s: %v", tableName, err) } else { diff --git a/client/pkg/storage/store.go b/client/pkg/storage/store.go index b56fc839..851fbb8a 100644 --- a/client/pkg/storage/store.go +++ b/client/pkg/storage/store.go @@ -4,8 +4,6 @@ import ( "database/sql" "fmt" "log" - "os" - "path/filepath" "strings" "github.com/aws/aws-sdk-go/aws" @@ -17,14 +15,83 @@ import ( ) // ExportExpiredData exports expired data from a specific table in ClickHouse to an external storage (S3 in this case). -func ExportExpiredData(tableName string, db *sql.DB, pvcMountPath string) error { // Create ClickHouse database client - +// func ExportExpiredData(tableName string, db *sql.DB, pvcMountPath string) error { // Create ClickHouse database client + +// columns, err := getTableColumns(db, tableName) +// if err != nil { +// return fmt.Errorf("error getting columns for table %s: %v", tableName, err) +// } +// // Construct CSV file path based on the provided mount path +// csvFilePath := filepath.Join(pvcMountPath, fmt.Sprintf("exported_data_%s.csv", tableName)) + +// // Construct SQL query +// query := fmt.Sprintf("SELECT * FROM %s WHERE ExportedAt IS NULL", tableName) + +// // Query expired data +// rows, err := db.Query(query) +// if err != nil { +// return fmt.Errorf("error querying ClickHouse: %v", err) +// } +// defer rows.Close() + +// // Create a CSV file to store the exported data +// csvFile, err := os.Create(fmt.Sprintf("exported_data_%s.csv", tableName)) +// if err != nil { +// return fmt.Errorf("error creating CSV file: %v", err) +// } +// defer csvFile.Close() + +// // Write CSV header +// csvFile.WriteString(fmt.Sprintf("%s\n", columns)) + +// // Write rows to CSV +// for rows.Next() { +// // Assuming a dynamic structure, scan the columns into a slice of interface{} +// columnValues := make([]interface{}, len(columns)) +// for i := range columnValues { +// columnValues[i] = new(interface{}) +// } + +// err := rows.Scan(columnValues...) +// if err != nil { +// return fmt.Errorf("error scanning ClickHouse row: %v", err) +// } + +// // Write the values to the CSV file +// var rowData []string +// for _, value := range columnValues { +// // Dereference the pointer to get the interface{} value, then format it as a string +// rowData = append(rowData, fmt.Sprintf("%v", *value.(*interface{}))) +// } +// csvline := strings.Join(rowData, ",") + "\n" +// _, err = csvFile.WriteString(fmt.Sprintf("%s\n", csvline)) +// if err != nil { +// return fmt.Errorf("error writing into csv file: %v", err) +// } +// } +// // Upload the CSV file to S3 with a custom object key +// err = uploadToS3(csvFilePath, fmt.Sprintf("exported_data_%s.csv", tableName)) +// if err != nil { +// return fmt.Errorf("error uploading CSV to S3: %v", err) +// } +// // err = uploadToS3(fmt.Sprintf("exported_data_%s.csv", tableName)) +// // if err != nil { +// // return fmt.Errorf("error uploading CSV to S3: %v", err) +// // } +// // Update ExportedAt column with the current timestamp for exported rows +// updateQuery := fmt.Sprintf("ALTER TABLE %s UPDATE ExportedAt = now() WHERE ExportedAt IS NULL", tableName) +// _, err = db.Exec(updateQuery) +// if err != nil { +// return fmt.Errorf("error updating ExportedAt column: %v", err) +// } + +// return nil +// } +func ExportExpiredData(tableName string, db *sql.DB) error { columns, err := getTableColumns(db, tableName) if err != nil { return fmt.Errorf("error getting columns for table %s: %v", tableName, err) } - // Construct CSV file path based on the provided mount path - csvFilePath := filepath.Join(pvcMountPath, fmt.Sprintf("exported_data_%s.csv", tableName)) // Construct SQL query query := fmt.Sprintf("SELECT * FROM %s WHERE ExportedAt IS NULL", tableName) @@ -36,20 +103,13 @@ func ExportExpiredData(tableName string, db *sql.DB, pvcMountPath string) error } defer rows.Close() - // Create a CSV file to store the exported data - csvFile, err := os.Create(fmt.Sprintf("exported_data_%s.csv", tableName)) - if err != nil { - return fmt.Errorf("error creating CSV file: %v", err) - } - defer csvFile.Close() + // Construct CSV data in memory + var csvData strings.Builder + csvData.WriteString(columns + "\n") // Write CSV header - // Write CSV header - csvFile.WriteString(fmt.Sprintf("%s\n", columns)) - - // Write rows to CSV for rows.Next() { // Assuming a dynamic structure, scan the columns into a slice of interface{} - columnValues := make([]interface{}, len(columns)) + columnValues := make([]interface{}, len(strings.Split(columns, ","))) for i := range columnValues { columnValues[i] = new(interface{}) } @@ -59,27 +119,21 @@ func ExportExpiredData(tableName string, db *sql.DB, pvcMountPath string) error return fmt.Errorf("error scanning ClickHouse row: %v", err) } - // Write the values to the CSV file + // Write the values to the CSV data var rowData []string for _, value := range columnValues { // Dereference the pointer to get the interface{} value, then format it as a string rowData = append(rowData, fmt.Sprintf("%v", *value.(*interface{}))) } - csvline := strings.Join(rowData, ",") + "\n" - _, err = csvFile.WriteString(fmt.Sprintf("%s\n", csvline)) - if err != nil { - return fmt.Errorf("error writing into csv file: %v", err) - } + csvData.WriteString(strings.Join(rowData, ",") + "\n") } - // Upload the CSV file to S3 with a custom object key - err = uploadToS3(csvFilePath, fmt.Sprintf("exported_data_%s.csv", tableName)) + + // Upload the CSV data to S3 + err = uploadToS3(&csvData, fmt.Sprintf("exported_data_%s.csv", tableName)) if err != nil { return fmt.Errorf("error uploading CSV to S3: %v", err) } - // err = uploadToS3(fmt.Sprintf("exported_data_%s.csv", tableName)) - // if err != nil { - // return fmt.Errorf("error uploading CSV to S3: %v", err) - // } + // Update ExportedAt column with the current timestamp for exported rows updateQuery := fmt.Sprintf("ALTER TABLE %s UPDATE ExportedAt = now() WHERE ExportedAt IS NULL", tableName) _, err = db.Exec(updateQuery) @@ -111,16 +165,48 @@ func getTableColumns(db *sql.DB, tableName string) (string, error) { } // func uploadToS3(filePath string) error { -func uploadToS3(filePath, s3ObjectKey string) error { +// func uploadToS3(filePath, s3ObjectKey string) error { +// cfg := &config.Config{} +// if err := envconfig.Process("", cfg); err != nil { +// log.Fatalf("Could not parse env Config: %v", err) +// } +// file, err := os.Open(filePath) +// if err != nil { +// return fmt.Errorf("error opening file: %v", err) +// } +// defer file.Close() + +// // Set up AWS S3 session +// sess, err := session.NewSession(&aws.Config{ +// Region: aws.String(cfg.AWSRegion), +// Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSSecretKey, ""), +// }) +// if err != nil { +// return fmt.Errorf("error creating S3 session: %v", err) +// } + +// // Create an S3 service client +// s3Client := s3.New(sess) + +// // Upload the file to S3 +// _, err = s3Client.PutObject(&s3.PutObjectInput{ +// Bucket: aws.String(cfg.S3BucketName), +// Key: aws.String((s3ObjectKey)), +// Body: file, +// }) +// if err != nil { +// return fmt.Errorf("error uploading file to S3: %v", err) +// } + +// fmt.Printf("File uploaded to S3: %s\n", filePath) + +// return nil +// } +func uploadToS3(csvData *strings.Builder, s3ObjectKey string) error { cfg := &config.Config{} if err := envconfig.Process("", cfg); err != nil { log.Fatalf("Could not parse env Config: %v", err) } - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("error opening file: %v", err) - } - defer file.Close() // Set up AWS S3 session sess, err := session.NewSession(&aws.Config{ @@ -134,17 +220,17 @@ func uploadToS3(filePath, s3ObjectKey string) error { // Create an S3 service client s3Client := s3.New(sess) - // Upload the file to S3 + // Upload the CSV data to S3 _, err = s3Client.PutObject(&s3.PutObjectInput{ Bucket: aws.String(cfg.S3BucketName), Key: aws.String((s3ObjectKey)), - Body: file, + Body: strings.NewReader(csvData.String()), }) if err != nil { - return fmt.Errorf("error uploading file to S3: %v", err) + return fmt.Errorf("error uploading data to S3: %v", err) } - fmt.Printf("File uploaded to S3: %s\n", filePath) + fmt.Printf("Data uploaded to S3: %s\n", s3ObjectKey) return nil } From 77a985dcae1de1eba3d94d8eb4590a01bc1abaa2 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Tue, 13 Feb 2024 08:18:20 +0530 Subject: [PATCH 5/8] fix --- client/pkg/application/application.go | 2 +- client/pkg/storage/store.go | 111 -------------------------- 2 files changed, 1 insertion(+), 112 deletions(-) diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 862fb9d2..f76d88a8 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -71,7 +71,7 @@ func Start() *Application { log.Fatal("Error establishing connection to NATS:", err) } c := cron.New() - _, err = c.AddFunc("*/5 * * * *", func() { + _, err = c.AddFunc("@daily", func() { if err := exportDataForTables(conn); err != nil { log.Println("Error exporting data:", err) } diff --git a/client/pkg/storage/store.go b/client/pkg/storage/store.go index 851fbb8a..b5684c6b 100644 --- a/client/pkg/storage/store.go +++ b/client/pkg/storage/store.go @@ -14,79 +14,6 @@ import ( "github.com/kelseyhightower/envconfig" ) -// ExportExpiredData exports expired data from a specific table in ClickHouse to an external storage (S3 in this case). -// func ExportExpiredData(tableName string, db *sql.DB, pvcMountPath string) error { // Create ClickHouse database client - -// columns, err := getTableColumns(db, tableName) -// if err != nil { -// return fmt.Errorf("error getting columns for table %s: %v", tableName, err) -// } -// // Construct CSV file path based on the provided mount path -// csvFilePath := filepath.Join(pvcMountPath, fmt.Sprintf("exported_data_%s.csv", tableName)) - -// // Construct SQL query -// query := fmt.Sprintf("SELECT * FROM %s WHERE ExportedAt IS NULL", tableName) - -// // Query expired data -// rows, err := db.Query(query) -// if err != nil { -// return fmt.Errorf("error querying ClickHouse: %v", err) -// } -// defer rows.Close() - -// // Create a CSV file to store the exported data -// csvFile, err := os.Create(fmt.Sprintf("exported_data_%s.csv", tableName)) -// if err != nil { -// return fmt.Errorf("error creating CSV file: %v", err) -// } -// defer csvFile.Close() - -// // Write CSV header -// csvFile.WriteString(fmt.Sprintf("%s\n", columns)) - -// // Write rows to CSV -// for rows.Next() { -// // Assuming a dynamic structure, scan the columns into a slice of interface{} -// columnValues := make([]interface{}, len(columns)) -// for i := range columnValues { -// columnValues[i] = new(interface{}) -// } - -// err := rows.Scan(columnValues...) -// if err != nil { -// return fmt.Errorf("error scanning ClickHouse row: %v", err) -// } - -// // Write the values to the CSV file -// var rowData []string -// for _, value := range columnValues { -// // Dereference the pointer to get the interface{} value, then format it as a string -// rowData = append(rowData, fmt.Sprintf("%v", *value.(*interface{}))) -// } -// csvline := strings.Join(rowData, ",") + "\n" -// _, err = csvFile.WriteString(fmt.Sprintf("%s\n", csvline)) -// if err != nil { -// return fmt.Errorf("error writing into csv file: %v", err) -// } -// } -// // Upload the CSV file to S3 with a custom object key -// err = uploadToS3(csvFilePath, fmt.Sprintf("exported_data_%s.csv", tableName)) -// if err != nil { -// return fmt.Errorf("error uploading CSV to S3: %v", err) -// } -// // err = uploadToS3(fmt.Sprintf("exported_data_%s.csv", tableName)) -// // if err != nil { -// // return fmt.Errorf("error uploading CSV to S3: %v", err) -// // } -// // Update ExportedAt column with the current timestamp for exported rows -// updateQuery := fmt.Sprintf("ALTER TABLE %s UPDATE ExportedAt = now() WHERE ExportedAt IS NULL", tableName) -// _, err = db.Exec(updateQuery) -// if err != nil { -// return fmt.Errorf("error updating ExportedAt column: %v", err) -// } - -// return nil -// } func ExportExpiredData(tableName string, db *sql.DB) error { columns, err := getTableColumns(db, tableName) if err != nil { @@ -164,44 +91,6 @@ func getTableColumns(db *sql.DB, tableName string) (string, error) { return strings.Join(columns, ","), nil } -// func uploadToS3(filePath string) error { -// func uploadToS3(filePath, s3ObjectKey string) error { -// cfg := &config.Config{} -// if err := envconfig.Process("", cfg); err != nil { -// log.Fatalf("Could not parse env Config: %v", err) -// } -// file, err := os.Open(filePath) -// if err != nil { -// return fmt.Errorf("error opening file: %v", err) -// } -// defer file.Close() - -// // Set up AWS S3 session -// sess, err := session.NewSession(&aws.Config{ -// Region: aws.String(cfg.AWSRegion), -// Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSSecretKey, ""), -// }) -// if err != nil { -// return fmt.Errorf("error creating S3 session: %v", err) -// } - -// // Create an S3 service client -// s3Client := s3.New(sess) - -// // Upload the file to S3 -// _, err = s3Client.PutObject(&s3.PutObjectInput{ -// Bucket: aws.String(cfg.S3BucketName), -// Key: aws.String((s3ObjectKey)), -// Body: file, -// }) -// if err != nil { -// return fmt.Errorf("error uploading file to S3: %v", err) -// } - -// fmt.Printf("File uploaded to S3: %s\n", filePath) - -// return nil -// } func uploadToS3(csvData *strings.Builder, s3ObjectKey string) error { cfg := &config.Config{} if err := envconfig.Process("", cfg); err != nil { From 26e3f4fb633571697e6a5deee5cdeae108524a50 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Tue, 13 Feb 2024 09:56:44 +0530 Subject: [PATCH 6/8] fix --- client/pkg/application/application.go | 37 +++++++++++++++------------ client/pkg/config/config.go | 1 + 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index f76d88a8..08c044fa 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -70,28 +70,31 @@ func Start() *Application { if err != nil { log.Fatal("Error establishing connection to NATS:", err) } - c := cron.New() - _, err = c.AddFunc("@daily", func() { - if err := exportDataForTables(conn); err != nil { - log.Println("Error exporting data:", err) + if cfg.AwsEnable { + c := cron.New() + _, err = c.AddFunc("@daily", func() { + if err := exportDataForTables(conn); err != nil { + log.Println("Error exporting data:", err) + } + }) + if err != nil { + log.Fatal("Error adding cron job:", err) } - }) - if err != nil { - log.Fatal("Error adding cron job:", err) - } - // Listen for interrupt signals to stop the program - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + // Listen for interrupt signals to stop the program + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) - // Start the cron job scheduler - c.Start() + // Start the cron job scheduler + c.Start() - // Wait for an interrupt signal to stop the program - <-interrupt + // Wait for an interrupt signal to stop the program + <-interrupt + + // Stop the cron scheduler gracefully + c.Stop() + } - // Stop the cron scheduler gracefully - c.Stop() return &Application{ Config: cfg, conn: natsContext, diff --git a/client/pkg/config/config.go b/client/pkg/config/config.go index 93fec150..db225edb 100644 --- a/client/pkg/config/config.go +++ b/client/pkg/config/config.go @@ -17,6 +17,7 @@ type Config struct { TrivyConsumer string `envconfig:"TRIVY_CONSUMER" required:"true"` TrivyImageConsumer string `envconfig:"TRIVY_IMAGE_CONSUMER" required:"true"` TrivySbomConsumer string `envconfig:"TRIVY_SBOM_CONSUMER" required:"true"` + AwsEnable bool `envconfig:"AWS_ENABLE" default:"false"` AWSRegion string `envconfig:"AWS_REGION"` AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` AWSSecretKey string `envconfig:"AWS_SECRET_KEY"` From 486098284a4a76a7d1cb9eb29345a003e4a70fd7 Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Fri, 16 Feb 2024 11:02:58 +0530 Subject: [PATCH 7/8] fix --- client/pkg/application/application.go | 60 ++++++++++++++++++--------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 08c044fa..05f40f54 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -70,30 +70,52 @@ func Start() *Application { if err != nil { log.Fatal("Error establishing connection to NATS:", err) } - if cfg.AwsEnable { - c := cron.New() - _, err = c.AddFunc("@daily", func() { - if err := exportDataForTables(conn); err != nil { - log.Println("Error exporting data:", err) - } - }) - if err != nil { - log.Fatal("Error adding cron job:", err) + c := cron.New() + _, err = c.AddFunc("@daily", func() { + if err := exportDataForTables(conn); err != nil { + log.Println("Error exporting data:", err) } + }) + if err != nil { + log.Fatal("Error adding cron job:", err) + } - // Listen for interrupt signals to stop the program - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + // Listen for interrupt signals to stop the program + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) - // Start the cron job scheduler - c.Start() + // Start the cron job scheduler + c.Start() - // Wait for an interrupt signal to stop the program - <-interrupt + // Wait for an interrupt signal to stop the program + <-interrupt - // Stop the cron scheduler gracefully - c.Stop() - } + // Stop the cron scheduler gracefully + c.Stop() + // if cfg.AwsEnable { + // c := cron.New() + // _, err = c.AddFunc("@daily", func() { + // if err := exportDataForTables(conn); err != nil { + // log.Println("Error exporting data:", err) + // } + // }) + // if err != nil { + // log.Fatal("Error adding cron job:", err) + // } + + // // Listen for interrupt signals to stop the program + // interrupt := make(chan os.Signal, 1) + // signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + + // // Start the cron job scheduler + // c.Start() + + // // Wait for an interrupt signal to stop the program + // <-interrupt + + // // Stop the cron scheduler gracefully + // c.Stop() + // } return &Application{ Config: cfg, From 1d13105fb824a7cc43cd76730a909f9c7b0683ad Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Fri, 16 Feb 2024 11:17:48 +0530 Subject: [PATCH 8/8] fix --- client/pkg/application/application.go | 82 +++++++++++++-------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 05f40f54..4584a724 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -70,52 +70,52 @@ func Start() *Application { if err != nil { log.Fatal("Error establishing connection to NATS:", err) } - c := cron.New() - _, err = c.AddFunc("@daily", func() { - if err := exportDataForTables(conn); err != nil { - log.Println("Error exporting data:", err) - } - }) - if err != nil { - log.Fatal("Error adding cron job:", err) - } - - // Listen for interrupt signals to stop the program - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) - - // Start the cron job scheduler - c.Start() - - // Wait for an interrupt signal to stop the program - <-interrupt - - // Stop the cron scheduler gracefully - c.Stop() - // if cfg.AwsEnable { - // c := cron.New() - // _, err = c.AddFunc("@daily", func() { - // if err := exportDataForTables(conn); err != nil { - // log.Println("Error exporting data:", err) - // } - // }) - // if err != nil { - // log.Fatal("Error adding cron job:", err) + // c := cron.New() + // _, err = c.AddFunc("@daily", func() { + // if err := exportDataForTables(conn); err != nil { + // log.Println("Error exporting data:", err) // } + // }) + // if err != nil { + // log.Fatal("Error adding cron job:", err) + // } - // // Listen for interrupt signals to stop the program - // interrupt := make(chan os.Signal, 1) - // signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + // // Listen for interrupt signals to stop the program + // interrupt := make(chan os.Signal, 1) + // signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + + // // Start the cron job scheduler + // c.Start() + + // // Wait for an interrupt signal to stop the program + // <-interrupt + + // // Stop the cron scheduler gracefully + // c.Stop() + if cfg.AwsEnable { + c := cron.New() + _, err = c.AddFunc("@daily", func() { + if err := exportDataForTables(conn); err != nil { + log.Println("Error exporting data:", err) + } + }) + if err != nil { + log.Fatal("Error adding cron job:", err) + } - // // Start the cron job scheduler - // c.Start() + // Listen for interrupt signals to stop the program + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) - // // Wait for an interrupt signal to stop the program - // <-interrupt + // Start the cron job scheduler + c.Start() - // // Stop the cron scheduler gracefully - // c.Stop() - // } + // Wait for an interrupt signal to stop the program + <-interrupt + + // Stop the cron scheduler gracefully + c.Stop() + } return &Application{ Config: cfg,