From a155f8f9e98b8d9c855409925fead229f5700b1e Mon Sep 17 00:00:00 2001 From: Nithunikzz Date: Fri, 16 Feb 2024 12:51:00 +0530 Subject: [PATCH] store expire data --- client/pkg/application/application.go | 99 +++++++++++++++++++- client/pkg/config/config.go | 38 ++++---- client/pkg/storage/store.go | 125 ++++++++++++++++++++++++++ go.mod | 2 + go.sum | 17 ++++ sql/0000010_trivy_misconfig.up.sql | 3 +- sql/0000011_trivyimage.up.sql | 3 +- sql/0000012_dockerhubbuild.up.sql | 3 +- sql/0000013_azurecontainerpush.up.sql | 3 +- sql/0000014_quaycontainerpush.up.sql | 3 +- sql/0000015_trivysbom.up.sql | 3 +- sql/0000016_azure_devops.up.sql | 3 +- sql/0000017_github.up.sql | 3 +- sql/0000018_gitlab.up.sql | 3 +- sql/0000019_bitbucket.up.sql | 3 +- sql/000001_events.up.sql | 3 +- sql/0000020_gitea.up.sql | 3 +- sql/0000021_kuberhealthy.up.sql | 3 +- sql/000002_rakkess.up.sql | 3 +- sql/000003_DeprecatedAPIs.up.sql | 3 +- sql/000004_DeletedAPIs.up.sql | 3 +- sql/000005_jfrogcontainerpush.up.sql | 3 +- sql/000006_getall_resources.up.sql | 3 +- sql/000007_outdated_images.up.sql | 3 +- sql/000008_kubescore.up.sql | 3 +- sql/000009_trivy_vul.up.sql | 3 +- 26 files changed, 304 insertions(+), 40 deletions(-) create mode 100644 client/pkg/storage/store.go diff --git a/client/pkg/application/application.go b/client/pkg/application/application.go index 4b78b3d8..6d9c3c99 100644 --- a/client/pkg/application/application.go +++ b/client/pkg/application/application.go @@ -2,13 +2,19 @@ package application import ( "context" + "database/sql" "log" + "os" + "os/signal" + "syscall" "github.com/intelops/kubviz/client/pkg/clickhouse" "github.com/intelops/kubviz/client/pkg/clients" "github.com/intelops/kubviz/client/pkg/config" + "github.com/intelops/kubviz/client/pkg/storage" "github.com/intelops/kubviz/pkg/opentelemetry" "github.com/kelseyhightower/envconfig" + "github.com/robfig/cron/v3" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" ) @@ -19,20 +25,44 @@ type Application struct { dbClient clickhouse.DBInterface } +const ( + EventsTable = "events" + RakkessTable = "rakkess" + DeprecatedAPIsTable = "DeprecatedAPIs" + DeletedAPIsTable = "DeletedAPIs" + JfrogContainerPushTable = "jfrogcontainerpush" + GetAllResourcesTable = "getall_resources" + OutdatedImagesTable = "outdated_images" + KubeScoreTable = "kubescore" + TrivyVulTable = "trivy_vul" + TrivyMisconfigTable = "trivy_misconfig" + TrivyImageTable = "trivyimage" + DockerHubBuildTable = "dockerhubbuild" + AzureContainerPushTable = "azurecontainerpush" + QuayContainerPushTable = "quaycontainerpush" + TrivySBOMTable = "trivysbom" + AzureDevOpsTable = "azure_devops" + GitHubTable = "github" + GitLabTable = "gitlab" + BitbucketTable = "bitbucket" + GiteaTable = "gitea" + KuberHealthy = "kuberhealthy" +) + func Start() *Application { log.Println("Client Application started...") - ctx:=context.Background() + ctx := context.Background() tracer := otel.Tracer("kubviz-client") _, span := tracer.Start(opentelemetry.BuildContext(ctx), "Start") span.SetAttributes(attribute.String("start-app-client", "application")) defer span.End() - + cfg := &config.Config{} if err := envconfig.Process("", cfg); err != nil { log.Fatalf("Could not parse env Config: %v", err) } - dbClient, _, err := clickhouse.NewDBClient(cfg) + dbClient, conn, err := clickhouse.NewDBClient(cfg) if err != nil { log.Fatal(err) } @@ -41,6 +71,53 @@ func Start() *Application { if err != nil { log.Fatal("Error establishing connection to NATS:", err) } + // c := cron.New() + // _, err = c.AddFunc("@daily", func() { + // if err := exportDataForTables(conn); err != nil { + // log.Println("Error exporting data:", err) + // } + // }) + // if err != nil { + // log.Fatal("Error adding cron job:", err) + // } + + // // Listen for interrupt signals to stop the program + // interrupt := make(chan os.Signal, 1) + // signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + + // // Start the cron job scheduler + // c.Start() + + // // Wait for an interrupt signal to stop the program + // <-interrupt + + // // Stop the cron scheduler gracefully + // c.Stop() + if cfg.AwsEnable { + c := cron.New() + _, err = c.AddFunc("@daily", func() { + if err := exportDataForTables(conn); err != nil { + log.Println("Error exporting data:", err) + } + }) + if err != nil { + log.Fatal("Error adding cron job:", err) + } + + // Listen for interrupt signals to stop the program + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) + + // Start the cron job scheduler + c.Start() + + // Wait for an interrupt signal to stop the program + <-interrupt + + // Stop the cron scheduler gracefully + c.Stop() + } + return &Application{ Config: cfg, conn: natsContext, @@ -53,3 +130,19 @@ func (app *Application) Close() { app.conn.Close() app.dbClient.Close() } +func exportDataForTables(db *sql.DB) error { + //pvcMountPath := "/mnt/client/kbz" + tables := []string{ + EventsTable, RakkessTable, DeprecatedAPIsTable, DeletedAPIsTable, JfrogContainerPushTable, GetAllResourcesTable, OutdatedImagesTable, KubeScoreTable, TrivyVulTable, TrivyMisconfigTable, TrivyImageTable, DockerHubBuildTable, AzureContainerPushTable, QuayContainerPushTable, TrivySBOMTable, AzureDevOpsTable, GitHubTable, GitLabTable, BitbucketTable, KuberHealthy, GiteaTable, + } + for _, tableName := range tables { + err := storage.ExportExpiredData(tableName, db) + if err != nil { + log.Printf("Error exporting data for table %s: %v", tableName, err) + } else { + log.Printf("Export completed successfully for table %s.\n", tableName) + } + } + + return nil +} diff --git a/client/pkg/config/config.go b/client/pkg/config/config.go index 7fb49a2c..d0dceff1 100644 --- a/client/pkg/config/config.go +++ b/client/pkg/config/config.go @@ -1,21 +1,27 @@ package config type Config struct { - NatsAddress string `envconfig:"NATS_ADDRESS"` - NatsToken string `envconfig:"NATS_TOKEN"` - DbPort int `envconfig:"DB_PORT"` - DBAddress string `envconfig:"DB_ADDRESS"` - ClickHouseUsername string `envconfig:"CLICKHOUSE_USERNAME"` - ClickHousePassword string `envconfig:"CLICKHOUSE_PASSWORD"` - KetallConsumer string `envconfig:"KETALL_EVENTS_CONSUMER" required:"true"` - RakeesConsumer string `envconfig:"RAKEES_METRICS_CONSUMER" required:"true"` - OutdatedConsumer string `envconfig:"OUTDATED_EVENTS_CONSUMER" required:"true"` - DeprecatedConsumer string `envconfig:"DEPRECATED_API_CONSUMER" required:"true"` - DeletedConsumer string `envconfig:"DELETED_API_CONSUMER" required:"true"` - KubvizConsumer string `envconfig:"KUBVIZ_EVENTS_CONSUMER" required:"true"` - KubscoreConsumer string `envconfig:"KUBSCORE_CONSUMER" required:"true"` - TrivyConsumer string `envconfig:"TRIVY_CONSUMER" required:"true"` - TrivyImageConsumer string `envconfig:"TRIVY_IMAGE_CONSUMER" required:"true"` - TrivySbomConsumer string `envconfig:"TRIVY_SBOM_CONSUMER" required:"true"` + NatsAddress string `envconfig:"NATS_ADDRESS"` + NatsToken string `envconfig:"NATS_TOKEN"` + DbPort int `envconfig:"DB_PORT"` + DBAddress string `envconfig:"DB_ADDRESS"` + ClickHouseUsername string `envconfig:"CLICKHOUSE_USERNAME"` + ClickHousePassword string `envconfig:"CLICKHOUSE_PASSWORD"` + KetallConsumer string `envconfig:"KETALL_EVENTS_CONSUMER" required:"true"` + RakeesConsumer string `envconfig:"RAKEES_METRICS_CONSUMER" required:"true"` + OutdatedConsumer string `envconfig:"OUTDATED_EVENTS_CONSUMER" required:"true"` + DeprecatedConsumer string `envconfig:"DEPRECATED_API_CONSUMER" required:"true"` + DeletedConsumer string `envconfig:"DELETED_API_CONSUMER" required:"true"` + KubvizConsumer string `envconfig:"KUBVIZ_EVENTS_CONSUMER" required:"true"` + KubscoreConsumer string `envconfig:"KUBSCORE_CONSUMER" required:"true"` + TrivyConsumer string `envconfig:"TRIVY_CONSUMER" required:"true"` + TrivyImageConsumer string `envconfig:"TRIVY_IMAGE_CONSUMER" required:"true"` + TrivySbomConsumer string `envconfig:"TRIVY_SBOM_CONSUMER" required:"true"` KuberhealthyConsumer string `envconfig:"KUBERHEALTHY_CONSUMER" required:"true"` + AwsEnable bool `envconfig:"AWS_ENABLE" default:"false"` + AWSRegion string `envconfig:"AWS_REGION"` + AWSAccessKey string `envconfig:"AWS_ACCESS_KEY"` + AWSSecretKey string `envconfig:"AWS_SECRET_KEY"` + S3BucketName string `envconfig:"S3_BUCKET_NAME"` + S3ObjectKey string `envconfig:"S3_OBJECT_KEY"` } diff --git a/client/pkg/storage/store.go b/client/pkg/storage/store.go new file mode 100644 index 00000000..b5684c6b --- /dev/null +++ b/client/pkg/storage/store.go @@ -0,0 +1,125 @@ +package storage + +import ( + "database/sql" + "fmt" + "log" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/intelops/kubviz/client/pkg/config" + "github.com/kelseyhightower/envconfig" +) + +func ExportExpiredData(tableName string, db *sql.DB) error { + columns, err := getTableColumns(db, tableName) + if err != nil { + return fmt.Errorf("error getting columns for table %s: %v", tableName, err) + } + + // Construct SQL query + query := fmt.Sprintf("SELECT * FROM %s WHERE ExportedAt IS NULL", tableName) + + // Query expired data + rows, err := db.Query(query) + if err != nil { + return fmt.Errorf("error querying ClickHouse: %v", err) + } + defer rows.Close() + + // Construct CSV data in memory + var csvData strings.Builder + csvData.WriteString(columns + "\n") // Write CSV header + + for rows.Next() { + // Assuming a dynamic structure, scan the columns into a slice of interface{} + columnValues := make([]interface{}, len(strings.Split(columns, ","))) + for i := range columnValues { + columnValues[i] = new(interface{}) + } + + err := rows.Scan(columnValues...) + if err != nil { + return fmt.Errorf("error scanning ClickHouse row: %v", err) + } + + // Write the values to the CSV data + var rowData []string + for _, value := range columnValues { + // Dereference the pointer to get the interface{} value, then format it as a string + rowData = append(rowData, fmt.Sprintf("%v", *value.(*interface{}))) + } + csvData.WriteString(strings.Join(rowData, ",") + "\n") + } + + // Upload the CSV data to S3 + err = uploadToS3(&csvData, fmt.Sprintf("exported_data_%s.csv", tableName)) + if err != nil { + return fmt.Errorf("error uploading CSV to S3: %v", err) + } + + // Update ExportedAt column with the current timestamp for exported rows + updateQuery := fmt.Sprintf("ALTER TABLE %s UPDATE ExportedAt = now() WHERE ExportedAt IS NULL", tableName) + _, err = db.Exec(updateQuery) + if err != nil { + return fmt.Errorf("error updating ExportedAt column: %v", err) + } + + return nil +} + +func getTableColumns(db *sql.DB, tableName string) (string, error) { + // Query to get column names + query := fmt.Sprintf("DESCRIBE TABLE %s", tableName) + rows, err := db.Query(query) + if err != nil { + return "", err + } + defer rows.Close() + + // Get column names + var columns []string + for rows.Next() { + var columnName string + rows.Scan(&columnName) + columns = append(columns, columnName) + } + + return strings.Join(columns, ","), nil +} + +func uploadToS3(csvData *strings.Builder, s3ObjectKey string) error { + cfg := &config.Config{} + if err := envconfig.Process("", cfg); err != nil { + log.Fatalf("Could not parse env Config: %v", err) + } + + // Set up AWS S3 session + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(cfg.AWSRegion), + Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSSecretKey, ""), + }) + if err != nil { + return fmt.Errorf("error creating S3 session: %v", err) + } + + // Create an S3 service client + s3Client := s3.New(sess) + + // Upload the CSV data to S3 + _, err = s3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(cfg.S3BucketName), + Key: aws.String((s3ObjectKey)), + Body: strings.NewReader(csvData.String()), + }) + if err != nil { + return fmt.Errorf("error uploading data to S3: %v", err) + } + + fmt.Printf("Data uploaded to S3: %s\n", s3ObjectKey) + + return nil +} diff --git a/go.mod b/go.mod index fa52bd8c..cbb62157 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/99designs/gqlgen v0.17.42 github.com/ClickHouse/clickhouse-go/v2 v2.10.1 github.com/aquasecurity/trivy v0.43.1 + github.com/aws/aws-sdk-go v1.44.245 github.com/blang/semver v3.5.1+incompatible github.com/corneliusweig/tabwriter v0.0.0-20190512204542-5f8a091e83b5 github.com/davecgh/go-spew v1.1.1 @@ -97,6 +98,7 @@ require ( github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/yaml v0.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect diff --git a/go.sum b/go.sum index 9aaeb1a6..4e586f75 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/aws/aws-sdk-go v1.25.24/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.44.245 h1:KtY2s4q31/kn33AdV63R5t77mdxsI7rq3YT7Mgo805M= +github.com/aws/aws-sdk-go v1.44.245/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -422,6 +423,9 @@ github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdi github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -676,6 +680,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0= github.com/zegl/kube-score v1.17.0 h1:vedzK0pm5yOb1ocm5gybMNYsJRG8iTAatbo3LFIWbUc= @@ -731,6 +736,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= @@ -771,6 +777,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -814,6 +821,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -841,6 +850,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -895,15 +905,20 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -915,6 +930,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -976,6 +992,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/sql/0000010_trivy_misconfig.up.sql b/sql/0000010_trivy_misconfig.up.sql index 89d758a5..978a9ff8 100644 --- a/sql/0000010_trivy_misconfig.up.sql +++ b/sql/0000010_trivy_misconfig.up.sql @@ -15,7 +15,8 @@ CREATE TABLE IF NOT EXISTS trivy_misconfig ( misconfig_severity String, misconfig_status String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000011_trivyimage.up.sql b/sql/0000011_trivyimage.up.sql index 572bb2b4..3787fa59 100644 --- a/sql/0000011_trivyimage.up.sql +++ b/sql/0000011_trivyimage.up.sql @@ -11,7 +11,8 @@ CREATE TABLE IF NOT EXISTS trivyimage ( vul_severity String, vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000012_dockerhubbuild.up.sql b/sql/0000012_dockerhubbuild.up.sql index 0485e52b..a448994e 100644 --- a/sql/0000012_dockerhubbuild.up.sql +++ b/sql/0000012_dockerhubbuild.up.sql @@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS dockerhubbuild ( Owner String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000013_azurecontainerpush.up.sql b/sql/0000013_azurecontainerpush.up.sql index a0f5916d..6dbb5aa2 100644 --- a/sql/0000013_azurecontainerpush.up.sql +++ b/sql/0000013_azurecontainerpush.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS azurecontainerpush ( Size Int32, SHAID String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; \ No newline at end of file diff --git a/sql/0000014_quaycontainerpush.up.sql b/sql/0000014_quaycontainerpush.up.sql index 2c249b56..6304fcc7 100644 --- a/sql/0000014_quaycontainerpush.up.sql +++ b/sql/0000014_quaycontainerpush.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS quaycontainerpush ( tag String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000015_trivysbom.up.sql b/sql/0000015_trivysbom.up.sql index 0e3a9c2e..7bc749ac 100644 --- a/sql/0000015_trivysbom.up.sql +++ b/sql/0000015_trivysbom.up.sql @@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS trivysbom ( serial_number String, version INTEGER, bom_format String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000016_azure_devops.up.sql b/sql/0000016_azure_devops.up.sql index 06a32dd2..bc7752a8 100644 --- a/sql/0000016_azure_devops.up.sql +++ b/sql/0000016_azure_devops.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS azure_devops ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000017_github.up.sql b/sql/0000017_github.up.sql index 96d9bf24..2f627dd5 100644 --- a/sql/0000017_github.up.sql +++ b/sql/0000017_github.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS github ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000018_gitlab.up.sql b/sql/0000018_gitlab.up.sql index 2403dff1..d47dd988 100644 --- a/sql/0000018_gitlab.up.sql +++ b/sql/0000018_gitlab.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS gitlab ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000019_bitbucket.up.sql b/sql/0000019_bitbucket.up.sql index adf08956..778e2ea7 100644 --- a/sql/0000019_bitbucket.up.sql +++ b/sql/0000019_bitbucket.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS bitbucket ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000001_events.up.sql b/sql/000001_events.up.sql index 2ba1313f..d165653f 100644 --- a/sql/000001_events.up.sql +++ b/sql/000001_events.up.sql @@ -13,7 +13,8 @@ CREATE TABLE IF NOT EXISTS events ( ImageName String, FirstTime String, LastTime String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000020_gitea.up.sql b/sql/0000020_gitea.up.sql index 1b42c0e7..3232bc56 100644 --- a/sql/0000020_gitea.up.sql +++ b/sql/0000020_gitea.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS gitea ( RepoName String, TimeStamp DateTime('UTC'), Event String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/0000021_kuberhealthy.up.sql b/sql/0000021_kuberhealthy.up.sql index 62ce6e51..3462ca6c 100644 --- a/sql/0000021_kuberhealthy.up.sql +++ b/sql/0000021_kuberhealthy.up.sql @@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS kuberhealthy ( Node String, LastRun DateTime('UTC'), AuthoritativePod String, - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000002_rakkess.up.sql b/sql/000002_rakkess.up.sql index 3542f59b..0bbd400a 100644 --- a/sql/000002_rakkess.up.sql +++ b/sql/000002_rakkess.up.sql @@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS rakkess ( List String, Update String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000003_DeprecatedAPIs.up.sql b/sql/000003_DeprecatedAPIs.up.sql index a7ed1b6e..46be7b93 100644 --- a/sql/000003_DeprecatedAPIs.up.sql +++ b/sql/000003_DeprecatedAPIs.up.sql @@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs ( Deprecated UInt8, Scope String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000004_DeletedAPIs.up.sql b/sql/000004_DeletedAPIs.up.sql index 931b93e7..d333b3a4 100644 --- a/sql/000004_DeletedAPIs.up.sql +++ b/sql/000004_DeletedAPIs.up.sql @@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs ( Deleted UInt8, Scope String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000005_jfrogcontainerpush.up.sql b/sql/000005_jfrogcontainerpush.up.sql index 6bac6d8d..a3a98fa6 100644 --- a/sql/000005_jfrogcontainerpush.up.sql +++ b/sql/000005_jfrogcontainerpush.up.sql @@ -9,7 +9,8 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush ( Tag String, Event String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000006_getall_resources.up.sql b/sql/000006_getall_resources.up.sql index 23991a21..32168213 100644 --- a/sql/000006_getall_resources.up.sql +++ b/sql/000006_getall_resources.up.sql @@ -5,7 +5,8 @@ CREATE TABLE IF NOT EXISTS getall_resources ( Resource String, Age String, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000007_outdated_images.up.sql b/sql/000007_outdated_images.up.sql index 479c0902..9779aead 100644 --- a/sql/000007_outdated_images.up.sql +++ b/sql/000007_outdated_images.up.sql @@ -7,7 +7,8 @@ CREATE TABLE IF NOT EXISTS outdated_images ( LatestVersion String, VersionsBehind Int64, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000008_kubescore.up.sql b/sql/000008_kubescore.up.sql index b6ee3e3f..2db441b1 100644 --- a/sql/000008_kubescore.up.sql +++ b/sql/000008_kubescore.up.sql @@ -13,7 +13,8 @@ CREATE TABLE IF NOT EXISTS kubescore ( file_name String, file_row BIGINT, EventTime DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate; diff --git a/sql/000009_trivy_vul.up.sql b/sql/000009_trivy_vul.up.sql index 8acb241d..2ebc670f 100644 --- a/sql/000009_trivy_vul.up.sql +++ b/sql/000009_trivy_vul.up.sql @@ -15,7 +15,8 @@ CREATE TABLE IF NOT EXISTS trivy_vul ( vul_severity String, vul_published_date DateTime('UTC'), vul_last_modified_date DateTime('UTC'), - ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}} + ExpiryDate DateTime DEFAULT now() + INTERVAL {{.TTLValue}} {{.TTLUnit}}, + ExportedAt DateTime DEFAULT NULL ) ENGINE = MergeTree() ORDER BY ExpiryDate TTL ExpiryDate;