added ttl

intelops · Oct 31, 2023 · e595c9d · e595c9d
1 parent 5fd64f0
commit e595c9d
Show file tree

Hide file tree

Showing 22 changed files with 160 additions and 40 deletions.
diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go
@@ -17,8 +17,12 @@ const kubvizTable DBStatement = `
 		Event        String,
 		FirstTime   String,
 		LastTime    String
-	) engine=File(TabSeparated)
-`
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
+	`
+
 const rakeesTable DBStatement = `
 CREATE TABLE IF NOT EXISTS rakkess (
 	ClusterName String,
@@ -28,7 +32,10 @@ CREATE TABLE IF NOT EXISTS rakkess (
 	List String,
 	Update String,
 	EventTime DateTime('UTC')
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, EventTime) 
+TTL EventTime + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const kubePugDepricatedTable DBStatement = `
@@ -40,7 +47,10 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs (
 	Deprecated UInt8,
 	Scope String,
 	EventTime DateTime('UTC')
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, EventTime) 
+TTL EventTime + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const kubepugDeletedTable DBStatement = `
@@ -54,7 +64,10 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs (
 	Deleted UInt8,
 	Scope String,
 	EventTime DateTime('UTC')
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, EventTime) 
+TTL EventTime + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const jfrogContainerPushEventTable DBStatement = `
@@ -69,7 +82,10 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush (
 	Tag String,
 	Event String,
 	EventTime DateTime('UTC')
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, EventTime) 
+TTL EventTime + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const ketallTable DBStatement = `
@@ -80,7 +96,10 @@ const ketallTable DBStatement = `
 		Resource String,
 		Age String,
 		EventTime DateTime('UTC')
-	) engine=File(TabSeparated)
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
 	`
 
 const outdateTable DBStatement = `
@@ -93,7 +112,10 @@ CREATE TABLE IF NOT EXISTS outdated_images (
 	LatestVersion String,
 	VersionsBehind Int64,
 	EventTime DateTime('UTC')
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, EventTime) 
+TTL EventTime + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const kubescoreTable DBStatement = `
@@ -103,7 +125,10 @@ CREATE TABLE IF NOT EXISTS kubescore (
 	cluster_name String,
 	recommendations String,
 	EventTime DateTime('UTC')
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, EventTime) 
+TTL EventTime + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const trivyTableVul DBStatement = `
@@ -145,7 +170,10 @@ const trivyTableMisconfig DBStatement = `
 		misconfig_severity String,
 		misconfig_status String,
 		EventTime DateTime('UTC')
-	) engine=File(TabSeparated)
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
 	`
 
 const trivyTableImage DBStatement = `
@@ -173,7 +201,10 @@ const dockerHubBuildTable DBStatement = `
 		Owner String,
 		Event String,
 		EventTime DateTime('UTC')
-	) engine=File(TabSeparated)
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
 	`
 
 const azureContainerPushEventTable DBStatement = `
@@ -186,7 +217,10 @@ const azureContainerPushEventTable DBStatement = `
 		Size Int32,
 		SHAID String,
 		EventTime DateTime('UTC')
-	) engine=File(TabSeparated)
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
 	`
 
 const quayContainerPushEventTable DBStatement = `
@@ -199,7 +233,10 @@ const quayContainerPushEventTable DBStatement = `
 		tag String,
 		Event String,
 		EventTime DateTime('UTC')
-	) engine=File(TabSeparated)
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
 	`
 
 const trivySbomTable DBStatement = `
@@ -225,7 +262,10 @@ const trivySbomTable DBStatement = `
 		component_license_exp String,
 		component_purl String,
 		dependency_ref String
-	) engine=File(TabSeparated)
+	) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL metadata_timestamp + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
 	`
 
 const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event, EventTime) VALUES (?, ?, ?, ?, ?, ?, ?)"

diff --git a/gitmodels/dbstatement/dbstatement.go b/gitmodels/dbstatement/dbstatement.go
@@ -12,7 +12,10 @@ CREATE TABLE IF NOT EXISTS azure_devops (
 	RepoName String,
 	TimeStamp DateTime('UTC'),
 	Event String
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, TimeStamp) 
+TTL TimeStamp + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const InsertAzureDevops DBStatement = "INSERT INTO azure_devops ( Author, Provider, CommitID, CommitUrl, EventType, RepoName, TimeStamp, Event) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
@@ -27,7 +30,10 @@ CREATE TABLE IF NOT EXISTS github (
 	RepoName String,
 	TimeStamp DateTime('UTC'),
 	Event String
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, TimeStamp) 
+TTL TimeStamp + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const InsertGithub DBStatement = "INSERT INTO github ( Author, Provider, CommitID, CommitUrl, EventType, RepoName, TimeStamp, Event) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
@@ -42,7 +48,10 @@ CREATE TABLE IF NOT EXISTS gitlab (
 	RepoName String,
 	TimeStamp DateTime('UTC'),
 	Event String
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, TimeStamp) 
+TTL TimeStamp + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const InsertGitlab DBStatement = "INSERT INTO gitlab ( Author, Provider, CommitID, CommitUrl, EventType, RepoName, TimeStamp, Event) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
@@ -57,7 +66,10 @@ CREATE TABLE IF NOT EXISTS bitbucket (
 	RepoName String,
 	TimeStamp DateTime('UTC'),
 	Event String
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, TimeStamp) 
+TTL TimeStamp + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
 
 const InsertBitbucket DBStatement = "INSERT INTO bitbucket ( Author, Provider, CommitID, CommitUrl, EventType, RepoName, TimeStamp, Event) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
@@ -72,7 +84,9 @@ CREATE TABLE IF NOT EXISTS gitea (
 	RepoName String,
 	TimeStamp DateTime('UTC'),
 	Event String
-) engine=File(TabSeparated)
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, TimeStamp) 
+TTL TimeStamp + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
 `
-
 const InsertGitea DBStatement = "INSERT INTO gitea ( Author, Provider, CommitID, CommitUrl, EventType, RepoName, TimeStamp, Event) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
diff --git a/sql/20230915101223_events.up.sql b/sql/20230915101223_events.up.sql
@@ -12,4 +12,7 @@ CREATE TABLE IF NOT EXISTS events (
 	Event       String,
 	FirstTime   String,
 	LastTime    String
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915101317_rakkess.up.sql b/sql/20230915101317_rakkess.up.sql
@@ -6,4 +6,8 @@ CREATE TABLE IF NOT EXISTS rakkess (
 	List        String,
 	Update      String,
 	EventTime   DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
+
diff --git a/sql/20230915101358_DeprecatedAPIs.up.sql b/sql/20230915101358_DeprecatedAPIs.up.sql
@@ -6,4 +6,8 @@ CREATE TABLE IF NOT EXISTS DeprecatedAPIs (
 	Deprecated      UInt8,
 	Scope           String,
 	EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
+
diff --git a/sql/20230915101437_DeletedAPIs.up.sql b/sql/20230915101437_DeletedAPIs.up.sql
@@ -8,4 +8,8 @@ CREATE TABLE IF NOT EXISTS DeletedAPIs (
 	Deleted         UInt8,
 	Scope           String,
 	EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
+
diff --git a/sql/20230915101512_jfrogcontainerpush.up.sql b/sql/20230915101512_jfrogcontainerpush.up.sql
@@ -9,4 +9,8 @@ CREATE TABLE IF NOT EXISTS jfrogcontainerpush (
 	Tag            String,
 	Event          String,
     EventTime      DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
+
diff --git a/sql/20230915101549_getall_resources.up.sql b/sql/20230915101549_getall_resources.up.sql
@@ -5,4 +5,7 @@ CREATE TABLE IF NOT EXISTS getall_resources (
 	Resource    String,
 	Age         String,
     EventTime   DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915101643_outdated_images.up.sql b/sql/20230915101643_outdated_images.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS outdated_images (
 	LatestVersion   String,
 	VersionsBehind  Int64,
     EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915101736_kubescore.up.sql b/sql/20230915101736_kubescore.up.sql
@@ -4,4 +4,7 @@ CREATE TABLE IF NOT EXISTS kubescore (
 	cluster_name    String,
 	recommendations String,
     EventTime       DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915101811_trivy_vul.up.sql b/sql/20230915101811_trivy_vul.up.sql
@@ -15,4 +15,8 @@ CREATE TABLE IF NOT EXISTS trivy_vul (
 	vul_severity          String,
 	vul_published_date    DateTime('UTC'),
 	vul_last_modified_date DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, vul_last_modified_date) 
+TTL vul_last_modified_date + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
+`
diff --git a/sql/20230915101844_trivy_misconfig.up.sql b/sql/20230915101844_trivy_misconfig.up.sql
@@ -15,4 +15,7 @@ CREATE TABLE IF NOT EXISTS trivy_misconfig (
 	misconfig_severity String,
 	misconfig_status   String,
     EventTime          DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915101910_trivyimage.up.sql b/sql/20230915101910_trivyimage.up.sql
@@ -11,4 +11,8 @@ CREATE TABLE IF NOT EXISTS trivyimage (
 	vul_severity        String,
 	vul_published_date  DateTime('UTC'),
 	vul_last_modified_date DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+ORDER BY (ClusterName, vul_last_modified_date) 
+TTL vul_last_modified_date + INTERVAL 30 DAY
+SETTINGS index_granularity = 8192;
+`
diff --git a/sql/20230915102122_dockerhubbuild.up.sql b/sql/20230915102122_dockerhubbuild.up.sql
@@ -6,4 +6,7 @@ CREATE TABLE IF NOT EXISTS dockerhubbuild (
 	Owner         String,
 	Event         String,
     EventTime     DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102157_azurecontainerpush.up.sql b/sql/20230915102157_azurecontainerpush.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS azurecontainerpush (
 	Size           Int32,
 	SHAID          String,
     EventTime DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102229_quaycontainerpush.up.sql b/sql/20230915102229_quaycontainerpush.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS quaycontainerpush (
 	tag           String,
 	Event         String,
     EventTime DateTime('UTC')
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, EventTime) 
+	TTL EventTime + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102314_trivysbom.up.sql b/sql/20230915102314_trivysbom.up.sql
@@ -20,4 +20,7 @@ CREATE TABLE IF NOT EXISTS trivysbom (
 	component_license_exp String,
 	component_purl        String,
 	dependency_ref        String
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, metadata_timestamp) 
+	TTL metadata_timestamp + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102348_azure_devops.up.sql b/sql/20230915102348_azure_devops.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS azure_devops (
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
 	Event        String
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, TimeStamp) 
+	TTL TimeStamp + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102437_github.up.sql b/sql/20230915102437_github.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS github (
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
 	Event        String
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, TimeStamp) 
+	TTL TimeStamp + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102739_gitlab.up.sql b/sql/20230915102739_gitlab.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS gitlab (
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
 	Event        String
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, TimeStamp) 
+	TTL TimeStamp + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;
diff --git a/sql/20230915102817_bitbucket.up.sql b/sql/20230915102817_bitbucket.up.sql
@@ -7,4 +7,7 @@ CREATE TABLE IF NOT EXISTS bitbucket (
 	RepoName     String,
 	TimeStamp    DateTime('UTC'),
 	Event        String
-) engine=File(TabSeparated);
+) ENGINE = MergeTree()
+	ORDER BY (ClusterName, TimeStamp) 
+	TTL TimeStamp + INTERVAL 30 DAY
+	SETTINGS index_granularity = 8192;