Add support for sensitive data detection in iRule

f5devcentral · Oct 31, 2024 · 3e5c030 · 3e5c030
1 parent 9c5e349
commit 3e5c030
Show file tree

Hide file tree

Showing 8 changed files with 76 additions and 205 deletions.
diff --git a/diagrams/api-detection-screenshot.png b/diagrams/api-detection-screenshot.png
diff --git a/services/clickhouse/docker-entrypoint-initdb.d/init-api-detection-db.sh b/services/clickhouse/docker-entrypoint-initdb.d/init-api-detection-db.sh
@@ -13,6 +13,12 @@ clickhouse client -nm <<-EOSQL
         accesses UInt64,
         auth_headers_seen UInt64,
         auth_errors UInt64,
+        sensitive_headers_seen UInt64,
+        sensitive_payload_seen UInt64,
+        ssn_seen UInt64,
+        dob_seen UInt64,
+        creditcard_seen UInt64,
+        email_seen UInt64,
     ) 
     ENGINE = MergeTree 
     ORDER BY (accesses, Timestamp)

diff --git a/services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh b/services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh
@@ -35,12 +35,15 @@ clickhouse client <<-EOSQL
         Timestamp DateTime,
         path String CODEC(ZSTD(1)),
         hasAuthorization Bool,
+        hasSensitiveHeaders Bool,
+        hasSensitivePayload Bool,
         httpv LowCardinality(String) CODEC(ZSTD(1)),
         reqCType LowCardinality(String) CODEC(ZSTD(1)),
         resCType LowCardinality(String) CODEC(ZSTD(1)),
         host LowCardinality(String) CODEC(ZSTD(1)),
         hostname LowCardinality(String) CODEC(ZSTD(1)),
         method LowCardinality(String) CODEC(ZSTD(1)),
+        sensitiveDataTypes LowCardinality(String) CODEC(ZSTD(1)),
         statusCode UInt16,
     )
     ENGINE = MergeTree
@@ -57,6 +60,9 @@ clickhouse client <<-EOSQL
     path(LogAttributes['uri']) AS path,
     LogAttributes['host'] AS host,
     LogAttributes['hasAuthorization'] AS hasAuthorization,
+    LogAttributes['sensitiveInHeaders'] AS hasSensitiveHeaders,
+    LogAttributes['sensitiveInPayload'] AS hasSensitivePayload,
+    LogAttributes['sensitiveDataTypes'] AS sensitiveDataTypes,
     LogAttributes['method'] AS method,
     LogAttributes['statusCode'] AS statusCode,
     LogAttributes['reqCType'] AS reqCType,

diff --git a/services/clickhouse/schema/access_log.schema b/services/clickhouse/schema/access_log.schema
diff --git a/services/clickhouse/schema/api_endpoints.schema b/services/clickhouse/schema/api_endpoints.schema
diff --git a/services/clickhouse/schema/otel_null_log.schema b/services/clickhouse/schema/otel_null_log.schema
diff --git a/services/clickhouse/schema/view.schema b/services/clickhouse/schema/view.schema