From 4fe9a24c2dd9b9ff5c7a3cd5d7a69f76ffb74450 Mon Sep 17 00:00:00 2001 From: mprahl Date: Fri, 29 Mar 2024 17:04:01 -0400 Subject: [PATCH] Fix index errors for large values A Postgres unique constraint fails if the spec or message is too large. The solution was to split out the spec to a separate table and include a hash of the message that the unique constraint is on. Relates: https://issues.redhat.com/browse/ACM-10794 Signed-off-by: mprahl --- ...02_compliance_history_fix_indices.down.sql | 43 +++++++ ...0002_compliance_history_fix_indices.up.sql | 78 ++++++++++++ controllers/complianceeventsapi/server.go | 15 ++- controllers/complianceeventsapi/types.go | 81 ++++++++++-- test/e2e/case18_compliance_api_test.go | 117 ++++++++++++------ 5 files changed, 285 insertions(+), 49 deletions(-) create mode 100644 controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.down.sql create mode 100644 controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.up.sql diff --git a/controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.down.sql b/controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.down.sql new file mode 100644 index 00000000..555a57bd --- /dev/null +++ b/controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.down.sql @@ -0,0 +1,43 @@ +BEGIN; + +DROP EXTENSION IF EXISTS pgcrypto; + +-- Drop message_hash +ALTER TABLE compliance_events DROP CONSTRAINT compliance_events_cluster_id_policy_id_parent_policy_id_com_key; +DROP INDEX compliance_events_null1; +ALTER TABLE compliance_events DROP COLUMN message_hash; +ALTER TABLE compliance_events ADD CONSTRAINT compliance_events_cluster_id_policy_id_parent_policy_id_com_key UNIQUE (cluster_id, policy_id, parent_policy_id, compliance, message, timestamp); +CREATE UNIQUE INDEX compliance_events_null1 ON compliance_events (cluster_id, policy_id, compliance, message, timestamp) WHERE parent_policy_id IS NULL; +DROP INDEX compliance_events_messages; + + +-- Add back the spec column directly on the policies table +ALTER TABLE policies ADD spec JSONB; + +DO +$DO$ +DECLARE temprow RECORD; +BEGIN FOR temprow IN + SELECT "id", "spec_id" FROM policies + LOOP + UPDATE policies SET spec = (SELECT spec FROM specs WHERE id=temprow.spec_id) WHERE id = temprow.id; + END LOOP; +END; +$DO$; + +ALTER TABLE policies DROP CONSTRAINT policies_kind_api_group_name_namespace_spec_id_severity_key; +DROP INDEX policies_null1; +DROP INDEX policies_null2; +DROP INDEX policies_null3; + +ALTER TABLE policies DROP COLUMN spec_id; + +DROP TABLE specs; + +ALTER TABLE policies ADD CONSTRAINT policies_kind_api_group_name_namespace_spec_severity_key UNIQUE (kind, api_group, name, namespace, spec, severity); +CREATE UNIQUE INDEX policies_null1 ON policies (kind, api_group, name, spec, severity) WHERE namespace IS NULL; +CREATE UNIQUE INDEX policies_null2 ON policies (kind, api_group, name, namespace, spec) WHERE severity IS NULL; +CREATE UNIQUE INDEX policies_null3 ON policies (kind, api_group, name, spec) WHERE namespace IS NULL AND severity IS NULL; +CREATE INDEX idx_policies_spec ON policies (spec); + +COMMIT; diff --git a/controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.up.sql b/controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.up.sql new file mode 100644 index 00000000..2c754cc2 --- /dev/null +++ b/controllers/complianceeventsapi/migrations/000002_compliance_history_fix_indices.up.sql @@ -0,0 +1,78 @@ +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +-- If compliance messages are too long, the unique index gets too large and fails. This is a workaround for a unique +-- constraint while still allowing for long messages. + +-- Drop the soon to be invalid unique constraints. +ALTER TABLE compliance_events DROP CONSTRAINT compliance_events_cluster_id_policy_id_parent_policy_id_com_key; +DROP INDEX compliance_events_null1; + +-- SHA1 hex of the message for the uniqueness constraint. +ALTER TABLE compliance_events ADD message_hash VARCHAR(40); + +-- Populate the SHA1 hex +DO +$DO$ +DECLARE temprow RECORD; +BEGIN FOR temprow IN + SELECT "id", "message" FROM compliance_events + LOOP + UPDATE compliance_events SET message_hash = encode(digest(temprow.message, 'sha1'), 'hex') WHERE id = temprow.id; + END LOOP; +END; +$DO$; + +ALTER TABLE compliance_events ALTER COLUMN message_hash SET NOT NULL; + +-- Set the unique constraints +ALTER TABLE compliance_events ADD CONSTRAINT compliance_events_cluster_id_policy_id_parent_policy_id_com_key UNIQUE (cluster_id, policy_id, parent_policy_id, compliance, message_hash, timestamp); + +CREATE UNIQUE INDEX compliance_events_null1 ON compliance_events (cluster_id, policy_id, compliance, message_hash, timestamp) WHERE parent_policy_id IS NULL; + +-- Provide an index for equality comparisons on the message. +CREATE INDEX compliance_events_messages ON compliance_events USING HASH (message); + +-- If the spec is too long, the unique index gets too large and fails. This is a workaround for a unique +-- constraint while still allowing for spec uniqueness. + +CREATE TABLE specs( + id serial PRIMARY KEY, + spec JSONB NOT NULL, + EXCLUDE USING HASH (spec with =) +); + +-- Drop the soon to be invalid unique constraints. +ALTER TABLE policies DROP CONSTRAINT policies_kind_api_group_name_namespace_spec_severity_key; +DROP INDEX policies_null1; +DROP INDEX policies_null2; +DROP INDEX policies_null3; +DROP INDEX idx_policies_spec; + +ALTER TABLE policies ADD spec_id INT; +ALTER TABLE policies ADD FOREIGN KEY (spec_id) REFERENCES specs(id); + +-- Populate the specs table +DO +$DO$ +DECLARE temprow RECORD; +BEGIN FOR temprow IN + SELECT "id", "spec" FROM policies + LOOP + INSERT INTO specs (spec) VALUES (temprow.spec) ON CONFLICT DO NOTHING; + UPDATE policies SET spec_id = (SELECT id FROM specs WHERE spec=temprow.spec) WHERE id = temprow.id; + END LOOP; +END; +$DO$; + +ALTER TABLE policies ALTER COLUMN spec_id SET NOT NULL; + +ALTER TABLE policies DROP spec; + +ALTER TABLE policies ADD CONSTRAINT policies_kind_api_group_name_namespace_spec_id_severity_key UNIQUE (kind, api_group, name, namespace, spec_id, severity); +CREATE UNIQUE INDEX policies_null1 ON policies (kind, api_group, name, spec_id, severity) WHERE namespace IS NULL; +CREATE UNIQUE INDEX policies_null2 ON policies (kind, api_group, name, namespace, spec_id) WHERE severity IS NULL; +CREATE UNIQUE INDEX policies_null3 ON policies (kind, api_group, name, spec_id) WHERE namespace IS NULL AND severity IS NULL; + +COMMIT; diff --git a/controllers/complianceeventsapi/server.go b/controllers/complianceeventsapi/server.go index 913274a7..20f567da 100644 --- a/controllers/complianceeventsapi/server.go +++ b/controllers/complianceeventsapi/server.go @@ -569,14 +569,18 @@ func setAuthorizedClusters(ctx context.Context, db *sql.DB, parsed *queryOptions // generateGetComplianceEventsQuery will return a SELECT query with results ready to be parsed by // scanIntoComplianceEvent. The caller is responsible for adding filters to the query. func generateGetComplianceEventsQuery(includeSpec bool) string { - return fmt.Sprintf(`SELECT %s + query := `SELECT %s FROM compliance_events LEFT JOIN clusters ON compliance_events.cluster_id = clusters.id LEFT JOIN parent_policies ON compliance_events.parent_policy_id = parent_policies.id - LEFT JOIN policies ON compliance_events.policy_id = policies.id`, - strings.Join(generateSelectedArgs(includeSpec), ", "), - ) + LEFT JOIN policies ON compliance_events.policy_id = policies.id` + + if includeSpec { + query += "\n LEFT JOIN specs ON policies.spec_id=specs.id" + } + + return fmt.Sprintf(query, strings.Join(generateSelectedArgs(includeSpec), ", ")) } func generateSelectedArgs(includeSpec bool) []string { @@ -604,7 +608,7 @@ func generateSelectedArgs(includeSpec bool) []string { } if includeSpec { - selectArgs = append(selectArgs, "policies.spec") + selectArgs = append(selectArgs, "specs.spec") } return selectArgs @@ -1145,6 +1149,7 @@ func getComplianceEventsQuery(whereClause string, queryArgs *queryOptions) strin // LEFT JOIN clusters ON compliance_events.cluster_id = clusters.id // LEFT JOIN parent_policies ON compliance_events.parent_policy_id = parent_policies.id // LEFT JOIN policies ON compliance_events.policy_id = policies.id + // LEFT JOIN specs ON policies.spec_id=specs.id // WHERE (policies.name=$1 OR policies.name=$2) AND (policies.kind=$3) // ORDER BY compliance_events.timestamp desc // LIMIT 20 diff --git a/controllers/complianceeventsapi/types.go b/controllers/complianceeventsapi/types.go index e9b25e6a..62a724bd 100644 --- a/controllers/complianceeventsapi/types.go +++ b/controllers/complianceeventsapi/types.go @@ -2,8 +2,10 @@ package complianceeventsapi import ( "context" + "crypto/sha1" "database/sql" "database/sql/driver" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -280,11 +282,24 @@ func (e EventDetails) Validate() error { } func (e *EventDetails) InsertQuery() (string, []any) { + hasher := sha1.New() + hasher.Write([]byte(e.Message)) + messageHash := hex.EncodeToString(hasher.Sum(nil)) + sql := `INSERT INTO compliance_events` + - `(cluster_id, compliance, message, metadata, parent_policy_id, policy_id, reported_by, timestamp) ` + - `VALUES($1, $2, $3, $4, $5, $6, $7, $8)` + `(cluster_id, compliance, message, message_hash, metadata, parent_policy_id, policy_id, reported_by, ` + + `timestamp) ` + + `VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9)` values := []any{ - e.ClusterID, e.Compliance, e.Message, e.Metadata, e.ParentPolicyID, e.PolicyID, e.ReportedBy, e.Timestamp, + e.ClusterID, + e.Compliance, + e.Message, + messageHash, + e.Metadata, + e.ParentPolicyID, + e.PolicyID, + e.ReportedBy, + e.Timestamp, } return sql, values @@ -408,6 +423,37 @@ func ParentPolicyFromPolicyObj(plc *policiesv1.Policy) ParentPolicy { } } +type Spec struct { + KeyID int32 `db:"id" json:"id"` + Spec JSONMap `json:"spec,omitempty"` +} + +func (s *Spec) InsertQuery() (string, []any) { + sql := `INSERT INTO specs(spec) VALUES($1)` + values := []any{s.Spec} + + return sql, values +} + +func (s *Spec) SelectQuery(returnedColumns ...string) (string, []any) { + if len(returnedColumns) == 0 { + returnedColumns = []string{"*"} + } + + sql := fmt.Sprintf( + `SELECT %s FROM specs WHERE spec=$1`, + strings.Join(returnedColumns, ", "), + ) + + values := []any{s.Spec} + + return sql, values +} + +func (s *Spec) GetOrCreate(ctx context.Context, db *sql.DB) error { + return getOrCreate(ctx, db, s) +} + func PolicyFromUnstructured(obj unstructured.Unstructured) *Policy { policy := &Policy{} @@ -447,7 +493,8 @@ type Policy struct { APIGroup string `db:"api_group" json:"apiGroup"` Name string `db:"name" json:"name"` Namespace *string `db:"namespace" json:"namespace"` - Spec JSONMap `db:"spec" json:"spec,omitempty"` + Spec JSONMap `json:"spec,omitempty"` + SpecID int32 `db:"spec_id" json:"-"` Severity *string `db:"severity" json:"severity"` } @@ -475,21 +522,30 @@ func (p *Policy) Validate() error { func (p *Policy) InsertQuery() (string, []any) { sql := `INSERT INTO policies` + - `(api_group, kind, name, namespace, severity, spec)` + + `(api_group, kind, name, namespace, severity, spec_id)` + `VALUES($1, $2, $3, $4, $5, $6)` - values := []any{p.APIGroup, p.Kind, p.Name, p.Namespace, p.Severity, p.Spec} + values := []any{p.APIGroup, p.Kind, p.Name, p.Namespace, p.Severity, p.SpecID} return sql, values } func (p *Policy) SelectQuery(returnedColumns ...string) (string, []any) { if len(returnedColumns) == 0 { - returnedColumns = []string{"*"} + returnedColumns = []string{ + "policies.id", + "policies.kind", + "policies.api_group", + "policies.name", + "policies.namespace", + "policies.severity", + "specs.spec", + } } sql := fmt.Sprintf( `SELECT %s FROM policies `+ - `WHERE api_group=$1 AND kind=$2 AND name=$3 AND spec=$4`, + `LEFT JOIN specs ON policies.spec_id=specs.id`+ + ` WHERE api_group=$1 AND kind=$2 AND name=$3 AND spec=$4`, strings.Join(returnedColumns, ", "), ) @@ -517,6 +573,15 @@ func (p *Policy) SelectQuery(returnedColumns ...string) (string, []any) { } func (p *Policy) GetOrCreate(ctx context.Context, db *sql.DB) error { + if p.SpecID == 0 { + spec := &Spec{Spec: p.Spec} + if err := getOrCreate(ctx, db, spec); err != nil { + return err + } + + p.SpecID = spec.KeyID + } + return getOrCreate(ctx, db, p) } diff --git a/test/e2e/case18_compliance_api_test.go b/test/e2e/case18_compliance_api_test.go index 3ea9e5f4..76a04868 100644 --- a/test/e2e/case18_compliance_api_test.go +++ b/test/e2e/case18_compliance_api_test.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/http" "strings" "time" @@ -47,8 +48,11 @@ var ( ) const ( - wrongSAYaml = "../resources/case18_compliance_api_test/wrong_service_account.yaml" - subsetSAYaml = "../resources/case18_compliance_api_test/subset_service_account.yaml" + wrongSAYaml = "../resources/case18_compliance_api_test/wrong_service_account.yaml" + subsetSAYaml = "../resources/case18_compliance_api_test/subset_service_account.yaml" + policiesQueryByName = "SELECT policies.id, policies.kind, policies.api_group, policies.name, " + + "policies.namespace, policies.severity, specs.spec FROM policies " + + "LEFT JOIN specs ON policies.spec_id=specs.id WHERE name = $1 " ) func getTableNames(db *sql.DB) ([]string, error) { @@ -159,14 +163,16 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" It("Migrates from a clean database", func(ctx context.Context) { tableNames, err := getTableNames(db) Expect(err).ToNot(HaveOccurred()) - Expect(tableNames).To(ContainElements("clusters", "parent_policies", "policies", "compliance_events")) + Expect(tableNames).To(ContainElements( + "clusters", "parent_policies", "policies", "compliance_events", "specs", + )) migrationVersionRows := db.QueryRow("SELECT version, dirty FROM schema_migrations") var version int var dirty bool err = migrationVersionRows.Scan(&version, &dirty) Expect(err).ToNot(HaveOccurred()) - Expect(version).To(Equal(1)) + Expect(version).To(Equal(2)) Expect(dirty).To(BeFalse()) }) }) @@ -260,7 +266,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) It("Should have created the policy in a table", func() { - rows, err := db.Query("SELECT * FROM policies WHERE name = $1", "etcd-encryption1") + rows, err := db.Query(policiesQueryByName, "etcd-encryption1") Expect(err).ToNot(HaveOccurred()) count := 0 @@ -271,11 +277,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" apiGroup string name string ns *string - spec complianceeventsapi.JSONMap severity *string + spec complianceeventsapi.JSONMap ) - err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &spec, &severity) + err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &severity, &spec) Expect(err).ToNot(HaveOccurred()) Expect(id).NotTo(Equal(0)) @@ -311,10 +317,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" timestamp string metadata complianceeventsapi.JSONMap reportedBy *string + messageHash string ) err := rows.Scan(&id, &clusterID, &policyID, &parentPolicyID, &compliance, &message, ×tamp, - &metadata, &reportedBy) + &metadata, &reportedBy, &messageHash) Expect(err).ToNot(HaveOccurred()) Expect(id).To(Equal(1)) @@ -326,6 +333,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Expect(message).To(Equal("configmaps [etcd] not found in namespace default")) Expect(timestamp).To(Equal("2023-01-01T01:01:01.111Z")) Expect(metadata).To(HaveKeyWithValue("test", true)) + Expect(messageHash).To(Equal("3feb697b1df4585ef4ac1623ca233a34b2a2ea84")) Expect(reportedBy).ToNot(BeNil()) Expect(*reportedBy).To(Equal("optional-test")) count++ @@ -456,7 +464,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" "event": { "compliance": "Compliant", "message": "configmaps [etcd] found in namespace default", - "timestamp": "2023-02-02T02:02:02.222Z" + "timestamp": "2023-02-02T02:02:02.223Z" } }`) @@ -488,7 +496,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) It("Should have created two policies in a table despite having the same name", func() { - rows, err := db.Query("SELECT * FROM policies WHERE name = $1", "etcd-encryption2") + rows, err := db.Query(policiesQueryByName, "etcd-encryption2") Expect(err).ToNot(HaveOccurred()) rowCount := 0 @@ -500,11 +508,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" apiGroup string name string ns *string - spec complianceeventsapi.JSONMap severity *string + spec complianceeventsapi.JSONMap ) - err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &spec, &severity) + err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &severity, &spec) Expect(err).ToNot(HaveOccurred()) rowCount++ @@ -515,8 +523,8 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) It("Should have created both events in a table", func() { - rows, err := db.Query("SELECT * FROM compliance_events WHERE timestamp = $1", - "2023-02-02T02:02:02.222Z") + rows, err := db.Query("SELECT * FROM compliance_events WHERE timestamp > $1", + "2023-02-02T02:02:02.221Z") Expect(err).ToNot(HaveOccurred()) messages := make([]string, 0) @@ -531,10 +539,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" timestamp string metadata *string reportedBy *string + messageHash string ) err := rows.Scan(&id, &clusterID, &policyID, &parentPolicyID, &compliance, &message, ×tamp, - &metadata, &reportedBy) + &metadata, &reportedBy, &messageHash) Expect(err).ToNot(HaveOccurred()) messages = append(messages, message) @@ -1066,7 +1075,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) It("Should have only created one policy in a table", func() { - rows, err := db.Query("SELECT * FROM policies WHERE name = $1", "common") + rows, err := db.Query(policiesQueryByName, "common") Expect(err).ToNot(HaveOccurred()) specs := make([]complianceeventsapi.JSONMap, 0, 1) @@ -1077,11 +1086,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" apiGroup string name string ns *string - spec complianceeventsapi.JSONMap severity *string + spec complianceeventsapi.JSONMap ) - err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &spec, &severity) + err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &severity, &spec) Expect(err).ToNot(HaveOccurred()) Expect(id).NotTo(Equal(0)) @@ -1109,10 +1118,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" timestamp string metadata *string reportedBy *string + messageHash string ) err := rows.Scan(&id, &clusterID, &policyID, &parentPolicyID, &compliance, &message, ×tamp, - &metadata, &reportedBy) + &metadata, &reportedBy, &messageHash) Expect(err).ToNot(HaveOccurred()) Expect(id).NotTo(Equal(0)) @@ -1247,7 +1257,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) It("Should have created a single policy", func() { - rows, err := db.Query("SELECT * FROM policies WHERE name = $1", "common-a") + rows, err := db.Query(policiesQueryByName, "common-a") Expect(err).ToNot(HaveOccurred()) ids := make([]int, 0) @@ -1258,11 +1268,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" apiGroup string name string ns *string - spec complianceeventsapi.JSONMap severity *string + spec complianceeventsapi.JSONMap ) - err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &spec, &severity) + err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &severity, &spec) Expect(err).ToNot(HaveOccurred()) Expect(id).NotTo(Equal(0)) @@ -1356,7 +1366,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) It("Should have created two policies in the table, with different namespaces", func() { - rows, err := db.Query("SELECT * FROM policies WHERE name = $1", "common-b") + rows, err := db.Query(policiesQueryByName, "common-b") Expect(err).ToNot(HaveOccurred()) ids := make([]int, 0) @@ -1370,11 +1380,11 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" apiGroup string name string ns *string - spec complianceeventsapi.JSONMap severity *string + spec complianceeventsapi.JSONMap ) - err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &spec, &severity) + err := rows.Scan(&id, &kind, &apiGroup, &name, &ns, &severity, &spec) Expect(err).ToNot(HaveOccurred()) Expect(id).NotTo(Equal(0)) @@ -1606,7 +1616,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Entry( "Filter by event.message_includes", []string{"event.message_includes=etcd"}, - []float64{2, 3, 1}, + []float64{3, 2, 1}, ), Entry( "Filter by event.message_includes and ensure special characters are escaped", @@ -1638,7 +1648,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" []string{ "event.timestamp_after=2023-01-01T01:01:01.111Z", "event.timestamp_before=2023-04-01T01:01:01.111Z", }, - []float64{4, 2, 3, 11, 10}, + []float64{4, 3, 2, 11, 10}, ), Entry( "Filter by parent_policy.categories", @@ -1648,7 +1658,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Entry( "Filter by parent_policy.categories is null", []string{"parent_policy.categories"}, - []float64{9, 8, 7, 2, 3, 11, 10}, + []float64{9, 8, 7, 3, 2, 11, 10}, ), Entry( "Filter by parent_policy.controls", @@ -1658,7 +1668,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Entry( "Filter by parent_policy.controls is null", []string{"parent_policy.controls"}, - []float64{9, 8, 7, 2, 3, 11, 10}, + []float64{9, 8, 7, 3, 2, 11, 10}, ), Entry( "Filter by parent_policy.id", @@ -1683,7 +1693,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Entry( "Filter by parent_policy.standards is null", []string{"parent_policy.standards"}, - []float64{9, 8, 2, 3, 11, 10}, + []float64{9, 8, 3, 2, 11, 10}, ), Entry( "Filter by policy.apiGroup", @@ -1723,7 +1733,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Entry( "Filter by policy.namespace is null", []string{"policy.namespace"}, - []float64{9, 8, 6, 7, 5, 4, 2, 3, 11}, + []float64{9, 8, 6, 7, 5, 4, 3, 2, 11}, ), Entry( "Filter by policy.severity", @@ -1733,7 +1743,7 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" Entry( "Filter by policy.severity is null", []string{"policy.severity"}, - []float64{2, 3}, + []float64{3, 2}, ), ) @@ -2047,6 +2057,41 @@ var _ = Describe("Test the compliance events API", Label("compliance-events-api" }) }) + Describe("Large values", func() { + It("Should allow a large spec and message", func(ctx context.Context) { + longString := "" + charset := []rune{ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + } + + for i := 0; i < 100000; i++ { + c := charset[rand.Intn(len(charset))] + longString += string(c) + } + + payload := []byte(fmt.Sprintf(`{ + "cluster": { + "name": "managed2", + "cluster_id": "test2-managed2-fake-uuid-2" + }, + "policy": { + "apiGroup": "policy.open-cluster-management.io", + "kind": "ConfigurationPolicy", + "name": "duplicate-test", + "spec": {"test": "%s"} + }, + "event": { + "compliance": "NonCompliant", + "message": "%s", + "timestamp": "2023-02-02T02:02:02.222Z" + } + }`, longString, longString)) + + By("POST the event") + Eventually(postEvent(ctx, payload, clientToken), "5s", "1s").ShouldNot(HaveOccurred()) + }) + }) + Describe("Test authorization", func() { Describe("Test method Get", func() { It("Should return unauthorized when it is empty token", func(ctx context.Context) { @@ -2309,8 +2354,8 @@ var _ = Describe("Test query generation", Label("compliance-events-api"), func() } sql, vals := policy.SelectQuery("id") Expect(sql).To(Equal( - "SELECT id FROM policies WHERE api_group=$1 AND kind=$2 AND name=$3 AND spec=$4 AND namespace is NULL " + - "AND severity is NULL", + "SELECT id FROM policies LEFT JOIN specs ON policies.spec_id=specs.id WHERE api_group=$1 AND kind=$2 " + + "AND name=$3 AND spec=$4 AND namespace is NULL AND severity is NULL", )) Expect(vals).To(HaveLen(4)) }) @@ -2329,8 +2374,8 @@ var _ = Describe("Test query generation", Label("compliance-events-api"), func() } sql, vals := policy.SelectQuery("id") Expect(sql).To(Equal( - "SELECT id FROM policies WHERE api_group=$1 AND kind=$2 AND name=$3 AND spec=$4 AND namespace=$5 " + - "AND severity=$6", + "SELECT id FROM policies LEFT JOIN specs ON policies.spec_id=specs.id WHERE api_group=$1 AND kind=$2 " + + "AND name=$3 AND spec=$4 AND namespace=$5 AND severity=$6", )) Expect(vals).To(HaveLen(6)) })