From c5218a9358245a39579d6919c0e9cc7eb310e48f Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Mon, 1 Feb 2021 10:56:16 -0800 Subject: [PATCH 1/7] feat: added Alert query for bucket cardinality --- .../influxdb2_cardinality_now.yml | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index 3087db07..961b2bc5 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -1,19 +1,39 @@ apiVersion: influxdata.com/v2alpha1 kind: Label metadata: - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 spec: color: '#fafafc' name: cardinality --- apiVersion: influxdata.com/v2alpha1 +kind: Task +metadata: + name: ridiculous-elion-cae001 +spec: + associations: + - kind: Label + name: admiring-ardinghelli-4ae001 + every: 1d + name: 'Cardinality Alert ' + offset: 20m0s + query: "import \"influxdata/influxdb\"\nimport \"slack\"\n\n\nalert = (givenCard) + => if givenCard >= 1 then \n() => slack.message(\nurl: \"https://hooks.slack.com/services/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\",\nchannel: + \"#team-compute\",\ntext: \"TEST: Cardinality at \\\"${string(v: givenCard)}\\\"!\",\ncolor: + \"danger\",\n)\nelse \n() => 0\n\nbuckets()\n |> map(fn: (r) => {\ncards + = influxdb.cardinality(start: -1d, bucket: r.name)\n |> findRecord(idx: + 0, fn: (key) => (true))\nresult = alert(givenCard:cards._value)\n return{\nr + with card:cards._value,\nsent: result()\n}\n})" + status: active +--- +apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: amazing-boyd-93b003 + name: infallible-hellman-cae005 spec: associations: - kind: Label - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 language: flux name: bucket query: |- @@ -26,11 +46,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: blissful-morse-53b003 + name: victorious-hertz-cae009 spec: associations: - kind: Label - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 language: flux name: measurement query: |- @@ -42,11 +62,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Dashboard metadata: - name: nostalgic-einstein-13b001 + name: terrifying-dubinsky-4ae001 spec: associations: - kind: Label - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 charts: - axes: - base: "10" @@ -69,7 +89,7 @@ spec: name: Nineteen Eighty Four type: scale geom: line - height: 3 + height: 4 hoverDimension: auto kind: Xy legendOpacity: 1 @@ -155,7 +175,7 @@ spec: verticalTimeAxis: true timeFormat: YYYY-MM-DD HH:mm:ss width: 12 - yPos: 3 + yPos: 4 - colors: - hex: '#ffffff' id: base @@ -194,7 +214,7 @@ spec: verticalTimeAxis: true timeFormat: YYYY-MM-DD HH:mm:ss width: 12 - yPos: 7 + yPos: 8 - colors: - hex: '#ffffff' id: base @@ -234,5 +254,5 @@ spec: verticalTimeAxis: true timeFormat: YYYY-MM-DD HH:mm:ss width: 12 - yPos: 10 + yPos: 11 name: Cardinality Now From 88b6477baea45b61f164449b804e6087fa2d0c7f Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Mon, 1 Feb 2021 10:56:16 -0800 Subject: [PATCH 2/7] feat: added Alert query for bucket cardinality Signed-off-by: Arnav Aggarwal --- .../influxdb2_cardinality_now.yml | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index 3087db07..961b2bc5 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -1,19 +1,39 @@ apiVersion: influxdata.com/v2alpha1 kind: Label metadata: - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 spec: color: '#fafafc' name: cardinality --- apiVersion: influxdata.com/v2alpha1 +kind: Task +metadata: + name: ridiculous-elion-cae001 +spec: + associations: + - kind: Label + name: admiring-ardinghelli-4ae001 + every: 1d + name: 'Cardinality Alert ' + offset: 20m0s + query: "import \"influxdata/influxdb\"\nimport \"slack\"\n\n\nalert = (givenCard) + => if givenCard >= 1 then \n() => slack.message(\nurl: \"https://hooks.slack.com/services/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\",\nchannel: + \"#team-compute\",\ntext: \"TEST: Cardinality at \\\"${string(v: givenCard)}\\\"!\",\ncolor: + \"danger\",\n)\nelse \n() => 0\n\nbuckets()\n |> map(fn: (r) => {\ncards + = influxdb.cardinality(start: -1d, bucket: r.name)\n |> findRecord(idx: + 0, fn: (key) => (true))\nresult = alert(givenCard:cards._value)\n return{\nr + with card:cards._value,\nsent: result()\n}\n})" + status: active +--- +apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: amazing-boyd-93b003 + name: infallible-hellman-cae005 spec: associations: - kind: Label - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 language: flux name: bucket query: |- @@ -26,11 +46,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: blissful-morse-53b003 + name: victorious-hertz-cae009 spec: associations: - kind: Label - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 language: flux name: measurement query: |- @@ -42,11 +62,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Dashboard metadata: - name: nostalgic-einstein-13b001 + name: terrifying-dubinsky-4ae001 spec: associations: - kind: Label - name: kind-spence-53b001 + name: admiring-ardinghelli-4ae001 charts: - axes: - base: "10" @@ -69,7 +89,7 @@ spec: name: Nineteen Eighty Four type: scale geom: line - height: 3 + height: 4 hoverDimension: auto kind: Xy legendOpacity: 1 @@ -155,7 +175,7 @@ spec: verticalTimeAxis: true timeFormat: YYYY-MM-DD HH:mm:ss width: 12 - yPos: 3 + yPos: 4 - colors: - hex: '#ffffff' id: base @@ -194,7 +214,7 @@ spec: verticalTimeAxis: true timeFormat: YYYY-MM-DD HH:mm:ss width: 12 - yPos: 7 + yPos: 8 - colors: - hex: '#ffffff' id: base @@ -234,5 +254,5 @@ spec: verticalTimeAxis: true timeFormat: YYYY-MM-DD HH:mm:ss width: 12 - yPos: 10 + yPos: 11 name: Cardinality Now From 38f1678c776160814150acae86103a866f954a2a Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Wed, 3 Feb 2021 11:07:56 -0800 Subject: [PATCH 3/7] feat: added parameters --- .../influxdb2_cardinality_now.yml | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index 961b2bc5..1ddd540b 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -1,7 +1,7 @@ apiVersion: influxdata.com/v2alpha1 kind: Label metadata: - name: admiring-ardinghelli-4ae001 + name: sad-liskov-90e001 spec: color: '#fafafc' name: cardinality @@ -9,64 +9,85 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Task metadata: - name: ridiculous-elion-cae001 + name: objective-lalande-90e001 spec: associations: - kind: Label - name: admiring-ardinghelli-4ae001 + name: sad-liskov-90e001 every: 1d name: 'Cardinality Alert ' offset: 20m0s - query: "import \"influxdata/influxdb\"\nimport \"slack\"\n\n\nalert = (givenCard) - => if givenCard >= 1 then \n() => slack.message(\nurl: \"https://hooks.slack.com/services/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\",\nchannel: - \"#team-compute\",\ntext: \"TEST: Cardinality at \\\"${string(v: givenCard)}\\\"!\",\ncolor: - \"danger\",\n)\nelse \n() => 0\n\nbuckets()\n |> map(fn: (r) => {\ncards - = influxdb.cardinality(start: -1d, bucket: r.name)\n |> findRecord(idx: - 0, fn: (key) => (true))\nresult = alert(givenCard:cards._value)\n return{\nr - with card:cards._value,\nsent: result()\n}\n})" - status: active + query: |- + import "influxdata/influxdb" + import "slack" + + + + slackWebhook = "https://hooks.slack.com/services/XXXX/XXXX/XXXXX" + slackChannel = "##" + cardinalityThreshold = 1000000 + alert = (bucketCard, bucketName) => + (if givenCard >= cardinalityThreshold then () => + (slack.message( + url: slackWebhook, + channel: slackChannel, + text: "${bucketName} cardinality at ${string(v: givenCard)}!", + color: "warning", + )) else () => + (0)) + + buckets() + |> map(fn: (r) => { + cards = influxdb.cardinality(start: -1d, bucket: r.name) + |> findRecord(idx: 0, fn: (key) => + (true)) + result = alert(bucketCard: cards._value, bucketName: r.name) + + return {r with card: cards._value, sent: result()} + }) + status: inactive --- apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: infallible-hellman-cae005 + name: competent-lovelace-90e009 spec: associations: - kind: Label - name: admiring-ardinghelli-4ae001 + name: sad-liskov-90e001 language: flux - name: bucket + name: measurement query: |- - buckets() - |> filter(fn: (r) => r.name !~ /^_/) - |> rename(columns: {name: "_value"}) - |> keep(columns: ["_value"]) + import "influxdata/influxdb/v1" + + v1.measurements(bucket: v.bucket) type: query --- apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: victorious-hertz-cae009 + name: toasty-franklin-90e005 spec: associations: - kind: Label - name: admiring-ardinghelli-4ae001 + name: sad-liskov-90e001 language: flux - name: measurement + name: bucket query: |- - import "influxdata/influxdb/v1" - - v1.measurements(bucket: v.bucket) + buckets() + |> filter(fn: (r) => r.name !~ /^_/) + |> rename(columns: {name: "_value"}) + |> keep(columns: ["_value"]) type: query --- apiVersion: influxdata.com/v2alpha1 kind: Dashboard metadata: - name: terrifying-dubinsky-4ae001 + name: hungry-easley-10e001 spec: associations: - kind: Label - name: admiring-ardinghelli-4ae001 + name: sad-liskov-90e001 charts: - axes: - base: "10" From ef8ced761ae09eb614e2584b2fa23a90e268d144 Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Wed, 3 Feb 2021 11:13:19 -0800 Subject: [PATCH 4/7] fix: refactored parameter names --- .../influxdb2_cardinality_now.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index 1ddd540b..fd232c70 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -1,7 +1,7 @@ apiVersion: influxdata.com/v2alpha1 kind: Label metadata: - name: sad-liskov-90e001 + name: determined-torvalds-b69001 spec: color: '#fafafc' name: cardinality @@ -9,11 +9,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Task metadata: - name: objective-lalande-90e001 + name: unruffled-sinoussi-769001 spec: associations: - kind: Label - name: sad-liskov-90e001 + name: determined-torvalds-b69001 every: 1d name: 'Cardinality Alert ' offset: 20m0s @@ -27,11 +27,11 @@ spec: slackChannel = "##" cardinalityThreshold = 1000000 alert = (bucketCard, bucketName) => - (if givenCard >= cardinalityThreshold then () => + (if bucketCard >= cardinalityThreshold then () => (slack.message( url: slackWebhook, channel: slackChannel, - text: "${bucketName} cardinality at ${string(v: givenCard)}!", + text: "${bucketName} cardinality at ${string(v: bucketCard)}!", color: "warning", )) else () => (0)) @@ -50,11 +50,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: competent-lovelace-90e009 + name: admiring-fermat-b69007 spec: associations: - kind: Label - name: sad-liskov-90e001 + name: determined-torvalds-b69001 language: flux name: measurement query: |- @@ -66,11 +66,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Variable metadata: - name: toasty-franklin-90e005 + name: vivid-aryabhata-b69003 spec: associations: - kind: Label - name: sad-liskov-90e001 + name: determined-torvalds-b69001 language: flux name: bucket query: |- @@ -83,11 +83,11 @@ spec: apiVersion: influxdata.com/v2alpha1 kind: Dashboard metadata: - name: hungry-easley-10e001 + name: elated-mirzakhani-f69001 spec: associations: - kind: Label - name: sad-liskov-90e001 + name: determined-torvalds-b69001 charts: - axes: - base: "10" From 344e1ef912c1ff526f572921229e213554ac2a4c Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Tue, 16 Feb 2021 09:54:51 -0800 Subject: [PATCH 5/7] chore: added comments --- .../influxdb2_cardinality_now.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index fd232c70..acfa0bfc 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -14,18 +14,24 @@ spec: associations: - kind: Label name: determined-torvalds-b69001 - every: 1d + every: 1h name: 'Cardinality Alert ' offset: 20m0s query: |- import "influxdata/influxdb" import "slack" - - + // TODO: create a slack notification endpoint and paste link here + // documentation: https://docs.influxdata.com/influxdb/v2.0/monitor-alert/notification-endpoints/create/ slackWebhook = "https://hooks.slack.com/services/XXXX/XXXX/XXXXX" + + // TODO: specify the channel based on the webhook slackChannel = "##" + + // default cardinality cardinalityThreshold = 1000000 + + alert = (bucketCard, bucketName) => (if bucketCard >= cardinalityThreshold then () => (slack.message( @@ -38,7 +44,7 @@ spec: buckets() |> map(fn: (r) => { - cards = influxdb.cardinality(start: -1d, bucket: r.name) + cards = influxdb.cardinality(start: -1h, bucket: r.name) |> findRecord(idx: 0, fn: (key) => (true)) result = alert(bucketCard: cards._value, bucketName: r.name) From c34b8c949c0ab2d2f0fb50b13fffe05a7f56d0c6 Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Tue, 16 Feb 2021 10:04:35 -0800 Subject: [PATCH 6/7] chore: shortened offset to 5m --- influxdb2_operational_monitoring/influxdb2_cardinality_now.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index acfa0bfc..43504ef5 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -16,7 +16,7 @@ spec: name: determined-torvalds-b69001 every: 1h name: 'Cardinality Alert ' - offset: 20m0s + offset: 5m0s query: |- import "influxdata/influxdb" import "slack" From 121da3e4c49d5c80f87319687d7d2ded3a9bae10 Mon Sep 17 00:00:00 2001 From: Arnav Aggarwal Date: Tue, 16 Feb 2021 10:11:13 -0800 Subject: [PATCH 7/7] chore: embedded time to flux query --- influxdb2_operational_monitoring/influxdb2_cardinality_now.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml index 43504ef5..bb02e071 100644 --- a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -44,7 +44,7 @@ spec: buckets() |> map(fn: (r) => { - cards = influxdb.cardinality(start: -1h, bucket: r.name) + cards = influxdb.cardinality(start: -task.every, bucket: r.name) |> findRecord(idx: 0, fn: (key) => (true)) result = alert(bucketCard: cards._value, bucketName: r.name)