From 5c369fb66e97a1b1ccc58cc9969132cba18ef8fb Mon Sep 17 00:00:00 2001 From: Anaisdg Date: Thu, 17 Dec 2020 12:52:14 -0600 Subject: [PATCH] fix: reorganized cardinality templates --- .../influxdb2_cardinality_now.yml | 238 ++++++++++++++++++ influxdb2_operational_monitoring/readme.md | 59 +++-- 2 files changed, 282 insertions(+), 15 deletions(-) create mode 100644 influxdb2_operational_monitoring/influxdb2_cardinality_now.yml diff --git a/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml new file mode 100644 index 00000000..3087db07 --- /dev/null +++ b/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml @@ -0,0 +1,238 @@ +apiVersion: influxdata.com/v2alpha1 +kind: Label +metadata: + name: kind-spence-53b001 +spec: + color: '#fafafc' + name: cardinality +--- +apiVersion: influxdata.com/v2alpha1 +kind: Variable +metadata: + name: amazing-boyd-93b003 +spec: + associations: + - kind: Label + name: kind-spence-53b001 + language: flux + name: bucket + query: |- + buckets() + |> filter(fn: (r) => r.name !~ /^_/) + |> rename(columns: {name: "_value"}) + |> keep(columns: ["_value"]) + type: query +--- +apiVersion: influxdata.com/v2alpha1 +kind: Variable +metadata: + name: blissful-morse-53b003 +spec: + associations: + - kind: Label + name: kind-spence-53b001 + language: flux + name: measurement + query: |- + import "influxdata/influxdb/v1" + + v1.measurements(bucket: v.bucket) + type: query +--- +apiVersion: influxdata.com/v2alpha1 +kind: Dashboard +metadata: + name: nostalgic-einstein-13b001 +spec: + associations: + - kind: Label + name: kind-spence-53b001 + charts: + - axes: + - base: "10" + name: x + scale: linear + - base: "10" + name: y + scale: linear + colors: + - hex: '#31C0F6' + id: 5b3aea53-4f8a-47e5-bda3-3e9299df5fa1 + name: Nineteen Eighty Four + type: scale + - hex: '#A500A5' + id: e8fec99c-5ff4-43aa-b30a-d3c88d343fb9 + name: Nineteen Eighty Four + type: scale + - hex: '#FF7E27' + id: 3f9c84bb-6dfa-42e3-aa35-2e87e6a88842 + name: Nineteen Eighty Four + type: scale + geom: line + height: 3 + hoverDimension: auto + kind: Xy + legendOpacity: 1 + legendOrientationThreshold: 10 + name: Cardinality Per Bucket + position: overlaid + queries: + - query: |- + import "influxdata/influxdb" + import "experimental" + + time = experimental.subDuration( + d: v.timeRangeStart, + from: now(), + ) + + buckets() + |> map(fn: (r) => { + cardinality = influxdb.cardinality(bucket: r.name, start: -7d) + |> findRecord(idx: 0, fn: (key) => + (true)) + return { + _time: time, + _measurement: "buckets", + bucket: r.name, + _field: "cardinality", + _value: cardinality._value, + } + }) + |> group(columns: ["bucket"], mode:"by") + width: 12 + xCol: _time + yCol: _value + - colors: + - hex: '#ffffff' + id: base + name: white + type: text + fieldOptions: + - displayName: _field + fieldName: _field + - displayName: _measurement + fieldName: _measurement + - displayName: _time + fieldName: _time + - displayName: bucket + fieldName: bucket + visible: true + - displayName: _value + fieldName: _value + visible: true + height: 4 + kind: Table + name: Top 10 Buckets + queries: + - query: |- + import "influxdata/influxdb" + import "experimental" + + time = experimental.subDuration( + d: v.timeRangeStart, + from: now(), + ) + + buckets() + |> map(fn: (r) => { + cardinality = influxdb.cardinality(bucket: r.name, start: -7d) + |> findRecord(idx: 0, fn: (key) => + (true)) + return { + _time: time, + _measurement: "buckets", + bucket: r.name, + _field: "cardinality", + _value: cardinality._value, + } + }) + |> group(columns: ["bucket"], mode:"by") + |> last() + |> group() + |> top(n:10) + tableOptions: + verticalTimeAxis: true + timeFormat: YYYY-MM-DD HH:mm:ss + width: 12 + yPos: 3 + - colors: + - hex: '#ffffff' + id: base + name: white + type: text + fieldOptions: + - displayName: bucket + fieldName: bucket + visible: true + - displayName: measurement + fieldName: measurement + visible: true + - displayName: cardinality + fieldName: _value + visible: true + height: 3 + kind: Table + name: Cardinality By Mesurement + queries: + - query: |- + import "influxdata/influxdb" + import "influxdata/influxdb/schema" + + + schema.measurements(bucket: v.bucket) + |> map(fn: (r) => { + m = r._value + return { + bucket: v.bucket, + measurement: m, + _value: (influxdb.cardinality(bucket: v.bucket, start: v.timeRangeStart, predicate: (r) => r._measurement == m) |> findRecord(idx:0, fn:(key) => true))._value + } + }) + |> sort(desc:true) + tableOptions: + verticalTimeAxis: true + timeFormat: YYYY-MM-DD HH:mm:ss + width: 12 + yPos: 7 + - colors: + - hex: '#ffffff' + id: base + name: white + type: text + fieldOptions: + - displayName: tag + fieldName: tag + visible: true + - displayName: cardinality + fieldName: _value + visible: true + height: 3 + kind: Table + name: Cardinality By Tag + queries: + - query: |- + import "influxdata/influxdb/schema" + + cardinalityByTag = (bucket, measurement) => schema.tagKeys(bucket: bucket, predicate: (r) => r._measurement == measurement) + |> filter(fn: (r) => r._value != "_start" and r._value != "_stop") + |> map( + fn: (r) => ({ + tag: r._value, + _value: (schema.tagValues(bucket: bucket, tag: r._value) + |> count() + |> findRecord(fn: (key) => true, idx: 0))._value, + }), + ) + |> group(columns: ["tag"]) + |> sum() + + cardinalityByTag(bucket: v.bucket, measurement: v.measurement) + |> group() + |> top(n: 10) + tableOptions: + verticalTimeAxis: true + timeFormat: YYYY-MM-DD HH:mm:ss + width: 12 + yPos: 10 + name: Cardinality Now diff --git a/influxdb2_operational_monitoring/readme.md b/influxdb2_operational_monitoring/readme.md index 1d07fd96..92d19594 100644 --- a/influxdb2_operational_monitoring/readme.md +++ b/influxdb2_operational_monitoring/readme.md @@ -2,7 +2,9 @@ Provided by: InfluxData -This InfluxDB Template can be used to monitor your InfluxDB tasks and cardinality on your already running InfluxDB 2 instance. This template is only compatible with both the open source and the Cloud version of InfluxDB 2. +This directory contains two InfluxDB Templates, [Operational Monitoring Template](./influxdb2_operational_monitoring.yml) and [Cardinality Now Template](./cardinality_now.yml), that can be used to monitor your InfluxDB tasks and cardinality on your already running InfluxDB 2 instance. These templates are only compatible with the Cloud version of InfluxDB 2. + +### Operational Monitoring Template Monitoring your tasks allows you to: 1) Verify that critical downsampling and data transforation tasks are succeeding. @@ -18,37 +20,64 @@ This `Cardinality Explorer` dashboard is used by InfluxData to monitor the cardi ![Cardinality Explorer Screenshot](img/cardinality-explorer-dashboard1.png) If the cardinality of a bucket is increasing exponentially, use this dashboard to drill into the cardinality of specific measurements to identify the source of runaway cardinality. ![Cardinality Explorer Screenshot](img/cardinality-explorer-dashboard2.png) -### Quick Install - -#### InfluxDB UI - -In the InfluxDB UI, go to Settings->Templates and enter this URL: https://raw.githubusercontent.com/influxdata/community-templates/master/influxdb2_operational_monitoring/influxdb2_operational_monitoring.yml - -#### Influx CLI -If you have your InfluxDB credentials [configured in the CLI](https://v2.docs.influxdata.com/v2.0/reference/cli/influx/config/), you can install this template with: -``` -influx apply -u https://raw.githubusercontent.com/influxdata/community-templates/master/influxdb2_operational_monitoring/influxdb2_operational_monitoring.yml -``` - -## Included Resources +#### Included Resources - 1 Bucket: `cardinality`, Forever - 1 Task: `cardinality_by_bucket`, 1h - 2 Labels: `operational_monitoring` - 2 Dashboard: `Task Summary Dashboard`, `Cardinality Explorer` -## Setup Instructions +### Cardinality Now + +This is a slimmed down version of the [Operational Monitoring Template](../influxdb2_operational_monitoring). This version allows you to monitor your currently cardinality without adding additional buckets or tasks to your instance (i.e. without increasing the cardinality of your instance). + +This `Cardinality Now` dashboard can help you identify your source of runaway cardinality when you have reached your cardinality limit. +![Cardinality Explorer Screenshot](img/cardinality-explorer-dashboard2.png) +Use this dashboard to drill into the cardinality of specific measurements to identify the source of runaway cardinality. + +#### Included Resources + - 1 Labels: `cardinality` + - 1 Dashboard: `Cardinality Now` + +### Quick Install + +#### InfluxDB UI + +In the InfluxDB UI, go to Settings->Templates and enter the URL for the template you wish to apply. +- For the Operational Monitoring Template: https://raw.githubusercontent.com/influxdata/community-templates/master/influxdb2_operational_monitoring/influxdb2_operational_monitoring.yml +- For the Cardinality Now Template: https://raw.githubusercontent.com/influxdata/community-templates/master/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml + +#### Influx CLI +If you have your InfluxDB credentials [configured in the CLI](https://v2.docs.influxdata.com/v2.0/reference/cli/influx/config/), you can install each template with their respective URLs: +- For the Operational Monitoring Template: + ``` + influx apply -u https://raw.githubusercontent.com/influxdata/community-templates/master/influxdb2_operational_monitoring/influxdb2_operational_monitoring.yml + ``` +- For the Cardinality Now Template: + ``` + influx apply -u https://raw.githubusercontent.com/influxdata/community-templates/master/influxdb2_operational_monitoring/influxdb2_cardinality_now.yml + ``` +### Setup Instructions General instructions on using InfluxDB Templates can be found in the [use a template](../docs/use_a_template.md) document. +#### Setup Instruction for the Operational Monitoring Template + The data for the `Task Summary Dashboard` is populated by data collected from the default `_tasks` bucket. The `Cardinality Explorer` dashboard is populated by data collected from the `cardinality` bucket. The best way to set this up is to first follow the instructions for installing any template. That will create the label, bucket, and dashboards for you. You must already have a task running in order to populate the `Task Summary Dashboard` with data. [Get started with InfluxDB tasks](https://docs.influxdata.com/influxdb/v2.0/process-data/get-started/) to create a task and populate the dashboard. +#### Setup Instructions for the Cardinality Now Template + + You must at least one bucket and have written data to that bucket for the `Cardinality Now` dashboard to be populated with data. + + ## Contact - Author: Anais Dotis-Georgiou - Email: anais@influxdata.com - Github: [@Anaisdg](https://github.com/Anaisdg) - Influx Slack: [@Anais](https://influxdata.com/slack) + +