diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d1ee09 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# Gatekeeper Log Exporter + +Gatekeeper Log Exporter (GKLE for shorten) provides an easy way to aggregate and export logs from Gatekeeper. + +## How it Works + +GKLE works by listening to the Gatekeeper log directory and processing a complete log file every time a new one is generated. + +While processing the log file, GKLE agreggates the lcore separated data and exports it (currently it only supports InfluxDB). + +## How to Set Up + +### Config file + +A config file should be located at `/etc/gkle.yaml`. GKLE uses it to read the Gatekeeper log directory and get InfluxDB credentials. + +The config file uses the following format: + +``` +gk_log_dir: "" + +influxdb: + url: "" + user: "" + password: "" + database: "" + retention_policy: "" + log_level : 0 + hostname: "" +``` + +`gk_log_dir` option receives the directory where gatekeeper is logging data (usually `/var/log/gatekeeper/`). + +`influxdb` option receives: connection URL, username and password, the desired database and retention policy, the log level (0 to 3, as described here), and finally the hostname of the server where Gatekeeper is running. + +### Running + +GKLE should be compiled and executed from systemd or another init system, so it can run on background listening to the files being created on the log directory. diff --git a/config.go b/config.go new file mode 100644 index 0000000..90e9153 --- /dev/null +++ b/config.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "os" + + "github.com/hostnetbr/gatekeeper-log-exporter/exporter/influx" + yaml "gopkg.in/yaml.v2" +) + +type Config struct { + GkLogDir string `yaml:"gk_log_dir"` + InfluxDB *influx.Config `yaml:"influxdb"` +} + +func parseConfig() (Config, error) { + data, err := os.ReadFile(confFile) + if err != nil { + return Config{}, fmt.Errorf("error reading config file: %w", err) + } + + var cfg Config + if err = yaml.Unmarshal(data, &cfg); err != nil { + return Config{}, fmt.Errorf("error parsing config: %w", err) + } + + if cfg.GkLogDir == "" { + return Config{}, fmt.Errorf("gk_log_dir empty: %w", err) + } + + if cfg.InfluxDB == nil { + return Config{}, fmt.Errorf("error parsing influxdb config") + } + if cfg.InfluxDB.User == "" || cfg.InfluxDB.Pass == "" || cfg.InfluxDB.Database == "" { + return Config{}, fmt.Errorf("not enough authentication credentials for influxdb") + } + if cfg.InfluxDB.Hostname == "" { + if cfg.InfluxDB.Hostname, err = os.Hostname(); err != nil { + return Config{}, fmt.Errorf("error parsing hostname: %w", err) + } + } + + return cfg, nil +} diff --git a/etc/gkle.yaml.example b/etc/gkle.yaml.example new file mode 100644 index 0000000..f1033c4 --- /dev/null +++ b/etc/gkle.yaml.example @@ -0,0 +1,10 @@ +gk_log_dir: "" + +influxdb: + url: "" + user: "" + password: "" + database: "" + retention_policy: "" + log_level: 0 + hostname: "" \ No newline at end of file diff --git a/exporter/exporter .go b/exporter/exporter .go new file mode 100644 index 0000000..b13d328 --- /dev/null +++ b/exporter/exporter .go @@ -0,0 +1,31 @@ +package exporter + +import ( + "time" +) + +type Measurements struct { + TotPktsNum uint64 + TotPktsSize uint64 + PktsNumGranted uint64 + PktsSizeGranted uint64 + PktsNumRequest uint64 + PktsSizeRequest uint64 + PktsNumDeclined uint64 + PktsSizeDeclined uint64 + TotPktsNumDropped uint64 + TotPktsSizeDropped uint64 + TotPktsNumDistributed uint64 + TotPktsSizeDistributed uint64 +} + +type Entry struct { + Time time.Time + Lcore int + Measurements Measurements +} + +type Interface interface { + Export(t time.Time, m *Measurements) error + Close() +} diff --git a/exporter/influx/influx.go b/exporter/influx/influx.go new file mode 100644 index 0000000..e9ce803 --- /dev/null +++ b/exporter/influx/influx.go @@ -0,0 +1,75 @@ +package influx + +import ( + "context" + "fmt" + "time" + + "github.com/hostnetbr/gatekeeper-log-exporter/exporter" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" +) + +type Config struct { + URL string `yaml:"url"` + User string `yaml:"user"` + Pass string `yaml:"password"` + Database string `yaml:"database"` + RetentionPolicy string `yaml:"retention_policy"` + LogLevel uint `yaml:"log_level"` + Hostname string `yaml:"hostname"` +} + +type Exporter struct { + client influxdb2.Client + config Config +} + +func NewExporter(config Config) Exporter { + options := influxdb2.DefaultOptions() + options.SetLogLevel(config.LogLevel) + client := influxdb2.NewClientWithOptions(config.URL, fmt.Sprintf("%s:%s", config.User, config.Pass), options) + return Exporter{client, config} +} + +func (e Exporter) Export(t time.Time, m *exporter.Measurements) error { + measurements := measurementsToMap(m) + p := influxdb2.NewPoint( + "gkle", + map[string]string{"host": e.config.Hostname}, + measurements, + t, + ) + + writeAPI := e.client.WriteAPIBlocking("", fmt.Sprintf("%s/%s", e.config.Database, e.config.RetentionPolicy)) + err := writeAPI.WritePoint(context.Background(), p) + if err != nil { + return fmt.Errorf("error writing to influxdb: %w\n", err) + } + + return nil +} + +func (e Exporter) Close() { + e.client.Close() +} + +func measurementsToMap(ms *exporter.Measurements) map[string]interface{} { + m := make(map[string]interface{}) + + // Parsing as int64 because InfluxDB seems to not support uint64. + // https://github.com/influxdata/influxdb/issues/9961 + m["tot_pkts_num"] = int64(ms.TotPktsNum) + m["tot_pkts_size"] = int64(ms.TotPktsSize) + m["pkts_num_granted"] = int64(ms.PktsNumGranted) + m["pkts_size_granted"] = int64(ms.PktsSizeGranted) + m["pkts_num_request"] = int64(ms.PktsNumRequest) + m["pkts_size_request"] = int64(ms.PktsSizeRequest) + m["pkts_num_declined"] = int64(ms.PktsNumDeclined) + m["pkts_size_declined"] = int64(ms.PktsSizeDeclined) + m["tot_pkts_num_dropped"] = int64(ms.TotPktsNumDropped) + m["tot_pkts_size_dropped"] = int64(ms.TotPktsSizeDropped) + m["tot_pkts_num_distributed"] = int64(ms.TotPktsNumDistributed) + m["tot_pkts_size_distributed"] = int64(ms.TotPktsSizeDistributed) + + return m +} diff --git a/exports/chronograf-example-dashboard.json b/exports/chronograf-example-dashboard.json new file mode 100644 index 0000000..2cd5474 --- /dev/null +++ b/exports/chronograf-example-dashboard.json @@ -0,0 +1,1453 @@ +{ + "meta": { + "chronografVersion": "1.9.1", + "sources": { + "1": { + "name": "Influx 1", + "link": "/chronograf/v1/sources/1" + } + } + }, + "dashboard": { + "id": "3", + "cells": [ + { + "i": "24e7411f-bc42-4e3a-81ae-50bfc0822858", + "x": 9, + "y": 8, + "w": 3, + "h": 4, + "name": "Total Packets Size Distributed", + "queries": [ + { + "query": "SELECT \"tot_pkts_size_distributed\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"tot_pkts_size_distributed\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/24e7411f-bc42-4e3a-81ae-50bfc0822858" + } + }, + { + "i": "97c93e38-b408-43a8-b137-5247451095ea", + "x": 6, + "y": 8, + "w": 3, + "h": 4, + "name": "Total Packets Number Distributed", + "queries": [ + { + "query": "SELECT \"tot_pkts_num_distributed\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"tot_pkts_num_distributed\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/97c93e38-b408-43a8-b137-5247451095ea" + } + }, + { + "i": "0dba23a4-aefa-47c0-8fff-8d9087ea44f1", + "x": 3, + "y": 8, + "w": 3, + "h": 4, + "name": "Total Packets Size Dropped", + "queries": [ + { + "query": "SELECT \"tot_pkts_size_dropped\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"tot_pkts_size_dropped\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/0dba23a4-aefa-47c0-8fff-8d9087ea44f1" + } + }, + { + "i": "ae3868d6-e314-4614-a83e-13408c6797eb", + "x": 6, + "y": 4, + "w": 3, + "h": 4, + "name": "Packets Number Declined", + "queries": [ + { + "query": "SELECT \"pkts_num_declined\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"pkts_num_declined\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/ae3868d6-e314-4614-a83e-13408c6797eb" + } + }, + { + "i": "c8b1806f-5352-4661-8cfb-9cb10c8eabc8", + "x": 9, + "y": 4, + "w": 3, + "h": 4, + "name": "Packets Size Declined", + "queries": [ + { + "query": "SELECT \"pkts_size_declined\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"pkts_size_declined\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/c8b1806f-5352-4661-8cfb-9cb10c8eabc8" + } + }, + { + "i": "ff8e2e6b-2b4a-44da-9787-1867bc5dea5b", + "x": 0, + "y": 8, + "w": 3, + "h": 4, + "name": "Total Packets Number Dropped", + "queries": [ + { + "query": "SELECT \"tot_pkts_num_dropped\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"tot_pkts_num_dropped\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/ff8e2e6b-2b4a-44da-9787-1867bc5dea5b" + } + }, + { + "i": "1e31be35-7ebb-4325-bc6b-a26c8fc7e6d7", + "x": 3, + "y": 4, + "w": 3, + "h": 4, + "name": "Packets Size Request", + "queries": [ + { + "query": "SELECT \"pkts_size_request\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"pkts_size_request\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/1e31be35-7ebb-4325-bc6b-a26c8fc7e6d7" + } + }, + { + "i": "92679f5f-9c43-4abd-b38e-c0239f6a7225", + "x": 6, + "y": 0, + "w": 3, + "h": 4, + "name": "Packets Number Granted", + "queries": [ + { + "query": "SELECT \"pkts_num_granted\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"pkts_num_granted\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/92679f5f-9c43-4abd-b38e-c0239f6a7225" + } + }, + { + "i": "b40f8fca-03f8-494d-a36d-a1f4d7fdf7b5", + "x": 9, + "y": 0, + "w": 3, + "h": 4, + "name": "Packets Size Granted", + "queries": [ + { + "query": "SELECT \"pkts_size_granted\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"pkts_size_granted\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/b40f8fca-03f8-494d-a36d-a1f4d7fdf7b5" + } + }, + { + "i": "a3e0aa96-a25f-4d8f-b35e-71f8b308ff01", + "x": 0, + "y": 4, + "w": 3, + "h": 4, + "name": "Packets Number Request", + "queries": [ + { + "query": "SELECT \"pkts_num_request\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"pkts_num_request\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/a3e0aa96-a25f-4d8f-b35e-71f8b308ff01" + } + }, + { + "i": "879a0aa2-c101-45f9-a113-f8d07ef21dc4", + "x": 0, + "y": 0, + "w": 3, + "h": 4, + "name": "Total Packets Number", + "queries": [ + { + "query": "SELECT \"tot_pkts_num\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"tot_pkts_num\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "fb568d6c-39cd-43ac-84b8-2f3ca4d41535", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "4d402448-d741-4d47-842e-cec025a5388c", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "56dbd59a-3ae2-44f5-8fc0-20ca3f9036e8", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/879a0aa2-c101-45f9-a113-f8d07ef21dc4" + } + }, + { + "i": "7a7827b7-bb24-4fcd-85ae-681948197dc4", + "x": 3, + "y": 0, + "w": 3, + "h": 4, + "name": "Total Packets Size", + "queries": [ + { + "query": "SELECT \"tot_pkts_size\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "queryConfig": { + "database": "", + "measurement": "", + "retentionPolicy": "", + "fields": [], + "tags": {}, + "groupBy": { + "time": "", + "tags": [] + }, + "areTagsAccepted": false, + "rawText": "SELECT \"tot_pkts_size\" FROM \"telegraf\".\"monitor\".\"gkle\" WHERE host = :host: AND time > :dashboardTime: AND time < :upperDashboardTime:", + "range": null, + "shifts": null + }, + "source": "", + "type": "influxql" + } + ], + "axes": { + "x": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + }, + "y2": { + "bounds": [ + "", + "" + ], + "label": "", + "prefix": "", + "suffix": "", + "base": "10", + "scale": "linear" + } + }, + "type": "line", + "colors": [ + { + "id": "a9e8768c-cfee-4455-8017-81e8d0cb6f16", + "type": "scale", + "hex": "#31C0F6", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "08f1a298-0749-4c24-9aa4-97da2867510d", + "type": "scale", + "hex": "#A500A5", + "name": "Nineteen Eighty Four", + "value": "0" + }, + { + "id": "6b60f9a2-2193-488a-824e-6bccc1cbd20c", + "type": "scale", + "hex": "#FF7E27", + "name": "Nineteen Eighty Four", + "value": "0" + } + ], + "legend": {}, + "tableOptions": { + "verticalTimeAxis": true, + "sortBy": { + "internalName": "time", + "displayName": "", + "visible": true + }, + "wrapping": "truncate", + "fixFirstColumn": true + }, + "fieldOptions": [ + { + "internalName": "time", + "displayName": "", + "visible": true + } + ], + "timeFormat": "MM/DD/YYYY HH:mm:ss", + "decimalPlaces": { + "isEnforced": true, + "digits": 2 + }, + "note": "", + "noteVisibility": "default", + "links": { + "self": "/chronograf/v1/dashboards/3/cells/7a7827b7-bb24-4fcd-85ae-681948197dc4" + } + } + ], + "templates": [ + { + "tempVar": ":host:", + "values": [ + { + "value": "gtk1.f1.k8.com.br", + "type": "tagValue", + "selected": true + } + ], + "id": "68c6d7ec-ac1e-4110-ac32-d9ad25cd9c00", + "type": "tagValues", + "label": "", + "query": { + "influxql": "SHOW TAG VALUES ON :database: FROM :measurement: WITH KEY=:tagKey:", + "db": "telegraf", + "measurement": "gkle", + "tagKey": "host", + "fieldKey": "" + }, + "sourceID": "dynamic", + "links": { + "self": "/chronograf/v1/dashboards/3/templates/68c6d7ec-ac1e-4110-ac32-d9ad25cd9c00" + } + } + ], + "name": "Gatekeeper Logs", + "organization": "default", + "links": { + "self": "/chronograf/v1/dashboards/3", + "cells": "/chronograf/v1/dashboards/3/cells", + "templates": "/chronograf/v1/dashboards/3/templates" + } + } +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..373f170 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/hostnetbr/gatekeeper-log-exporter + +go 1.16 + +require ( + github.com/fsnotify/fsnotify v1.5.1 + github.com/influxdata/influxdb-client-go/v2 v2.8.2 + gopkg.in/yaml.v2 v2.3.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..04ec0a3 --- /dev/null +++ b/go.sum @@ -0,0 +1,88 @@ +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/influxdata/influxdb-client-go/v2 v2.8.2 h1:NuWmf/xPx/izeDS1lwJRSPT54VsSZE2+p+2/7g7GfJk= +github.com/influxdata/influxdb-client-go/v2 v2.8.2/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..9616cf5 --- /dev/null +++ b/main.go @@ -0,0 +1,289 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "time" + + "github.com/hostnetbr/gatekeeper-log-exporter/exporter" + "github.com/hostnetbr/gatekeeper-log-exporter/exporter/influx" + + "github.com/fsnotify/fsnotify" +) + +const ( + confFile = "/etc/gkle.yaml" + lastLogFile = "/var/lib/gkle/last" + timeLayout = "2006-01-02 15:04:05" +) + +var logLineRegex = regexp.MustCompile(`^GK\/(\d+):\s+Basic\s+measurements\s+at\s+(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\[tot_pkts_num\s+=\s+(\d+),\s+tot_pkts_size\s+=\s+(\d+),\s+pkts_num_granted\s+=\s+(\d+),\s+pkts_size_granted\s+=\s+(\d+),\s+pkts_num_request\s+=\s+(\d+),\s+pkts_size_request\s+=\s+(\d+),\s+pkts_num_declined\s+=\s+(\d+),\s+pkts_size_declined\s+=\s+(\d+),\s+tot_pkts_num_dropped\s+=\s+(\d+),\s+tot_pkts_size_dropped\s+=\s+(\d+),\s+tot_pkts_num_distributed\s+=\s+(\d+),\s+tot_pkts_size_distributed\s+=\s+(\d+)\]$`) +var logFileRegex = regexp.MustCompile(`gatekeeper_\d{4}_\d{2}_\d{2}_\d{2}_\d{2}.log`) + +func main() { + os.Exit(run()) +} + +func run() int { + cfg, err := parseConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "error reading config file: %v\n", err) + return 1 + } + + watcher, err := fsnotify.NewWatcher() + if err != nil { + fmt.Fprintf(os.Stderr, "error creating fsnotify watcher: %v\n", err) + return 1 + } + defer watcher.Close() + + done := make(chan bool) + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + if event.Op&fsnotify.Create == fsnotify.Create { + filesToParse, err := getFilesToParse(cfg.GkLogDir) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting log files to parse: %v", err) + return + } + + for _, file := range filesToParse { + ex := influx.NewExporter(*cfg.InfluxDB) + parseLogFile(file, ex) + ex.Close() + + if saveLastLog(file); err != nil { + fmt.Fprintf(os.Stderr, "error saving last read log: %v", err) + return + } + } + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + fmt.Fprintf(os.Stderr, "error while watching log dir: %v\n", err) + } + } + }() + + err = watcher.Add(cfg.GkLogDir) + if err != nil { + fmt.Fprintf(os.Stderr, "error adding watcher to log dir: %s. %v\n", cfg.GkLogDir, err) + return 1 + } + <-done + + return 0 +} + +func getFilesToParse(path string) ([]string, error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, fmt.Errorf("error reading gatekeeper log dir %s: %w", path, err) + } + + parseAll := false + lastParsedLog, err := os.ReadFile(lastLogFile) + if err != nil { + if os.IsNotExist(err) { + parseAll = true + } else { + return nil, fmt.Errorf("error reading last log file: %w", err) + } + } + + var files []string + var lastParsedLogPosition int + // We don't parse the last file of the directory because it's still being + // written by gatekeeper. + for _, entry := range entries { + if !entry.IsDir() && logFileRegex.MatchString(entry.Name()) { + fileName := filepath.Join(path, entry.Name()) + files = append(files, fileName) + if fileName == string(lastParsedLog) { + lastParsedLogPosition = len(files) - 1 + } + } + } + + if parseAll || lastParsedLogPosition == 0 { + return files[:len(files)-1], nil + } + + return files[lastParsedLogPosition+1 : len(files)-1], nil +} + +func parseLogFile(filename string, ex exporter.Interface) error { + logFile, err := os.Open(filename) + if err != nil { + return fmt.Errorf("error opening log file: %w", err) + } + + defer logFile.Close() + + fileScanner := bufio.NewScanner(logFile) + fileScanner.Split(bufio.ScanLines) + + err = match(fileScanner, ex.Export) + if err != nil { + return fmt.Errorf("error reading log file: %w", err) + } + + return nil +} + +var errNoMatch = errors.New("line does not match") + +func match(sc *bufio.Scanner, f func(time.Time, *exporter.Measurements) error) error { + entries := make(map[int]exporter.Entry) + for sc.Scan() { + line := sc.Text() + entry, err := parseEntry(line) + if err != nil { + // line doesn't match up with regex; ignornig + if err == errNoMatch { + continue + } + return fmt.Errorf("error parsing entry: %w", err) + } + if _, repeat := entries[entry.Lcore]; repeat { + // repeated lcore; proccessing previous minute and starting new one + time, aggr := aggregate(entries) + if err := f(time, &aggr); err != nil { + return fmt.Errorf("error exporting data: %w", err) + } + entries = make(map[int]exporter.Entry) + } + + entries[entry.Lcore] = entry + } + if sc.Err() == nil { + // EOF + time, aggr := aggregate(entries) + if err := f(time, &aggr); err != nil { + return fmt.Errorf("error exporting data: %w", err) + } + } + return nil +} + +func parseEntry(line string) (exporter.Entry, error) { + matches := logLineRegex.FindStringSubmatch(line) + if matches == nil { + return exporter.Entry{}, errNoMatch + } + + logTime, err := time.Parse(timeLayout, matches[2]) + if err != nil { + return exporter.Entry{}, fmt.Errorf("error parsing log time: %w", err) + } + + lcore, err := strconv.Atoi(matches[1]) + if err != nil { + return exporter.Entry{}, fmt.Errorf("error parsing lcore: %w", err) + } + + measurements := exporter.Measurements{ + TotPktsNum: mustParseUint(matches[3]), + TotPktsSize: mustParseUint(matches[4]), + PktsNumGranted: mustParseUint(matches[5]), + PktsSizeGranted: mustParseUint(matches[6]), + PktsNumRequest: mustParseUint(matches[7]), + PktsSizeRequest: mustParseUint(matches[8]), + PktsNumDeclined: mustParseUint(matches[9]), + PktsSizeDeclined: mustParseUint(matches[10]), + TotPktsNumDropped: mustParseUint(matches[11]), + TotPktsSizeDropped: mustParseUint(matches[12]), + TotPktsNumDistributed: mustParseUint(matches[13]), + TotPktsSizeDistributed: mustParseUint(matches[14]), + } + + entry := exporter.Entry{ + Time: logTime, + Lcore: lcore, + Measurements: measurements, + } + return entry, nil +} + +func mustParseUint(s string) uint64 { + u, err := strconv.ParseUint(s, 10, 64) + if err != nil { + panic(err) + } + return u +} + +func aggregate(entries map[int]exporter.Entry) (time.Time, exporter.Measurements) { + var time time.Time + var aggr exporter.Measurements + + for _, e := range entries { + time = e.Time + m := e.Measurements + + aggr.TotPktsNum += m.TotPktsNum + aggr.TotPktsSize += m.TotPktsSize + aggr.PktsNumGranted += m.PktsNumGranted + aggr.PktsSizeGranted += m.PktsSizeGranted + aggr.PktsNumRequest += m.PktsNumRequest + aggr.PktsSizeRequest += m.PktsSizeRequest + aggr.PktsNumDeclined += m.PktsNumDeclined + aggr.PktsSizeDeclined += m.PktsSizeDeclined + aggr.TotPktsNumDropped += m.TotPktsNumDropped + aggr.TotPktsSizeDropped += m.TotPktsSizeDropped + aggr.TotPktsNumDistributed += m.TotPktsNumDistributed + aggr.TotPktsSizeDistributed += m.TotPktsSizeDistributed + } + return time, aggr +} + +func saveLastLog(logFile string) error { + dir, err := os.Open(filepath.Dir(lastLogFile)) + if err != nil { + return fmt.Errorf("open directory failed: %w", err) + } + defer dir.Close() + + tmp := fmt.Sprintf("%s.tmp", lastLogFile) + if err := safeWrite(tmp, logFile); err != nil { + return fmt.Errorf("safe write failed: %w", err) + } + if err := os.Rename(tmp, logFile); err != nil { + return fmt.Errorf("rename failed: %w", err) + } + if err := dir.Sync(); err != nil { + return fmt.Errorf("sync directory failed: %w", err) + } + + return nil +} + +func safeWrite(path string, data string) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("create failed: %w", err) + } + defer file.Close() + + if _, err := file.WriteString(data); err != nil { + return fmt.Errorf("write failed: %w", err) + } + if err := file.Sync(); err != nil { + return fmt.Errorf("sync file failed: %w", err) + } + + return nil +}