From 527ed8d0ef34d4724e95b25d6cf8f77e4b6ad770 Mon Sep 17 00:00:00 2001 From: jasonzhu <491537461q@gmail.com> Date: Mon, 25 Apr 2022 02:59:09 +0800 Subject: [PATCH 1/2] feat: add influx support(not implement test case) and has some todo item --- cmd/craned/app/manager.go | 9 ++ cmd/craned/app/options/options.go | 2 + go.mod | 3 + go.sum | 25 +++ pkg/metricquery/type.go | 27 +++- pkg/providers/config.go | 9 +- pkg/providers/influxdb/ctx.go | 49 ++++++ pkg/providers/influxdb/influxdb.go | 70 ++++++++ pkg/providers/influxdb/influxdb_client.go | 13 ++ pkg/providers/influxdb/influxdb_test.go | 1 + .../influxdb/builder.go | 152 ++++++++++++++++++ .../influxdb/builder_test.go | 1 + 12 files changed, 355 insertions(+), 6 deletions(-) create mode 100644 pkg/providers/influxdb/ctx.go create mode 100644 pkg/providers/influxdb/influxdb.go create mode 100644 pkg/providers/influxdb/influxdb_client.go create mode 100644 pkg/providers/influxdb/influxdb_test.go create mode 100644 pkg/querybuilder-providers/influxdb/builder.go create mode 100644 pkg/querybuilder-providers/influxdb/builder_test.go diff --git a/cmd/craned/app/manager.go b/cmd/craned/app/manager.go index f8ba3c739..b55613887 100644 --- a/cmd/craned/app/manager.go +++ b/cmd/craned/app/manager.go @@ -3,6 +3,7 @@ package app import ( "context" "flag" + "github.com/gocrane/crane/pkg/providers/influxdb" "os" "strings" @@ -176,6 +177,14 @@ func initializationDataSource(mgr ctrl.Manager, opts *options.Options) (map[prov klog.Exitf("unable to create datasource provider %v, err: %v", datasource, err) } hybridDataSources[providers.MockDataSource] = provider + case "influxdb": + provider, err := influxdb.NewProvider(&opts.DataSourceInfluxDBConfig) + if err != nil { + klog.Exitf("unable to create datasource provider %v, err: %v", datasource, err) + } + hybridDataSources[providers.InfluxDataSource] = provider + realtimeDataSources[providers.InfluxDataSource] = provider + historyDataSources[providers.InfluxDataSource] = provider case "prometheus", "prom": fallthrough default: diff --git a/cmd/craned/app/options/options.go b/cmd/craned/app/options/options.go index f55cb9afe..a55122c47 100644 --- a/cmd/craned/app/options/options.go +++ b/cmd/craned/app/options/options.go @@ -33,6 +33,8 @@ type Options struct { DataSourcePromConfig providers.PromConfig // DataSourceMockConfig is the mock data provider DataSourceMockConfig providers.MockConfig + // DataSourceInfluxDBConfig is the prometheus datasource config + DataSourceInfluxDBConfig providers.InfluxDBConfig // AlgorithmModelConfig AlgorithmModelConfig config.AlgorithmModelConfig diff --git a/go.mod b/go.mod index c3710bcf0..11c66e185 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-echarts/go-echarts/v2 v2.2.4 github.com/gocrane/api v0.3.0 github.com/google/cadvisor v0.39.2 + github.com/influxdata/influxdb-client-go/v2 v2.8.0 github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.26.0 @@ -61,6 +62,7 @@ require ( github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect @@ -92,6 +94,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/leodido/go-urn v1.2.0 // indirect diff --git a/go.sum b/go.sum index d5c3efd6f..90d342d8b 100644 --- a/go.sum +++ b/go.sum @@ -185,12 +185,15 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/csmarchbanks/sdk v0.0.0-20220120205302-870d00a83f4e h1:FjcnI9tmUibKj0UH9P761WwkEhAWUAinuuKVI2SyEB4= github.com/csmarchbanks/sdk v0.0.0-20220120205302-870d00a83f4e/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -244,6 +247,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.3.1 h1:doAsuITavI4IOcd0Y19U4B+O0dNWihRyX//nn4sEmgA= @@ -256,6 +260,7 @@ github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmC github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-echarts/go-echarts/v2 v2.2.4 h1:SKJpdyNIyD65XjbUZjzg6SwccTNXEgmh+PlaO23g2H0= github.com/go-echarts/go-echarts/v2 v2.2.4/go.mod h1:6TOomEztzGDVDkOSCFBq3ed7xOYfbOqhaBzD0YV771A= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -357,6 +362,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -456,6 +462,10 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb-client-go/v2 v2.8.0 h1:iaS3NrKUk6D0nkZZWjDm+fFWjrNKkix5YF2YrdVRJ8I= +github.com/influxdata/influxdb-client-go/v2 v2.8.0/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -499,6 +509,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -514,8 +526,13 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -726,6 +743,9 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -818,7 +838,9 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI= golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -904,6 +926,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -945,6 +968,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -994,6 +1018,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/metricquery/type.go b/pkg/metricquery/type.go index acbbae3c6..42422f970 100644 --- a/pkg/metricquery/type.go +++ b/pkg/metricquery/type.go @@ -13,16 +13,18 @@ type MetricSource string const ( PrometheusMetricSource MetricSource = "prom" MetricServerMetricSource MetricSource = "metricserver" + InfluxDBMetricSource MetricSource = "influxdb" ) type MetricType string const ( - WorkloadMetricType MetricType = "workload" - PodMetricType MetricType = "pod" - ContainerMetricType MetricType = "container" - NodeMetricType MetricType = "node" - PromQLMetricType MetricType = "promql" + WorkloadMetricType MetricType = "workload" + PodMetricType MetricType = "pod" + ContainerMetricType MetricType = "container" + NodeMetricType MetricType = "node" + PromQLMetricType MetricType = "promql" + InfluxDBQLMetricType MetricType = "influxdbql" ) var ( @@ -47,6 +49,8 @@ type Metric struct { Node *NodeNamerInfo // Prom can support any MetricName, user give the promQL Prom *PromNamerInfo + // InfluxDB can support any MetricName, user give the influxQL + InfluxDB *InfluxDBNamerInfo } type WorkloadNamerInfo struct { @@ -84,6 +88,13 @@ type PromNamerInfo struct { Selector labels.Selector } +type InfluxDBNamerInfo struct { + QueryExpr string + Namespace string + Selector labels.Selector +} + + func (m *Metric) ValidateMetric() error { if m == nil { return fmt.Errorf("metric is null") @@ -213,6 +224,7 @@ type Query struct { Type MetricSource MetricServer *MetricServerQuery Prometheus *PrometheusQuery + InfluxDB *InfluxDBQuery } // MetricServerQuery is used to do query for metric server @@ -224,3 +236,8 @@ type MetricServerQuery struct { type PrometheusQuery struct { Query string } + +// InfluxDBQuery is used to do query for prometheus +type InfluxDBQuery struct { + Query string +} diff --git a/pkg/providers/config.go b/pkg/providers/config.go index d003e820e..7ac2ed24d 100644 --- a/pkg/providers/config.go +++ b/pkg/providers/config.go @@ -45,11 +45,18 @@ func (auth *ClientAuth) Apply(req *http.Request) { type MockConfig struct { SeedFile string } - type DataSourceType string const ( MockDataSource DataSourceType = "mock" PrometheusDataSource DataSourceType = "prom" MetricServerDataSource DataSourceType = "metricserver" + InfluxDataSource DataSourceType = "influxdb" ) +// InfluxDBConfig represents the config of prometheus +type InfluxDBConfig struct { + Url string + Token string + QueryOrg string + Timeout time.Duration +} \ No newline at end of file diff --git a/pkg/providers/influxdb/ctx.go b/pkg/providers/influxdb/ctx.go new file mode 100644 index 000000000..1ed679d01 --- /dev/null +++ b/pkg/providers/influxdb/ctx.go @@ -0,0 +1,49 @@ +package influxdb + +import ( + gocontext "context" + "github.com/gocrane/crane/pkg/common" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" + "github.com/influxdata/influxdb-client-go/v2/api/query" + "k8s.io/klog/v2" + "time" +) + +const ( + InfluxDBClientOrg = "crane" +) + +type context struct { + api api.QueryAPI +} + +// NewContext creates a new InfluxDB querying context from the given client. +func NewContext(client influxdb2.Client) *context { + return &context{ + api: client.QueryAPI(InfluxDBClientOrg), + } +} + +// QueryRangeSync range query influxDB in sync way +func (c *context) QueryRangeSync(ctx gocontext.Context, query string, start, end time.Time, step time.Duration) ([]*common.TimeSeries, error) { + // TODO +} + +// QuerySync query influxDB in sync way +func (c *context) QuerySync(ctx gocontext.Context, query string) ([]*common.TimeSeries, error) { + // TODO + var ts []*common.TimeSeries + results, err := c.api.Query(ctx, ``) + if err != nil { + return ts, err + } + klog.V(8).InfoS("InfluxDB query result", "result", results.Record().String()) + return c.convertInfluxDBResultsToTimeSeriesMap(results) +} + +func (c *context) convertInfluxDBResultsToTimeSeriesMap(record query.FluxRecord) ([]*common.TimeSeries, error) { + // TODO + var results []*common.TimeSeries + record.Start() +} \ No newline at end of file diff --git a/pkg/providers/influxdb/influxdb.go b/pkg/providers/influxdb/influxdb.go new file mode 100644 index 000000000..e9f9eeb06 --- /dev/null +++ b/pkg/providers/influxdb/influxdb.go @@ -0,0 +1,70 @@ +package influxdb + +import ( + gocontext "context" + "github.com/gocrane/crane/pkg/common" + "github.com/gocrane/crane/pkg/metricnaming" + "github.com/gocrane/crane/pkg/metricquery" + "github.com/gocrane/crane/pkg/providers" + "k8s.io/klog/v2" + "time" +) + +type influxDB struct { + ctx *context + config *providers.InfluxDBConfig +} + +// NewProvider return a prometheus data provider +func NewProvider(config *providers.InfluxDBConfig) (providers.Interface, error) { + client, err := NewInfluxDBClient(config) + if err != nil { + return nil, err + } + + ctx := NewContext(client) + + return &influxDB{ctx: ctx, config: config}, nil +} + + +func (i *influxDB) QueryTimeSeries(namer metricnaming.MetricNamer, startTime time.Time, endTime time.Time, step time.Duration) ([]*common.TimeSeries, error) { + influxdbBuilder := namer.QueryBuilder().Builder(metricquery.PrometheusMetricSource) + influxdbQuery, err := influxdbBuilder.BuildQuery() + if err != nil { + klog.Errorf("Failed to BuildQuery: %v", err) + return nil, err + } + klog.V(6).Infof("QueryTimeSeries metricNamer %v, timeout: %v", namer.BuildUniqueKey(), i.config.Timeout) + timeoutCtx, cancelFunc := gocontext.WithTimeout(gocontext.Background(), i.config.Timeout) + defer cancelFunc() + timeSeries, err := i.ctx.QueryRangeSync(timeoutCtx, influxdbQuery.Prometheus.Query, startTime, endTime, step) + if err != nil { + klog.Errorf("Failed to QueryTimeSeries: %v, metricNamer: %v, query: %v", err, namer.BuildUniqueKey(), influxdbQuery.Prometheus.Query) + return nil, err + } + return timeSeries, nil +} + +func (i *influxDB) QueryLatestTimeSeries(namer metricnaming.MetricNamer) ([]*common.TimeSeries, error) { + influxdbBuilder := namer.QueryBuilder().Builder(metricquery.InfluxDBMetricSource) + influxdbQuery, err := influxdbBuilder.BuildQuery() + if err != nil { + klog.Errorf("Failed to QueryLatestTimeSeries metricNamer %v, err: %v", namer.BuildUniqueKey(), err) + return nil, err + } + klog.V(6).Infof("QueryLatestTimeSeries metricNamer %v", namer.BuildUniqueKey()) + // use range query for latest too. because the queryExpr is an range in crd spec + //end := time.Now() + // avoid no data latest. multiply 2 + //start := end.Add(-step * 2) + klog.V(6).Infof("QueryLatestTimeSeries metricNamer %v, timeout: %v", namer.BuildUniqueKey(), i.config.Timeout) + timeoutCtx, cancelFunc := gocontext.WithTimeout(gocontext.Background(), i.config.Timeout) + defer cancelFunc() + timeSeries, err := i.ctx.QuerySync(timeoutCtx, influxdbQuery.Prometheus.Query) + if err != nil { + klog.Errorf("Failed to QueryLatestTimeSeries: %v, metricNamer: %v, query: %v", err, namer.BuildUniqueKey(), influxdbQuery.Prometheus.Query) + return nil, err + } + return timeSeries, nil +} \ No newline at end of file diff --git a/pkg/providers/influxdb/influxdb_client.go b/pkg/providers/influxdb/influxdb_client.go new file mode 100644 index 000000000..a71596b93 --- /dev/null +++ b/pkg/providers/influxdb/influxdb_client.go @@ -0,0 +1,13 @@ +package influxdb + +import ( + "github.com/gocrane/crane/pkg/providers" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" +) + +// NewPrometheusClient returns a prometheus.Client +func NewInfluxDBClient(config *providers.InfluxDBConfig) (influxdb2.Client, error) { + client := influxdb2.NewClient(config.Url, config.Token) + + return client, nil +} \ No newline at end of file diff --git a/pkg/providers/influxdb/influxdb_test.go b/pkg/providers/influxdb/influxdb_test.go new file mode 100644 index 000000000..ab711e3b8 --- /dev/null +++ b/pkg/providers/influxdb/influxdb_test.go @@ -0,0 +1 @@ +package influxdb diff --git a/pkg/querybuilder-providers/influxdb/builder.go b/pkg/querybuilder-providers/influxdb/builder.go new file mode 100644 index 000000000..8ab137d30 --- /dev/null +++ b/pkg/querybuilder-providers/influxdb/builder.go @@ -0,0 +1,152 @@ +package influxdb + +import ( + "fmt" + "github.com/gocrane/crane/pkg/metricquery" + "github.com/gocrane/crane/pkg/querybuilder" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "strings" +) + +const ( + // WorkloadCpuUsageExprTemplate is used to query workload cpu usage by promql, param is namespace,workload-name,duration str + WorkloadCpuUsageExprTemplate = `sum(irate(container_cpu_usage_seconds_total{container!="",image!="",container!="POD",namespace="%s",pod=~"^%s-.*$"}[%s]))` + // WorkloadMemUsageExprTemplate is used to query workload mem usage by promql, param is namespace, workload-name + WorkloadMemUsageExprTemplate = `sum(container_memory_working_set_bytes{container!="",image!="",container!="POD",namespace="%s",pod=~"^%s-.*$"})` + + // following is node exporter metric for node cpu/memory usage + // NodeCpuUsageExprTemplate is used to query node cpu usage by promql, param is node name which prometheus scrape, duration str + NodeCpuUsageExprTemplate = `sum(count(node_cpu_seconds_total{mode="idle",instance=~"(%s)(:\\d+)?"}) by (mode, cpu)) - sum(irate(node_cpu_seconds_total{mode="idle",instance=~"(%s)(:\\d+)?"}[%s]))` + // NodeMemUsageExprTemplate is used to query node cpu memory by promql, param is node name, node name which prometheus scrape + NodeMemUsageExprTemplate = `sum(node_memory_MemTotal_bytes{instance=~"(%s)(:\\d+)?"} - node_memory_MemAvailable_bytes{instance=~"(%s)(:\\d+)?"})` + + // PodCpuUsageExprTemplate is used to query pod cpu usage by promql, param is namespace,pod, duration str + PodCpuUsageExprTemplate = `sum(irate(container_cpu_usage_seconds_total{container!="POD",namespace="%s",pod="%s"}[%s]))` + // PodMemUsageExprTemplate is used to query pod cpu usage by promql, param is namespace,pod + PodMemUsageExprTemplate = `sum(container_memory_working_set_bytes{container!="POD",namespace="%s",pod="%s"})` + + // ContainerCpuUsageExprTemplate is used to query container cpu usage by promql, param is namespace,pod,container duration str + ContainerCpuUsageExprTemplate = `irate(container_cpu_usage_seconds_total{container!="POD",namespace="%s",pod=~"^%s.*$",container="%s"}[%s])` + // ContainerMemUsageExprTemplate is used to query container cpu usage by promql, param is namespace,pod,container + ContainerMemUsageExprTemplate = `container_memory_working_set_bytes{container!="POD",namespace="%s",pod=~"^%s.*$",container="%s"}` +) + +var supportedResources = sets.NewString(v1.ResourceCPU.String(), v1.ResourceMemory.String()) + +var _ querybuilder.Builder = &builder{} + +type builder struct { + metric *metricquery.Metric +} + +func NewInfluxDBQueryBuilder(metric *metricquery.Metric) querybuilder.Builder { + return &builder{ + metric: metric, + } +} + +func (b builder) BuildQuery() (*metricquery.Query, error) { + switch b.metric.Type { + case metricquery.WorkloadMetricType: + return b.workloadQuery(b.metric) + case metricquery.PodMetricType: + return b.podQuery(b.metric) + case metricquery.ContainerMetricType: + return b.containerQuery(b.metric) + case metricquery.NodeMetricType: + return b.nodeQuery(b.metric) + case metricquery.InfluxDBQLMetricType: + return b.influxDBQuery(b.metric) + default: + return nil, fmt.Errorf("metric type %v not supported", b.metric.Type) + } +} + +func (b *builder) workloadQuery(metric *metricquery.Metric) (*metricquery.Query, error) { + if metric.Workload == nil { + return nil, fmt.Errorf("metric type %v, but no WorkloadNamerInfo provided", metric.Type) + } + switch strings.ToLower(metric.MetricName) { + case v1.ResourceCPU.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(WorkloadCpuUsageExprTemplate, metric.Workload.Namespace, metric.Workload.Name, "3m"), + }), nil + case v1.ResourceMemory.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(WorkloadMemUsageExprTemplate, metric.Workload.Namespace, metric.Workload.Name), + }), nil + default: + return nil, fmt.Errorf("metric type %v do not support resource metric %v. only support %v now", metric.Type, metric.MetricName, supportedResources.List()) + } +} + +func (b *builder) containerQuery(metric *metricquery.Metric) (*metricquery.Query, error) { + if metric.Container == nil { + return nil, fmt.Errorf("metric type %v, but no ContainerNamerInfo provided", metric.Type) + } + switch strings.ToLower(metric.MetricName) { + case v1.ResourceCPU.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(ContainerCpuUsageExprTemplate, metric.Container.Namespace, metric.Container.PodName, metric.Container.ContainerName, "3m"), + }), nil + case v1.ResourceMemory.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(ContainerMemUsageExprTemplate, metric.Container.Namespace, metric.Container.PodName, metric.Container.ContainerName), + }), nil + default: + return nil, fmt.Errorf("metric type %v do not support resource metric %v. only support %v now", metric.Type, metric.MetricName, supportedResources.List()) + } +} + +func (b *builder) podQuery(metric *metricquery.Metric) (*metricquery.Query, error) { + if metric.Pod == nil { + return nil, fmt.Errorf("metric type %v, but no PodNamerInfo provided", metric.Type) + } + switch strings.ToLower(metric.MetricName) { + case v1.ResourceCPU.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(PodCpuUsageExprTemplate, metric.Pod.Namespace, metric.Pod.Name, "3m"), + }), nil + case v1.ResourceMemory.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(PodMemUsageExprTemplate, metric.Pod.Namespace, metric.Pod.Name), + }), nil + default: + return nil, fmt.Errorf("metric type %v do not support resource metric %v. only support %v now", metric.Type, metric.MetricName, supportedResources.List()) + } +} + +func (b *builder) nodeQuery(metric *metricquery.Metric) (*metricquery.Query, error) { + if metric.Node == nil { + return nil, fmt.Errorf("metric type %v, but no NodeNamerInfo provided", metric.Type) + } + switch strings.ToLower(metric.MetricName) { + case v1.ResourceCPU.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(NodeCpuUsageExprTemplate, metric.Node.Name, metric.Node.Name, "3m"), + }), nil + case v1.ResourceMemory.String(): + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: fmt.Sprintf(NodeMemUsageExprTemplate, metric.Node.Name, metric.Node.Name), + }), nil + default: + return nil, fmt.Errorf("metric type %v do not support resource metric %v. only support %v now", metric.Type, metric.MetricName, supportedResources.List()) + } +} + +func (b *builder) influxDBQuery(metric *metricquery.Metric) (*metricquery.Query, error) { + if metric.Prom == nil { + return nil, fmt.Errorf("metric type %v, but no PromNamerInfo provided", metric.Type) + } + return influxDBQuery(&metricquery.InfluxDBQuery{ + Query: metric.Prom.QueryExpr, + }), nil +} + +func influxDBQuery(influxDB *metricquery.InfluxDBQuery) *metricquery.Query { + return &metricquery.Query{ + Type: metricquery.PrometheusMetricSource, + InfluxDB: influxDB, + } +} diff --git a/pkg/querybuilder-providers/influxdb/builder_test.go b/pkg/querybuilder-providers/influxdb/builder_test.go new file mode 100644 index 000000000..ab711e3b8 --- /dev/null +++ b/pkg/querybuilder-providers/influxdb/builder_test.go @@ -0,0 +1 @@ +package influxdb From 1e821002a2ad9c53b5a0f7ad8a5d56bde0f0251d Mon Sep 17 00:00:00 2001 From: jasonzhu <491537461q@gmail.com> Date: Mon, 25 Apr 2022 03:22:59 +0800 Subject: [PATCH 2/2] fix: add todo for influxdb query builder. --- pkg/querybuilder-providers/influxdb/builder.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/querybuilder-providers/influxdb/builder.go b/pkg/querybuilder-providers/influxdb/builder.go index 8ab137d30..b013db432 100644 --- a/pkg/querybuilder-providers/influxdb/builder.go +++ b/pkg/querybuilder-providers/influxdb/builder.go @@ -9,27 +9,28 @@ import ( "strings" ) +// TODO implement influxDB metrics semantic const ( // WorkloadCpuUsageExprTemplate is used to query workload cpu usage by promql, param is namespace,workload-name,duration str - WorkloadCpuUsageExprTemplate = `sum(irate(container_cpu_usage_seconds_total{container!="",image!="",container!="POD",namespace="%s",pod=~"^%s-.*$"}[%s]))` + WorkloadCpuUsageExprTemplate = `from(bucket:"my-bucket")|> range(start: -%sh) |> aggregate.rate() |> cumulativeSum()` // WorkloadMemUsageExprTemplate is used to query workload mem usage by promql, param is namespace, workload-name - WorkloadMemUsageExprTemplate = `sum(container_memory_working_set_bytes{container!="",image!="",container!="POD",namespace="%s",pod=~"^%s-.*$"})` + WorkloadMemUsageExprTemplate = `` // following is node exporter metric for node cpu/memory usage // NodeCpuUsageExprTemplate is used to query node cpu usage by promql, param is node name which prometheus scrape, duration str - NodeCpuUsageExprTemplate = `sum(count(node_cpu_seconds_total{mode="idle",instance=~"(%s)(:\\d+)?"}) by (mode, cpu)) - sum(irate(node_cpu_seconds_total{mode="idle",instance=~"(%s)(:\\d+)?"}[%s]))` + NodeCpuUsageExprTemplate = `` // NodeMemUsageExprTemplate is used to query node cpu memory by promql, param is node name, node name which prometheus scrape - NodeMemUsageExprTemplate = `sum(node_memory_MemTotal_bytes{instance=~"(%s)(:\\d+)?"} - node_memory_MemAvailable_bytes{instance=~"(%s)(:\\d+)?"})` + NodeMemUsageExprTemplate = `` // PodCpuUsageExprTemplate is used to query pod cpu usage by promql, param is namespace,pod, duration str - PodCpuUsageExprTemplate = `sum(irate(container_cpu_usage_seconds_total{container!="POD",namespace="%s",pod="%s"}[%s]))` + PodCpuUsageExprTemplate = `` // PodMemUsageExprTemplate is used to query pod cpu usage by promql, param is namespace,pod - PodMemUsageExprTemplate = `sum(container_memory_working_set_bytes{container!="POD",namespace="%s",pod="%s"})` + PodMemUsageExprTemplate = `` // ContainerCpuUsageExprTemplate is used to query container cpu usage by promql, param is namespace,pod,container duration str - ContainerCpuUsageExprTemplate = `irate(container_cpu_usage_seconds_total{container!="POD",namespace="%s",pod=~"^%s.*$",container="%s"}[%s])` + ContainerCpuUsageExprTemplate = `` // ContainerMemUsageExprTemplate is used to query container cpu usage by promql, param is namespace,pod,container - ContainerMemUsageExprTemplate = `container_memory_working_set_bytes{container!="POD",namespace="%s",pod=~"^%s.*$",container="%s"}` + ContainerMemUsageExprTemplate = `` ) var supportedResources = sets.NewString(v1.ResourceCPU.String(), v1.ResourceMemory.String())