diff --git a/go.mod b/go.mod index d74a51e018..8b99a9c6c7 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,6 @@ require ( go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 - go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 go.opentelemetry.io/collector v0.45.0 go.opentelemetry.io/otel v1.4.1 go.opentelemetry.io/otel/bridge/opentracing v1.4.1 @@ -111,7 +110,6 @@ require ( github.com/dennwc/varint v1.0.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect @@ -121,12 +119,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/google/btree v1.0.1 // indirect github.com/gorilla/handlers v1.5.1 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/hcl v1.0.1-0.20190611123218-cf7d376da96d // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -160,18 +153,12 @@ require ( github.com/rs/cors v1.8.2 // indirect github.com/shirou/gopsutil v3.21.6+incompatible // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/tinylib/msgp v1.1.0 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/twmb/murmur3 v1.1.6 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/client/v2 v2.306.0-alpha.0 // indirect - go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 // indirect - go.etcd.io/etcd/raft/v3 v3.6.0-alpha.0 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/collector/model v0.45.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0 // indirect @@ -182,7 +169,6 @@ require ( go.opentelemetry.io/otel/trace v1.4.1 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.uber.org/multierr v1.7.0 // indirect - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/text v0.4.0 // indirect @@ -191,8 +177,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect ) // NB(nate): upgrading to the latest msgpack is not backwards compatibile as msgpack will no longer attempt to automatically diff --git a/go.sum b/go.sum index 03e1b6a240..ff43f2c484 100644 --- a/go.sum +++ b/go.sum @@ -104,7 +104,6 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/gcnotifier v0.0.0-20170518020117-39b0596a2da3/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= @@ -258,7 +257,6 @@ github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuD github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= @@ -286,9 +284,6 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 h1:KwaoQzs/WeUxxJqiJsZ4euOly1Az/IgZXXSxlD/UBNk= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= @@ -446,7 +441,6 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -495,7 +489,6 @@ github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -690,9 +683,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -821,14 +813,10 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -1437,8 +1425,6 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/snowflakedb/gosnowflake v1.3.13/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -1495,8 +1481,6 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.4/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= @@ -1553,7 +1537,6 @@ github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHM github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= @@ -1583,16 +1566,9 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3 go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 h1:2UyRzFWbZZzgu/xzxoRukgixvafiJtGyxO+3IKUyJ6c= go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0/go.mod h1:Vl/FkH40bHqmBFwhr8WVKtV47neyts36zl1voccRq8s= -go.etcd.io/etcd/client/v2 v2.305.0-alpha.0.0.20211029212747-6656181d312a h1:QPs92HLtaB/YLNnHLxnmkn0H6ZqRSfx5vvyPU/Mk5As= go.etcd.io/etcd/client/v2 v2.305.0-alpha.0.0.20211029212747-6656181d312a/go.mod h1:FJu6BdnY7u/JiFObq/nS0UPx+p2L3p1tw2kMj/UZGg0= go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 h1:hHaJ8CvTPJ9iv7xPz3G0gxt3csEqJW8evgty/kYICwo= go.etcd.io/etcd/client/v3 v3.6.0-alpha.0/go.mod h1:a9JuChoQBDnw7WclHYBYCtTOIC12Wwj+Fw0LX4TI/Gs= -go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 h1:cV/VsaYde/tcc2G9aHN5DQwx6CtUsWSEW4UqYzXuyyk= -go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0/go.mod h1:tXqWms0MpOJAS6L0B9nhFqZr0C/WEYzj/OtN90G8xzo= -go.etcd.io/etcd/raft/v3 v3.6.0-alpha.0 h1:BQ6CnNP4pIpy5rusFlTBxAacDgPXhuiHFwoTsBNsVpI= -go.etcd.io/etcd/raft/v3 v3.6.0-alpha.0/go.mod h1:/kZdrBXlc5fUgYXfIEQ0B5sb7ejXPKbtF4jWzF1exiQ= -go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 h1:BQUVqBqNFZZyrRbfydrRLzq9hYvCcRj97SsX1YwD7CA= -go.etcd.io/etcd/server/v3 v3.6.0-alpha.0/go.mod h1:3QM2rLq3B3hSXmVEvgVt3vEEbG/AumSs0Is7EgrlKzU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -2184,7 +2160,6 @@ google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200420144010-e5e8543f8aeb/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -2273,7 +2248,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml index c93b41ee25..9ad000907d 100644 --- a/scripts/docker-integration-tests/aggregator/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml @@ -1,17 +1,28 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +37,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +51,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +61,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml index c2f26330bd..1aa1c7c091 100644 --- a/scripts/docker-integration-tests/aggregator/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator/m3aggregator.yml @@ -38,7 +38,7 @@ kvClient: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index dfc757b8c4..a1e44edda4 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -24,7 +24,7 @@ clusters: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh index a65b912c8a..a5b44122c8 100755 --- a/scripts/docker-integration-tests/aggregator/test.sh +++ b/scripts/docker-integration-tests/aggregator/test.sh @@ -14,6 +14,8 @@ echo "Pull containers required for test" docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml index c93b41ee25..76eaac7862 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml @@ -1,17 +1,24 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +33,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +47,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +57,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml index 569ea9a9c2..e10c7cbe96 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml @@ -57,7 +57,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 281662faca..d15fd1a78c 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -22,7 +22,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh index e101eae186..7f5050005b 100755 --- a/scripts/docker-integration-tests/aggregator_legacy/test.sh +++ b/scripts/docker-integration-tests/aggregator_legacy/test.sh @@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD) COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml export REVISION +echo "Run etcd" +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/carbon/docker-compose.yml +++ b/scripts/docker-integration-tests/carbon/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index 7f33d8701c..b069fbf7b6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 carbon: findResultsIncludeBothExpandableAndLeaf: true diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh index f1c499a71e..6c811cb3b8 100755 --- a/scripts/docker-integration-tests/carbon/test.sh +++ b/scripts/docker-integration-tests/carbon/test.sh @@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang METRIC_EMIT_PID="-1" @@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s) end=$(date +%s) -start=$(($end-20)) +start=$(($end-20)) # 1. no max datapoints set, should not adjust number of datapoints coming back ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end" # 2. max datapoints with LTTB, should be an existing value (i.e. 42) diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh index 999ef1b20d..235c870e2b 100755 --- a/scripts/docker-integration-tests/cold_writes_simple/test.sh +++ b/scripts/docker-integration-tests/cold_writes_simple/test.sh @@ -8,9 +8,8 @@ SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/cold_writes_simple COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION -echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +echo "Run etcd, m3dbnode and m3coordinator containers" +docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 952c9a7ddf..3ca941aa8d 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh index a1590983c8..48dce08870 100755 --- a/scripts/docker-integration-tests/coordinator_config_rules/test.sh +++ b/scripts/docker-integration-tests/coordinator_config_rules/test.sh @@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml index ee8207bd26..edb2a7528c 100644 --- a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend coordinator01: expose: - "7201" @@ -10,33 +21,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_noop/test.sh b/scripts/docker-integration-tests/coordinator_noop/test.sh index 738ea863d0..0f91ec91da 100755 --- a/scripts/docker-integration-tests/coordinator_noop/test.sh +++ b/scripts/docker-integration-tests/coordinator_noop/test.sh @@ -9,8 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run coordinator with no etcd" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes function defer { docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml index e155eb5334..c39ef36c0c 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" @@ -14,33 +25,7 @@ services: - M3DB_HOST_ID=dbnode01 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/docker-compose-etcd.yml b/scripts/docker-integration-tests/docker-compose-etcd.yml new file mode 100644 index 0000000000..90af19b550 --- /dev/null +++ b/scripts/docker-integration-tests/docker-compose-etcd.yml @@ -0,0 +1,15 @@ +version: "3.5" +services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend + +networks: + backend: diff --git a/scripts/docker-integration-tests/m3coordinator.Dockerfile b/scripts/docker-integration-tests/m3coordinator.Dockerfile index 0319613977..489030dbea 100644 --- a/scripts/docker-integration-tests/m3coordinator.Dockerfile +++ b/scripts/docker-integration-tests/m3coordinator.Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3coordinator ADD ./m3coordinator /bin/ -ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml +ADD ./m3coordinator-local-docker-etcd.yml /etc/m3coordinator/m3coordinator.yml EXPOSE 7201/tcp 7203/tcp diff --git a/scripts/docker-integration-tests/m3dbnode.Dockerfile b/scripts/docker-integration-tests/m3dbnode.Dockerfile index a352ad4bf5..6d430fe803 100644 --- a/scripts/docker-integration-tests/m3dbnode.Dockerfile +++ b/scripts/docker-integration-tests/m3dbnode.Dockerfile @@ -4,9 +4,9 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3dbnode ADD ./m3dbnode /bin/ -ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml +ADD ./m3dbnode-local-docker-etcd.yml /etc/m3dbnode/m3dbnode.yml -EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp +EXPOSE 7201/tcp 7203/tcp 9000-9004/tcp ENV PANIC_ON_INVARIANT_VIOLATED=true diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml index 6c38ae39de..f89e401db3 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend m3aggregator01: expose: - "6001" @@ -12,6 +23,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: expose: - "6002" @@ -24,6 +37,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -34,6 +49,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd coordinatoradmin: expose: - "7201" @@ -44,6 +61,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd prometheusraw: expose: - "9090" @@ -60,6 +79,8 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" + depends_on: + - etcd prometheusagg: expose: - "9091" @@ -76,33 +97,7 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml index 1d77b0a035..37ac3f2aa4 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml @@ -40,7 +40,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml index d6c54c8430..935bc8ed5e 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml @@ -36,7 +36,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh index 208bb88565..7ba4c0435e 100755 --- a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh +++ b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh @@ -20,7 +20,7 @@ docker pull $PROMREMOTECLI_IMAGE trap 'cleanup ${COMPOSE_FILE} ${TEST_SUCCESS}' EXIT echo "Run ETCD" -docker-compose -f "${COMPOSE_FILE}" up -d etcd01 +docker-compose -f "${COMPOSE_FILE}" up -d etcd echo "Run Coordinator in Admin mode" docker-compose -f "${COMPOSE_FILE}" up -d coordinatoradmin diff --git a/scripts/docker-integration-tests/prometheus/docker-compose.yml b/scripts/docker-integration-tests/prometheus/docker-compose.yml index 03b839832b..2bfe100b1e 100644 --- a/scripts/docker-integration-tests/prometheus/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus/docker-compose.yml @@ -1,17 +1,28 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,6 +35,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd prometheus01: expose: - "9090" @@ -34,5 +47,7 @@ services: image: prom/prometheus:latest volumes: - "./prometheus.yml:/etc/prometheus/prometheus.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 79f849a151..7ebe2792b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -13,7 +13,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 query: restrictTags: diff --git a/scripts/docker-integration-tests/repair/docker-compose.yml b/scripts/docker-integration-tests/repair/docker-compose.yml index b91284adcf..bef03881d9 100644 --- a/scripts/docker-integration-tests/repair/docker-compose.yml +++ b/scripts/docker-integration-tests/repair/docker-compose.yml @@ -1,9 +1,19 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9012:9002" - "0.0.0.0:9013:9003" @@ -14,10 +24,11 @@ services: - M3DB_HOST_ID=m3db_local_1 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd dbnode02: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9022:9002" - "0.0.0.0:9023:9003" @@ -28,6 +39,8 @@ services: - M3DB_HOST_ID=m3db_local_2 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -42,5 +55,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 29dcc22d4b..62f3e88cf8 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -13,12 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 - seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 - + - etcd:2379 # Enable repairs. repair: enabled: true diff --git a/scripts/docker-integration-tests/repair/test.sh b/scripts/docker-integration-tests/repair/test.sh index ee32d632a9..4cb6e3b4f4 100755 --- a/scripts/docker-integration-tests/repair/test.sh +++ b/scripts/docker-integration-tests/repair/test.sh @@ -9,9 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode02 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh index bf1cb9bf8f..fa7f2901e3 100755 --- a/scripts/docker-integration-tests/run.sh +++ b/scripts/docker-integration-tests/run.sh @@ -3,22 +3,55 @@ set -ex TESTS=( - scripts/docker-integration-tests/cold_writes_simple/test.sh - scripts/docker-integration-tests/prometheus_replication/test.sh - scripts/docker-integration-tests/carbon/test.sh - scripts/docker-integration-tests/aggregator/test.sh - scripts/docker-integration-tests/aggregator_legacy/test.sh - scripts/docker-integration-tests/query_fanout/test.sh - scripts/docker-integration-tests/repair/test.sh - scripts/docker-integration-tests/replication/test.sh - scripts/docker-integration-tests/multi_cluster_write/test.sh - scripts/docker-integration-tests/coordinator_config_rules/test.sh - scripts/docker-integration-tests/coordinator_noop/test.sh - scripts/docker-integration-tests/prom_remote_write_backend/test.sh + scripts/docker-integration-tests/cold_writes_simple/test.sh + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/prometheus_replication/test.sh + + scripts/docker-integration-tests/carbon/test.sh + scripts/docker-integration-tests/aggregator/test.sh + scripts/docker-integration-tests/aggregator_legacy/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/query_fanout/test.sh + + scripts/docker-integration-tests/repair/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/replication/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/multi_cluster_write/test.sh + + scripts/docker-integration-tests/coordinator_config_rules/test.sh + scripts/docker-integration-tests/coordinator_noop/test.sh + scripts/docker-integration-tests/prom_remote_write_backend/test.sh ) +# Some systems, including our default Buildkite hosts, don't come with netcat +# installed and we may not have perms to install it. "Install" it in the worst +# possible way. +if ! command -v nc && [[ "$BUILDKITE" == "true" ]]; then + echo "installing netcat" + NCDIR="$(mktemp -d)" + + yumdownloader -y --destdir "$NCDIR" --resolve nc + ( + cd "$NCDIR" + RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1) + rpm2cpio "$RPM" | cpio -id + ) + + export PATH="$PATH:$NCDIR/usr/bin" + + function cleanup_nc() { + rm -rf "$NCDIR" + } + + trap cleanup_nc EXIT +fi + if [[ -z "$SKIP_SETUP" ]] || [[ "$SKIP_SETUP" == "false" ]]; then - scripts/docker-integration-tests/setup.sh + scripts/docker-integration-tests/setup.sh fi NUM_TESTS=${#TESTS[@]} @@ -27,16 +60,16 @@ MAX_IDX=$(((NUM_TESTS*(BUILDKITE_PARALLEL_JOB+1)/BUILDKITE_PARALLEL_JOB_COUNT)-1 ITER=0 for test in "${TESTS[@]}"; do - if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then - # Ensure all docker containers have been stopped so we don't run into issues - # trying to bind ports. - docker rm -f $(docker ps -aq) 2>/dev/null || true - echo "----------------------------------------------" - echo "running $test" - if ! (export M3_PATH=$(pwd) && $test); then - echo "--- :bk-status-failed: $test FAILED" - exit 1 - fi - fi - ITER="$((ITER+1))" + if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then + # Ensure all docker containers have been stopped so we don't run into issues + # trying to bind ports. + docker rm -f $(docker ps -aq) 2>/dev/null || true + echo "----------------------------------------------" + echo "running $test" + if ! (export M3_PATH=$(pwd) && $test); then + echo "--- :bk-status-failed: $test FAILED" + exit 1 + fi + fi + ITER="$((ITER+1))" done diff --git a/scripts/docker-integration-tests/setup.sh b/scripts/docker-integration-tests/setup.sh index 76659a49a2..d90ecb4967 100755 --- a/scripts/docker-integration-tests/setup.sh +++ b/scripts/docker-integration-tests/setup.sh @@ -15,8 +15,8 @@ mkdir -p ./bin # by keeping all the required files in ./bin, it makes the build context # for docker much smaller -cp ./src/query/config/m3coordinator-local-etcd.yml ./bin -cp ./src/dbnode/config/m3dbnode-local-etcd.yml ./bin +cp ./src/query/config/m3coordinator-local-docker-etcd.yml ./bin +cp ./src/dbnode/config/m3dbnode-local-docker-etcd.yml ./bin cp ./src/aggregator/config/m3aggregator.yml ./bin # build images @@ -26,7 +26,9 @@ function build_image { local svc=$1 echo "creating image for $svc" make ${svc}-linux-amd64 - docker build -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin + docker build \ + --no-cache \ + -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin } if [[ "$SERVICE" != "" ]]; then diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml index bce43d559e..7ed9f8f5ef 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,5 +33,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 7d5cb12598..945b26e3b5 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -13,5 +13,5 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 useV2BatchAPIs: true diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index 6825fd22ce..28174b7d59 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -25,7 +25,6 @@ import ( "errors" "fmt" "net/url" - "path" "strings" "time" @@ -42,9 +41,7 @@ import ( "github.com/m3db/m3/src/x/opentracing" "github.com/m3dbx/vellum/regexp" - "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" - "go.etcd.io/etcd/server/v3/embed" "gopkg.in/yaml.v2" ) @@ -681,80 +678,6 @@ func (c *ProtoConfiguration) Validate() error { return nil } -// NewEtcdEmbedConfig creates a new embedded etcd config from kv config. -func NewEtcdEmbedConfig(cfg DBConfiguration) (*embed.Config, error) { - newKVCfg := embed.NewConfig() - - hostID, err := cfg.HostIDOrDefault().Resolve() - if err != nil { - return nil, fmt.Errorf("failed resolving hostID %w", err) - } - - discoveryCfg := cfg.DiscoveryOrDefault() - envCfg, err := discoveryCfg.EnvironmentConfig(hostID) - if err != nil { - return nil, fmt.Errorf("failed getting env config from discovery config %w", err) - } - - kvCfg := envCfg.SeedNodes - newKVCfg.Name = hostID - - dir := kvCfg.RootDir - if dir == "" { - dir = path.Join(cfg.Filesystem.FilePathPrefixOrDefault(), defaultEtcdDirSuffix) - } - newKVCfg.Dir = dir - - LPUrls, err := convertToURLsWithDefault(kvCfg.ListenPeerUrls, newURL(defaultEtcdListenHost, DefaultEtcdServerPort)) - if err != nil { - return nil, err - } - newKVCfg.LPUrls = LPUrls - - LCUrls, err := convertToURLsWithDefault(kvCfg.ListenClientUrls, newURL(defaultEtcdListenHost, DefaultEtcdClientPort)) - if err != nil { - return nil, err - } - newKVCfg.LCUrls = LCUrls - - host, endpoint, err := getHostAndEndpointFromID(kvCfg.InitialCluster, hostID) - if err != nil { - return nil, err - } - - if host.ClusterState != "" { - newKVCfg.ClusterState = host.ClusterState - } - - APUrls, err := convertToURLsWithDefault(kvCfg.InitialAdvertisePeerUrls, newURL(endpoint, DefaultEtcdServerPort)) - if err != nil { - return nil, err - } - newKVCfg.APUrls = APUrls - - ACUrls, err := convertToURLsWithDefault(kvCfg.AdvertiseClientUrls, newURL(endpoint, DefaultEtcdClientPort)) - if err != nil { - return nil, err - } - newKVCfg.ACUrls = ACUrls - - newKVCfg.InitialCluster = initialClusterString(kvCfg.InitialCluster) - - copySecurityDetails := func(tls *transport.TLSInfo, ysc *environment.SeedNodeSecurityConfig) { - tls.TrustedCAFile = ysc.CAFile - tls.CertFile = ysc.CertFile - tls.KeyFile = ysc.KeyFile - tls.ClientCertAuth = ysc.CertAuth - tls.TrustedCAFile = ysc.TrustedCAFile - } - copySecurityDetails(&newKVCfg.ClientTLSInfo, &kvCfg.ClientTransportSecurity) - copySecurityDetails(&newKVCfg.PeerTLSInfo, &kvCfg.PeerTransportSecurity) - newKVCfg.ClientAutoTLS = kvCfg.ClientTransportSecurity.AutoTLS - newKVCfg.PeerAutoTLS = kvCfg.PeerTransportSecurity.AutoTLS - - return newKVCfg, nil -} - func newURL(host string, port int) string { return fmt.Sprintf("%s:%d", host, port) } diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index 45b15a41bd..d55d7dae1c 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -916,28 +916,6 @@ func TestGetHostAndEndpointFromID(t *testing.T) { } } -func TestNewEtcdEmbedConfig(t *testing.T) { - fd, err := ioutil.TempFile("", "config2.yaml") - require.NoError(t, err) - defer func() { - assert.NoError(t, fd.Close()) - assert.NoError(t, os.Remove(fd.Name())) - }() - - _, err = fd.Write([]byte(testBaseConfig)) - require.NoError(t, err) - - // Verify is valid - var cfg Configuration - err = xconfig.LoadFile(&cfg, fd.Name(), xconfig.Options{}) - require.NoError(t, err) - - embedCfg, err := NewEtcdEmbedConfig(*cfg.DB) - require.NoError(t, err) - - assert.Equal(t, "existing", embedCfg.ClusterState) -} - func TestNewJaegerTracer(t *testing.T) { fd, err := ioutil.TempFile("", "config_jaeger.yaml") require.NoError(t, err) diff --git a/src/cmd/services/m3dbnode/main/main_test.go b/src/cmd/services/m3dbnode/main/main_test.go index f46e6c39aa..25af8403e6 100644 --- a/src/cmd/services/m3dbnode/main/main_test.go +++ b/src/cmd/services/m3dbnode/main/main_test.go @@ -250,220 +250,6 @@ func TestConfig(t *testing.T) { serverWg.Wait() } -// TestEmbeddedConfig tests booting a server using an embedded KV. -func TestEmbeddedConfig(t *testing.T) { - // Create config file - tmpl, err := template.New("config").Parse(testConfig + embeddedKVConfigPortion) - require.NoError(t, err) - - configFd, cleanup := tempFile(t, "config.yaml") - defer cleanup() - - logFile, cleanupLogFile := tempFileTouch(t, "m3dbnode.log") - defer cleanupLogFile() - - configServiceCacheDir, cleanupConfigServiceCacheDir := tempDir(t, "kv") - defer cleanupConfigServiceCacheDir() - - embeddedKVDir, cleanupEmbeddedKVDir := tempDir(t, "embedded") - defer cleanupEmbeddedKVDir() - - dataDir, cleanupDataDir := tempDir(t, "data") - defer cleanupDataDir() - - servicePort := nextServicePort() - err = tmpl.Execute(configFd, struct { - HostID string - LogFile string - DataDir string - ServicePort string - ServiceName string - ServiceEnv string - ServiceZone string - ConfigServiceCacheDir string - EmbeddedKVDir string - LPURL string - LCURL string - APURL string - ACURL string - EtcdEndpoint string - InitialClusterHostID string - InitialClusterEndpoint string - }{ - HostID: hostID, - LogFile: logFile, - DataDir: dataDir, - ServicePort: strconv.Itoa(int(servicePort)), - ServiceName: serviceName, - ServiceEnv: serviceEnv, - ServiceZone: serviceZone, - ConfigServiceCacheDir: configServiceCacheDir, - EmbeddedKVDir: embeddedKVDir, - LPURL: lpURL, - LCURL: lcURL, - APURL: apURL, - ACURL: acURL, - EtcdEndpoint: etcdEndpoint, - InitialClusterHostID: initialClusterHostID, - InitialClusterEndpoint: initialClusterEndpoint, - }) - require.NoError(t, err) - - // Run server - var ( - interruptCh = make(chan error, 1) - bootstrapCh = make(chan struct{}, 1) - embeddedKVCh = make(chan struct{}, 1) - serverWg sync.WaitGroup - ) - serverWg.Add(1) - go func() { - server.Run(server.RunOptions{ - ConfigFile: configFd.Name(), - BootstrapCh: bootstrapCh, - EmbeddedKVCh: embeddedKVCh, - InterruptCh: interruptCh, - }) - serverWg.Done() - }() - defer func() { - // Resetting DefaultServeMux to prevent multiple assignments - // to /debug/dump in Server.Run() - http.DefaultServeMux = http.NewServeMux() - }() - - // Wait for embedded KV to be up. - <-embeddedKVCh - - // Setup the placement - var cfg config.Configuration - err = xconfig.LoadFile(&cfg, configFd.Name(), xconfig.Options{}) - require.NoError(t, err) - - discoveryCfg := cfg.DB.DiscoveryOrDefault() - envCfg, err := discoveryCfg.EnvironmentConfig(hostID) - require.NoError(t, err) - - syncCluster, err := envCfg.Services.SyncCluster() - require.NoError(t, err) - configSvcClient, err := syncCluster.Service.NewClient(instrument.NewOptions(). - SetLogger(zap.NewNop())) - require.NoError(t, err) - - svcs, err := configSvcClient.Services(services.NewOverrideOptions()) - require.NoError(t, err) - - serviceID := services.NewServiceID(). - SetName(serviceName). - SetEnvironment(serviceEnv). - SetZone(serviceZone) - - metadata := services.NewMetadata(). - SetPort(servicePort). - SetLivenessInterval(time.Minute). - SetHeartbeatInterval(10 * time.Second) - - err = svcs.SetMetadata(serviceID, metadata) - require.NoError(t, err) - - placementOpts := placement.NewOptions(). - SetValidZone(serviceZone) - placementSvc, err := svcs.PlacementService(serviceID, placementOpts) - require.NoError(t, err) - - var ( - instance = placement.NewInstance(). - SetID(hostID). - SetEndpoint(endpoint("127.0.0.1", servicePort)). - SetPort(servicePort). - SetIsolationGroup("local"). - SetWeight(1). - SetZone(serviceZone) - instances = []placement.Instance{instance} - // Use a low number of shards to avoid having to tune F.D limits. - shards = 4 - replicas = 1 - ) - - _, err = placementSvc.BuildInitialPlacement(instances, shards, replicas) - require.NoError(t, err) - - // Setup the namespace - ns, err := newNamespaceProtoValue(namespaceID) - require.NoError(t, err) - - kvStore, err := configSvcClient.KV() - require.NoError(t, err) - - _, err = kvStore.Set(kvconfig.NamespacesKey, ns) - require.NoError(t, err) - - // Wait for bootstrap - <-bootstrapCh - - // Create client, read and write some data - // NB(r): Make sure client config points to the root config - // service since we're going to instantiate the client configuration - // just by itself. - cfg.DB.Client.EnvironmentConfig = &envCfg - - cli, err := cfg.DB.Client.NewClient(client.ConfigurationParameters{}) - require.NoError(t, err) - - adminCli := cli.(client.AdminClient) - adminSession, err := adminCli.DefaultAdminSession() - require.NoError(t, err) - defer adminSession.Close() - - // Propagation of shard state from Initializing --> Available post-bootstrap is eventually - // consistent, so we must wait. - waitUntilAllShardsAreAvailable(t, adminSession) - - // Cast to narrower-interface instead of grabbing DefaultSession to make sure - // we use the same topology.Map that we validated in waitUntilAllShardsAreAvailable. - session := adminSession.(client.Session) - - start := xtime.Now().Add(-time.Minute) - values := []struct { - value float64 - at xtime.UnixNano - unit xtime.Unit - }{ - {value: 1.0, at: start, unit: xtime.Second}, - {value: 2.0, at: start.Add(1 * time.Second), unit: xtime.Second}, - {value: 3.0, at: start.Add(2 * time.Second), unit: xtime.Second}, - } - - for _, v := range values { - err := session.Write(ident.StringID(namespaceID), ident.StringID("foo"), v.at, v.value, v.unit, nil) - require.NoError(t, err) - } - - // Account for first value inserted at xtime.Second precision - fetchStart := start.Truncate(time.Second) - - // Account for last value being inserted at xtime.Second and - // the "end" param to fetch being exclusive - fetchEnd := values[len(values)-1].at.Truncate(time.Second).Add(time.Nanosecond) - - iter, err := session.Fetch(ident.StringID(namespaceID), ident.StringID("foo"), fetchStart, fetchEnd) - require.NoError(t, err) - - for _, v := range values { - require.True(t, iter.Next()) - dp, unit, _ := iter.Current() - assert.Equal(t, v.value, dp.Value) - // Account for xtime.Second precision on values going in - expectAt := v.at.Truncate(time.Second) - assert.Equal(t, expectAt, dp.TimestampNanos) - assert.Equal(t, v.unit, unit) - } - - // Wait for server to stop - interruptCh <- fmt.Errorf("test complete") - serverWg.Wait() -} - var ( testConfig = ` db: @@ -635,31 +421,4 @@ db: endpoints: {{.EtcdEndpoints}} autoSyncInterval: -1 ` - - embeddedKVConfigPortion = ` - discovery: - config: - service: - env: {{.ServiceEnv}} - zone: {{.ServiceZone}} - service: {{.ServiceName}} - cacheDir: {{.ConfigServiceCacheDir}} - etcdClusters: - - zone: {{.ServiceZone}} - endpoints: - - {{.EtcdEndpoint}} - seedNodes: - rootDir: {{.EmbeddedKVDir}} - listenPeerUrls: - - {{.LPURL}} - listenClientUrls: - - {{.LCURL}} - initialAdvertisePeerUrls: - - {{.APURL}} - advertiseClientUrls: - - {{.ACURL}} - initialCluster: - - hostID: {{.InitialClusterHostID}} - endpoint: {{.InitialClusterEndpoint}} -` ) diff --git a/src/dbnode/config/m3dbnode-local-docker-etcd.yml b/src/dbnode/config/m3dbnode-local-docker-etcd.yml new file mode 100644 index 0000000000..83650bc894 --- /dev/null +++ b/src/dbnode/config/m3dbnode-local-docker-etcd.yml @@ -0,0 +1,11 @@ +coordinator: {} +"db": + discovery: + "config": + "service": + "etcdClusters": + - "endpoints": ["http://etcd:2379"] + "zone": "embedded" + "service": "m3db" + "zone": "embedded" + "env": "default_env" \ No newline at end of file diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 3a91c27bdd..b92b651b39 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -100,7 +100,6 @@ import ( "github.com/opentracing/opentracing-go" "github.com/uber-go/tally" "github.com/uber/tchannel-go" - "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" ) @@ -355,24 +354,7 @@ func Run(runOpts RunOptions) { if !config.IsSeedNode(seedNodes, hostID) { logger.Info("not a seed node, using cluster seed nodes") } else { - logger.Info("seed node, starting etcd server") - - etcdCfg, err := config.NewEtcdEmbedConfig(cfg) - if err != nil { - logger.Fatal("unable to create etcd config", zap.Error(err)) - } - - e, err := embed.StartEtcd(etcdCfg) - if err != nil { - logger.Fatal("could not start embedded etcd", zap.Error(err)) - } - - if runOpts.EmbeddedKVCh != nil { - // Notify on embedded KV bootstrap chan if specified - runOpts.EmbeddedKVCh <- struct{}{} - } - - defer e.Close() + logger.Warn("seed node with embedded etcd no longer supported; skipping etcd setup.") } } diff --git a/src/query/config/m3coordinator-local-docker-etcd.yml b/src/query/config/m3coordinator-local-docker-etcd.yml new file mode 100644 index 0000000000..c25dff706c --- /dev/null +++ b/src/query/config/m3coordinator-local-docker-etcd.yml @@ -0,0 +1,16 @@ +clusters: + - namespaces: + - namespace: default + type: unaggregated + retention: 48h + client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd:2379