From 0c487b433d41364a9adaa2d7fb00d722fd504429 Mon Sep 17 00:00:00 2001 From: Tarun Date: Tue, 20 Sep 2022 12:42:58 +0200 Subject: [PATCH 01/62] First version of the tool: tick every slot --- go.mod | 32 ++ go.sum | 554 +++++++++++++++++++++++++++++++++++ main.go | 44 +++ pkg/app/service.go | 88 ++++++ pkg/chain_stats/chaintime.go | 19 ++ pkg/client_api/clientapi.go | 44 +++ pkg/cmd/scorer_cmd.go | 51 ++++ 7 files changed, 832 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pkg/app/service.go create mode 100644 pkg/chain_stats/chaintime.go create mode 100644 pkg/client_api/clientapi.go create mode 100644 pkg/cmd/scorer_cmd.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b1fd6ad --- /dev/null +++ b/go.mod @@ -0,0 +1,32 @@ +module github.com/tdahar/block-scorer + +go 1.17 + +require ( + github.com/attestantio/go-eth2-client v0.13.6 + github.com/pkg/errors v0.9.1 + github.com/rs/zerolog v1.28.0 + github.com/sirupsen/logrus v1.9.0 + github.com/urfave/cli/v2 v2.16.3 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/fatih/color v1.13.0 // indirect + github.com/ferranbt/fastssz v0.1.1 // indirect + github.com/goccy/go-yaml v1.9.5 // indirect + github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect + github.com/r3labs/sse/v2 v2.7.4 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f9cbd98 --- /dev/null +++ b/go.sum @@ -0,0 +1,554 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/attestantio/go-eth2-client v0.13.6 h1:KivmTvEdW5Ah3wSsfb8u/8cZZh/BtaRwNJuPpLqhQdg= +github.com/attestantio/go-eth2-client v0.13.6/go.mod h1:bcg5gfjVcm+MtcaZfzv/uSWNHU4i8hGamVG+9JCZnC0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/ferranbt/fastssz v0.1.1 h1:hBYNxKu51wjPC9sQYCjicmy5wtJqubENp3IiRVcdJBM= +github.com/ferranbt/fastssz v0.1.1/go.mod h1:U2ZsxlYyvGeQGmadhz8PlEqwkBzDIhHwd3xuKrg2JIs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= +github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= +github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= +github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= +github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= +github.com/r3labs/sse/v2 v2.7.4/go.mod h1:hUrYMKfu9WquG9MyI0r6TKiNH+6Sw/QPKm2YbNbU5g8= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= +github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli/v2 v2.16.3 h1:gHoFIwpPjoyIMbJp/VFd+/vuD0dAgFK4B6DpEMFJfQk= +github.com/urfave/cli/v2 v2.16.3/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= +gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..6e1554f --- /dev/null +++ b/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/sirupsen/logrus" + "github.com/tdahar/block-scorer/pkg/cmd" + cli "github.com/urfave/cli/v2" +) + +var ( + Version = "v0.0.1" + CliName = "Eth2 Block Scorer" + log = logrus.WithField( + "cli", "CliName", + ) +) + +func main() { + fmt.Println(CliName, Version) + + app := &cli.App{ + Name: CliName, + Usage: "Tinny client that requests and processes the Beacon Block proposals for each client.", + UsageText: "block-scorer [commands] [arguments...]", + Authors: []*cli.Author{ + { + Name: "Tarun", + Email: "tarsuno@gmail.com", + }, + }, + EnableBashCompletion: true, + Commands: []*cli.Command{ + cmd.ScorerCommand, + }, + } + // generate the crawler + if err := app.RunContext(context.Background(), os.Args); err != nil { + log.Errorf("error: %v\n", err) + os.Exit(1) + } +} diff --git a/pkg/app/service.go b/pkg/app/service.go new file mode 100644 index 0000000..321def9 --- /dev/null +++ b/pkg/app/service.go @@ -0,0 +1,88 @@ +package app + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" + "github.com/tdahar/block-scorer/pkg/chain_stats" + "github.com/tdahar/block-scorer/pkg/client_api" +) + +var ( + modName = "App" + log = logrus.WithField( + "module", modName, + ) +) + +type AppService struct { + ctx context.Context + Clients []*client_api.APIClient + initTime time.Time + ChainTime chain_stats.ChainTime + HeadSlot phase0.Slot +} + +func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, error) { + + clients := make([]*client_api.APIClient, 0) + + for i := range bnEndpoints { + if !strings.Contains(bnEndpoints[i], "/") { + log.Errorf("incorrect format for endpoint: %s", bnEndpoints[i]) + } + label := strings.Split(bnEndpoints[i], "/")[0] + endpoint := strings.Split(bnEndpoints[i], "/")[1] + newClient, err := client_api.NewAPIClient(ctx, label, endpoint, time.Second*5) + if err != nil { + log.Errorf("could not create client for endpoint: %s ", endpoint, err) + continue + } + clients = append(clients, newClient) + } + + genesis, err := clients[0].Api.GenesisTime(ctx) + if err != nil { + return nil, fmt.Errorf("could not obtain genesis time: %s", err) + } + headHeader, err := clients[0].Api.BeaconBlockHeader(ctx, "head") + if err != nil { + return nil, fmt.Errorf("could not obtain head block header: %s", err) + } + return &AppService{ + ctx: ctx, + Clients: clients, + initTime: time.Now(), + HeadSlot: headHeader.Header.Message.Slot, + ChainTime: chain_stats.ChainTime{ + GenesisTime: genesis, + }, + }, nil +} + +func (s *AppService) Run() { + + ticker := time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) + + for { + + select { + case <-s.ctx.Done(): + + return + + case <-ticker: + + s.HeadSlot++ + log.Infof("Entered a new slot!: %d, time: %s", s.HeadSlot, time.Now()) + ticker = time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) + // a new slot has begun, therefore execute all needed actions + log.Infof("Next Duration: %d", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1)))) + default: + } + } +} diff --git a/pkg/chain_stats/chaintime.go b/pkg/chain_stats/chaintime.go new file mode 100644 index 0000000..dd4bd2a --- /dev/null +++ b/pkg/chain_stats/chaintime.go @@ -0,0 +1,19 @@ +package chain_stats + +import ( + "time" + + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +const ( + SLOT_DURATION = 12 +) + +type ChainTime struct { + GenesisTime time.Time +} + +func (c ChainTime) SlotTime(slot phase0.Slot) time.Time { + return c.GenesisTime.Add(time.Duration(slot) * SLOT_DURATION * time.Second) +} diff --git a/pkg/client_api/clientapi.go b/pkg/client_api/clientapi.go new file mode 100644 index 0000000..b9bb78d --- /dev/null +++ b/pkg/client_api/clientapi.go @@ -0,0 +1,44 @@ +package client_api + +import ( + "context" + "time" + + "github.com/attestantio/go-eth2-client/http" + "github.com/rs/zerolog" + "github.com/sirupsen/logrus" +) + +var ( + moduleName = "API-Cli" + log = logrus.WithField( + "module", "") +) + +type APIClient struct { + ctx context.Context + Api *http.Service + Label string +} + +func NewAPIClient(ctx context.Context, label string, cliEndpoint string, timeout time.Duration) (*APIClient, error) { + log.Debugf("generating http client at %s", cliEndpoint) + httpCli, err := http.New( + ctx, + http.WithAddress(cliEndpoint), + http.WithLogLevel(zerolog.WarnLevel), + http.WithTimeout(timeout), + ) + if err != nil { + return &APIClient{}, err + } + + hc, ok := httpCli.(*http.Service) + if !ok { + log.Error("gernerating the http api client") + } + return &APIClient{ + ctx: ctx, + Api: hc, + }, nil +} diff --git a/pkg/cmd/scorer_cmd.go b/pkg/cmd/scorer_cmd.go new file mode 100644 index 0000000..18f6a57 --- /dev/null +++ b/pkg/cmd/scorer_cmd.go @@ -0,0 +1,51 @@ +package cmd + +import ( + "fmt" + "log" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/tdahar/block-scorer/pkg/app" + cli "github.com/urfave/cli/v2" +) + +var ScorerCommand = &cli.Command{ + Name: "block-scorer", + Usage: "Receive Block proposals from clients and evaluate score", + Action: LaunchBlockScorer, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "bn-endpoints", + Usage: "beacon node endpoints (label/endpoint,label/endpoint)", + DefaultText: "lh/localhost:5052", + }}, +} + +var logRewardsRewards = logrus.WithField( + "module", "ScorerCommand", +) + +var QueryTimeout = 90 * time.Second + +// CrawlAction is the function that is called when running `eth2`. +func LaunchBlockScorer(c *cli.Context) error { + logRewardsRewards.Info("parsing flags") + // check if a config file is set + if !c.IsSet("bn-endpoints") { + return errors.New("bn endpoint not provided") + } + + bnEndpoints := strings.Split(c.String("bn-endpoints"), ",") + fmt.Printf("%v", bnEndpoints) + + service, err := app.NewAppService(c.Context, bnEndpoints) + if err != nil { + log.Fatal("could not start app: %s", err.Error()) + } + service.Run() + + return nil +} From ef123870e1597cbe551785ec3f603fcf5ed220bf Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 10:36:23 +0200 Subject: [PATCH 02/62] Add gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 66fd13c..0de58e4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +.vscode/** +block-scorer \ No newline at end of file From 02ebe8f16fc665dd948980a26dee599557fb0404 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 14:40:23 +0200 Subject: [PATCH 03/62] Base Setup to get blocks every 12 seconds --- .gitmodules | 4 ++ Makefile | 22 ++++++++++ README.md | 2 +- go-eth2-client | 1 + go.mod | 12 ++--- go.sum | 21 +++++---- main.go | 5 ++- pkg/app/service.go | 34 +++++++++----- pkg/client_api/clientapi.go | 40 +++++++++++++++-- pkg/cmd/scorer_cmd.go | 14 +++--- pkg/postgresql/block_score.go | 83 +++++++++++++++++++++++++++++++++++ pkg/postgresql/service.go | 76 ++++++++++++++++++++++++++++++++ pkg/score/scorer.go | 52 ++++++++++++++++++++++ 13 files changed, 324 insertions(+), 42 deletions(-) create mode 100644 .gitmodules create mode 100644 Makefile create mode 160000 go-eth2-client create mode 100644 pkg/postgresql/block_score.go create mode 100644 pkg/postgresql/service.go create mode 100644 pkg/score/scorer.go diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ca31329 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "go-eth2-client"] + path = go-eth2-client + url = https://github.com/tdahar/go-eth2-client + branch = feature/skip-randao diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1b0e872 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +GOCC=go +MKDIR_P=mkdir -p +GIT_SUBM=git submodule + +BIN_PATH=./build +BIN="./build/block-scorer" + +.PHONY: check dependencies build install clean + +build: + $(GOCC) build -o $(BIN) + + +install: + $(GOCC) install + + +dependencies: + $(GIT_SUBM) update --init + +clean: + rm -r $(BIN_PATH) \ No newline at end of file diff --git a/README.md b/README.md index 9822a9a..b141220 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # block-scorer -Tool to ask for blocks from the Ethereum CL clients and evaluate block score +Tool to ask for blocks from the Ethereum CL clients and evaluate block score. diff --git a/go-eth2-client b/go-eth2-client new file mode 160000 index 0000000..fb3b635 --- /dev/null +++ b/go-eth2-client @@ -0,0 +1 @@ +Subproject commit fb3b635a21fa102b167bb427631e7bb1d24672e6 diff --git a/go.mod b/go.mod index b1fd6ad..5023334 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tdahar/block-scorer go 1.17 require ( - github.com/attestantio/go-eth2-client v0.13.6 + github.com/attestantio/go-eth2-client v0.11.7 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.28.0 github.com/sirupsen/logrus v1.9.0 @@ -13,20 +13,22 @@ require ( require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/ferranbt/fastssz v0.1.1 // indirect + github.com/ferranbt/fastssz v0.0.0-20220103083642-bc5fefefa28b // indirect github.com/goccy/go-yaml v1.9.5 // indirect - github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/klauspost/cpuid/v2 v2.0.11 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/minio/sha256-simd v1.0.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect github.com/r3labs/sse/v2 v2.7.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) + +replace github.com/attestantio/go-eth2-client => ./go-eth2-client \ No newline at end of file diff --git a/go.sum b/go.sum index f9cbd98..291e880 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/attestantio/go-eth2-client v0.13.6 h1:KivmTvEdW5Ah3wSsfb8u/8cZZh/BtaRwNJuPpLqhQdg= -github.com/attestantio/go-eth2-client v0.13.6/go.mod h1:bcg5gfjVcm+MtcaZfzv/uSWNHU4i8hGamVG+9JCZnC0= +github.com/attestantio/go-eth2-client v0.11.7 h1:mRCtVZYg/Tmp47mNkWKnro/Wj63+jj1vz9BWNQN8MYo= +github.com/attestantio/go-eth2-client v0.11.7/go.mod h1:zXL/BxC0cBBhxj+tP7QG7t9Ufoa8GwQLdlbvZRd9+dM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -66,8 +66,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/ferranbt/fastssz v0.1.1 h1:hBYNxKu51wjPC9sQYCjicmy5wtJqubENp3IiRVcdJBM= -github.com/ferranbt/fastssz v0.1.1/go.mod h1:U2ZsxlYyvGeQGmadhz8PlEqwkBzDIhHwd3xuKrg2JIs= +github.com/ferranbt/fastssz v0.0.0-20220103083642-bc5fefefa28b h1:Jea4sHxe4sTegJgpfhWvxSjFF2nyq4/R/qWm6AziPiI= +github.com/ferranbt/fastssz v0.0.0-20220103083642-bc5fefefa28b/go.mod h1:S8yiDeAXy8f88W4Ul+0dBMPx49S05byYbmZD6Uv94K4= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -158,8 +158,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= -github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.0.11 h1:i2lw1Pm7Yi/4O6XCSyJWqEHI2MDw2FzUK6o/D21xn2A= +github.com/klauspost/cpuid/v2 v2.0.11/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -183,8 +183,8 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -384,10 +384,9 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= -golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 6e1554f..3b47d3b 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ import ( ) var ( - Version = "v0.0.1" + Version = "v1.0.0" CliName = "Eth2 Block Scorer" log = logrus.WithField( "cli", "CliName", @@ -21,6 +21,9 @@ var ( func main() { fmt.Println(CliName, Version) + // Set the general log configurations for the entire tool + logrus.SetLevel(logrus.DebugLevel) + app := &cli.App{ Name: CliName, Usage: "Tinny client that requests and processes the Beacon Block proposals for each client.", diff --git a/pkg/app/service.go b/pkg/app/service.go index 321def9..93e88c8 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -10,21 +10,23 @@ import ( "github.com/sirupsen/logrus" "github.com/tdahar/block-scorer/pkg/chain_stats" "github.com/tdahar/block-scorer/pkg/client_api" + "github.com/tdahar/block-scorer/pkg/score" ) var ( - modName = "App" + modName = "Main App" log = logrus.WithField( "module", modName, ) ) type AppService struct { - ctx context.Context - Clients []*client_api.APIClient - initTime time.Time - ChainTime chain_stats.ChainTime - HeadSlot phase0.Slot + ctx context.Context + Clients []*client_api.APIClient + ScoreService *score.BlockScorer + initTime time.Time + ChainTime chain_stats.ChainTime + HeadSlot phase0.Slot } func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, error) { @@ -53,11 +55,13 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro if err != nil { return nil, fmt.Errorf("could not obtain head block header: %s", err) } + return &AppService{ - ctx: ctx, - Clients: clients, - initTime: time.Now(), - HeadSlot: headHeader.Header.Message.Slot, + ctx: ctx, + Clients: clients, + initTime: time.Now(), + HeadSlot: headHeader.Header.Message.Slot, + ScoreService: score.NewBlockScorer(ctx), ChainTime: chain_stats.ChainTime{ GenesisTime: genesis, }, @@ -65,7 +69,8 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro } func (s *AppService) Run() { - + log = log.WithField("routine", "main") + go s.ScoreService.ListenBlocks() ticker := time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) for { @@ -81,7 +86,12 @@ func (s *AppService) Run() { log.Infof("Entered a new slot!: %d, time: %s", s.HeadSlot, time.Now()) ticker = time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) // a new slot has begun, therefore execute all needed actions - log.Infof("Next Duration: %d", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1)))) + log.Tracef("Next Duration: %s", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1))).String()) + + for _, client := range s.Clients { + + go client.RequestBlock(s.HeadSlot, s.ScoreService.BlockChan) + } default: } } diff --git a/pkg/client_api/clientapi.go b/pkg/client_api/clientapi.go index b9bb78d..9070af5 100644 --- a/pkg/client_api/clientapi.go +++ b/pkg/client_api/clientapi.go @@ -5,6 +5,8 @@ import ( "time" "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec" + "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" ) @@ -12,7 +14,7 @@ import ( var ( moduleName = "API-Cli" log = logrus.WithField( - "module", "") + "module", moduleName) ) type APIClient struct { @@ -38,7 +40,39 @@ func NewAPIClient(ctx context.Context, label string, cliEndpoint string, timeout log.Error("gernerating the http api client") } return &APIClient{ - ctx: ctx, - Api: hc, + ctx: ctx, + Api: hc, + Label: label, }, nil } + +func (p APIClient) String() string { + return p.Label + "->" + p.Api.Address() +} + +func (p APIClient) RequestBlock(slot phase0.Slot, resC chan *APIBlockAnswer) error { + log = log.WithField("process", "block-request") + randaoReveal := phase0.BLSSignature{} + graffiti := []byte("") + blockAnswer := APIBlockAnswer{ + Error: nil, + Label: p.Label, + } + log.Debugf("Asking for a new block...") + block, err := p.Api.BeaconBlockProposal(p.ctx, slot, randaoReveal, graffiti) + if err != nil { + log.Errorf("error requesting block from %s: %s", p.Label, err) + blockAnswer.Error = err + } + blockAnswer.Block = block + log.Tracef("Sending block answer for analysis...") + resC <- &blockAnswer + + return nil +} + +type APIBlockAnswer struct { + Block *spec.VersionedBeaconBlock + Label string + Error error +} diff --git a/pkg/cmd/scorer_cmd.go b/pkg/cmd/scorer_cmd.go index 18f6a57..07d7f30 100644 --- a/pkg/cmd/scorer_cmd.go +++ b/pkg/cmd/scorer_cmd.go @@ -1,13 +1,11 @@ package cmd import ( - "fmt" - "log" "strings" "time" "github.com/pkg/errors" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "github.com/tdahar/block-scorer/pkg/app" cli "github.com/urfave/cli/v2" ) @@ -24,22 +22,20 @@ var ScorerCommand = &cli.Command{ }}, } -var logRewardsRewards = logrus.WithField( - "module", "ScorerCommand", -) - var QueryTimeout = 90 * time.Second // CrawlAction is the function that is called when running `eth2`. func LaunchBlockScorer(c *cli.Context) error { - logRewardsRewards.Info("parsing flags") + logLauncher := log.WithField( + "module", "ScorerCommand", + ) + logLauncher.Info("parsing flags") // check if a config file is set if !c.IsSet("bn-endpoints") { return errors.New("bn endpoint not provided") } bnEndpoints := strings.Split(c.String("bn-endpoints"), ",") - fmt.Printf("%v", bnEndpoints) service, err := app.NewAppService(c.Context, bnEndpoints) if err != nil { diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go new file mode 100644 index 0000000..8feffa6 --- /dev/null +++ b/pkg/postgresql/block_score.go @@ -0,0 +1,83 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +var ( + CREATE_SCORE_TABLE = ` + CREATE TABLE IF NOT EXISTS t_score_metrics( + f_slot INT, + f_label VARCHAR(100), + f_score FLOAT, + f_duration FLOAT, + f_correct_source INT, + f_correct_target INT, + f_correct_head INT, + f_sync_bits INT, + f_att_num INT, + f_new_votes INT, + CONSTRAINT PK_SlotAddr PRIMARY KEY (f_slot,f_label));` + + InsertNewScore = ` + INSERT INTO t_score_metrics ( + f_slot, + f_label, + f_score, + f_duration, + f_correct_source, + f_correct_target, + f_correct_head, + f_sync_bits, + f_att_num, + f_new_votes) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);` +) + +// in case the table did not exist +func (p *PostgresDBService) createScoreMetricsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, CREATE_SCORE_TABLE) + if err != nil { + return errors.Wrap(err, "error creating score metrics table") + } + return nil +} + +func (p *PostgresDBService) InsertNewScore(slot int, label string, score float64, duration float64, attMetrics AttestationMetrics) error { + + _, err := p.psqlPool.Exec(p.ctx, InsertNewScore, + slot, + label, + score, + duration, + attMetrics.CorrectSource, + attMetrics.CorrectTarget, + attMetrics.CorrectHead, + attMetrics.Sync1Bits, + attMetrics.AttNum, + attMetrics.NewVotes) + + if err != nil { + return errors.Wrap(err, "error inserting row in score metrics table") + } + return nil +} + +type AttestationMetrics struct { + CorrectSource int + CorrectTarget int + CorrectHead int + Sync1Bits int + AttNum int + NewVotes int +} diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go new file mode 100644 index 0000000..60e65c6 --- /dev/null +++ b/pkg/postgresql/service.go @@ -0,0 +1,76 @@ +package postgresql + +import ( + "context" + "strings" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// Static postgres queries, for each modification in the tables, the table needs to be reseted +var ( + // logrus associated with the postgres db + PsqlType = "postgres-db" + log = logrus.WithField( + "module", PsqlType, + ) + MAX_BATCH_QUEUE = 300 +) + +type PostgresDBService struct { + // Control Variables + ctx context.Context + cancel context.CancelFunc + connectionUrl string // the url might not be necessary (better to remove it?¿) + psqlPool *pgxpool.Pool + + // Network DB Model +} + +// Connect to the PostgreSQL Database and get the multithread-proof connection +// from the given url-composed credentials +func ConnectToDB(ctx context.Context, url string) (*PostgresDBService, error) { + mainCtx, cancel := context.WithCancel(ctx) + // spliting the url to don't share any confidential information on logs + log.Infof("Conneting to postgres DB %s", url) + if strings.Contains(url, "@") { + log.Debugf("Connecting to PostgresDB at %s", strings.Split(url, "@")[1]) + } + psqlPool, err := pgxpool.Connect(mainCtx, url) + if err != nil { + return nil, err + } + if strings.Contains(url, "@") { + log.Infof("PostgresDB %s succesfully connected", strings.Split(url, "@")[1]) + } + // filter the type of network that we are filtering + + psqlDB := &PostgresDBService{ + ctx: mainCtx, + cancel: cancel, + connectionUrl: url, + psqlPool: psqlPool, + } + // init the psql db + err = psqlDB.init(ctx, psqlDB.psqlPool) + if err != nil { + return psqlDB, errors.Wrap(err, "error initializing the tables of the psqldb") + } + return psqlDB, err +} + +// Close the connection with the PostgreSQL +func (p *PostgresDBService) Close() { +} + +func (p *PostgresDBService) init(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + err := p.createScoreMetricsTable(ctx, pool) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/score/scorer.go b/pkg/score/scorer.go new file mode 100644 index 0000000..a01292e --- /dev/null +++ b/pkg/score/scorer.go @@ -0,0 +1,52 @@ +package score + +import ( + "context" + + "github.com/sirupsen/logrus" + "github.com/tdahar/block-scorer/pkg/client_api" +) + +var ( + moduleName = "Scorer" + log = logrus.WithField( + "module", moduleName) +) + +type BlockScorer struct { + ctx context.Context + BlockChan chan *client_api.APIBlockAnswer +} + +func NewBlockScorer(ctx context.Context) *BlockScorer { + return &BlockScorer{ + ctx: ctx, + BlockChan: make(chan *client_api.APIBlockAnswer, 10), + } +} + +func (s *BlockScorer) ListenBlocks() { + for { + + select { + case <-s.ctx.Done(): + log.Infof("context has died, closing block listener routine") + close(s.BlockChan) + return + + case blockTask, ok := <-s.BlockChan: + if !ok { + log.Errorf("could not receive new block task") + } else { + log.Infof("Analyzing block from %s!", blockTask.Label) + } + + default: + } + + } +} + +type BeaconBlockMetrics struct { + Score float64 +} From 87d53fd6f96aece9e93b05f2abe0337ec11b8774 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 15:54:14 +0200 Subject: [PATCH 04/62] Reorganized repository and added attestation evaluation --- go.mod | 13 +- go.sum | 113 ++++++++++++++++++ pkg/{score/scorer.go => analysis/analyzer.go} | 17 ++- pkg/analysis/score.go | 62 ++++++++++ pkg/app/service.go | 2 +- pkg/utils/attestation.go | 27 +++++ 6 files changed, 226 insertions(+), 8 deletions(-) rename pkg/{score/scorer.go => analysis/analyzer.go} (70%) create mode 100644 pkg/analysis/score.go create mode 100644 pkg/utils/attestation.go diff --git a/go.mod b/go.mod index 5023334..9d13c27 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,15 @@ require ( github.com/fatih/color v1.13.0 // indirect github.com/ferranbt/fastssz v0.0.0-20220103083642-bc5fefefa28b // indirect github.com/goccy/go-yaml v1.9.5 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.13.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgx/v4 v4.17.2 // indirect + github.com/jackc/puddle v1.3.0 // indirect github.com/klauspost/cpuid/v2 v2.0.11 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect @@ -24,11 +33,13 @@ require ( github.com/r3labs/sse/v2 v2.7.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) -replace github.com/attestantio/go-eth2-client => ./go-eth2-client \ No newline at end of file +replace github.com/attestantio/go-eth2-client => ./go-eth2-client diff --git a/go.sum b/go.sum index 291e880..7673a97 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,7 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -52,10 +53,14 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -88,6 +93,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -147,6 +153,55 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= +github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= +github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= +github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -161,20 +216,30 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.11 h1:i2lw1Pm7Yi/4O6XCSyJWqEHI2MDw2FzUK6o/D21xn2A= github.com/klauspost/cpuid/v2 v2.0.11/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -221,25 +286,37 @@ github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= github.com/r3labs/sse/v2 v2.7.4/go.mod h1:hUrYMKfu9WquG9MyI0r6TKiNH+6Sw/QPKm2YbNbU5g8= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/urfave/cli/v2 v2.16.3 h1:gHoFIwpPjoyIMbJp/VFd+/vuD0dAgFK4B6DpEMFJfQk= github.com/urfave/cli/v2 v2.16.3/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -248,19 +325,38 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -306,6 +402,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -324,6 +421,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -346,7 +444,9 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -354,7 +454,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -387,6 +489,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -394,7 +497,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -406,14 +511,18 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -421,6 +530,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -440,6 +550,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -528,6 +640,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/score/scorer.go b/pkg/analysis/analyzer.go similarity index 70% rename from pkg/score/scorer.go rename to pkg/analysis/analyzer.go index a01292e..963db3a 100644 --- a/pkg/score/scorer.go +++ b/pkg/analysis/analyzer.go @@ -1,4 +1,4 @@ -package score +package analysis import ( "context" @@ -13,19 +13,19 @@ var ( "module", moduleName) ) -type BlockScorer struct { +type BlockAnalyzer struct { ctx context.Context BlockChan chan *client_api.APIBlockAnswer } -func NewBlockScorer(ctx context.Context) *BlockScorer { - return &BlockScorer{ +func NewBlockScorer(ctx context.Context) *BlockAnalyzer { + return &BlockAnalyzer{ ctx: ctx, BlockChan: make(chan *client_api.APIBlockAnswer, 10), } } -func (s *BlockScorer) ListenBlocks() { +func (s *BlockAnalyzer) ListenBlocks() { for { select { @@ -48,5 +48,10 @@ func (s *BlockScorer) ListenBlocks() { } type BeaconBlockMetrics struct { - Score float64 + Score float64 + AttestationNum int64 + NewVotes int64 + CorrectSource int64 + CorrectTarget int64 + CorrectHead int64 } diff --git a/pkg/analysis/score.go b/pkg/analysis/score.go new file mode 100644 index 0000000..291610f --- /dev/null +++ b/pkg/analysis/score.go @@ -0,0 +1,62 @@ +package analysis + +import ( + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/prysmaticlabs/go-bitfield" + "github.com/tdahar/block-scorer/pkg/utils" +) + +const ( + TIMELY_SOURCE_WEIGHT = 14 + TIMELY_TARGET_WEIGHT = 26 + TIMELY_HEAD_WEIGHT = 14 +) + +func BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (BeaconBlockMetrics, error) { + totalNewVotes := 0 + totalScore := 0 + // Map is attestation slot -> committee index -> validator committee index -> aggregate. + attested := make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist) + for _, attestation := range block.Body.Attestations { + newVotes := 0 + slot := attestation.Data.Slot + + if _, exists := attested[slot]; !exists { + // add slot to map + attested[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) + } + + committeIndex := attestation.Data.Index + if _, exists := attested[slot][committeIndex]; !exists { + attested[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) + } + + attestingIndices := attestation.AggregationBits.BitIndices() + + for _, idx := range attestingIndices { + if attested[slot][committeIndex].BitAt(uint64(idx)) { + // already registered vote + continue + } + attested[slot][committeIndex].SetBitAt(uint64(idx), true) + newVotes++ + } + score := 0 + if utils.IsCorrectHead(*attestation, *block) { + score += newVotes * TIMELY_HEAD_WEIGHT + } + + if utils.IsCorrectTarget(*attestation, *block) { + score += newVotes * TIMELY_TARGET_WEIGHT + } + + totalNewVotes += newVotes + totalScore += score + + } + + return BeaconBlockMetrics{ + Score: float64(totalScore), + }, nil +} diff --git a/pkg/app/service.go b/pkg/app/service.go index 93e88c8..b94aea3 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -23,7 +23,7 @@ var ( type AppService struct { ctx context.Context Clients []*client_api.APIClient - ScoreService *score.BlockScorer + ScoreService *score.BlockAnalyzer initTime time.Time ChainTime chain_stats.ChainTime HeadSlot phase0.Slot diff --git a/pkg/utils/attestation.go b/pkg/utils/attestation.go new file mode 100644 index 0000000..f4d5c8d --- /dev/null +++ b/pkg/utils/attestation.go @@ -0,0 +1,27 @@ +package utils + +import ( + "bytes" + + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/phase0" +) + +func IsCorrectSource(attestation phase0.Attestation, block bellatrix.BeaconBlock) bool { + return true +} + +func IsCorrectTarget(attestation phase0.Attestation, block bellatrix.BeaconBlock) bool { + + return true + +} + +func IsCorrectHead(attestation phase0.Attestation, block bellatrix.BeaconBlock) bool { + if bytes.Equal(block.ParentRoot[:], attestation.Data.BeaconBlockRoot[:]) { + if block.Slot-attestation.Data.Slot == 1 { + return true + } + } + return false +} From 1b7424551692b1b98cc561b633eeba207632d6d9 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 18:56:21 +0200 Subject: [PATCH 05/62] Improved flag checks --- pkg/utils/attestation.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/utils/attestation.go b/pkg/utils/attestation.go index f4d5c8d..4e1b768 100644 --- a/pkg/utils/attestation.go +++ b/pkg/utils/attestation.go @@ -8,12 +8,20 @@ import ( ) func IsCorrectSource(attestation phase0.Attestation, block bellatrix.BeaconBlock) bool { - return true + return block.Slot-attestation.Data.Slot <= 5 } -func IsCorrectTarget(attestation phase0.Attestation, block bellatrix.BeaconBlock) bool { +func IsCorrectTarget(attestation phase0.Attestation, block bellatrix.BeaconBlock, rootHistory map[phase0.Slot]phase0.Root) bool { + attEpoch := int(attestation.Data.Slot / 32) + firstSlotOfEpoch := phase0.Slot(attEpoch * 32) - return true + if root, ok := rootHistory[firstSlotOfEpoch]; !ok { + // assume it is okay, as we dont have any data about the root + return true + } else { + // we have data, compare the roots + return bytes.Equal(root[:], attestation.Data.Target.Root[:]) + } } From 69f30de7efd7b13cd8df2f3540d8654df7a8ff7f Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 18:57:11 +0200 Subject: [PATCH 06/62] Add history and block metrics --- pkg/analysis/analyzer.go | 70 ++++++++++++++++++++++++++------------ pkg/analysis/metrics.go | 72 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 22 deletions(-) create mode 100644 pkg/analysis/metrics.go diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 963db3a..9ca64b9 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -2,53 +2,79 @@ package analysis import ( "context" + "fmt" + "time" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/prysmaticlabs/go-bitfield" "github.com/sirupsen/logrus" "github.com/tdahar/block-scorer/pkg/client_api" ) var ( - moduleName = "Scorer" + moduleName = "Analysis" log = logrus.WithField( "module", moduleName) ) type BlockAnalyzer struct { - ctx context.Context - BlockChan chan *client_api.APIBlockAnswer + ctx context.Context + Eth2Provider client_api.APIClient + AttHistory map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist + BlockRootHistory map[phase0.Slot]phase0.Root + log *logrus.Entry } -func NewBlockScorer(ctx context.Context) *BlockAnalyzer { - return &BlockAnalyzer{ - ctx: ctx, - BlockChan: make(chan *client_api.APIBlockAnswer, 10), +func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, timeout time.Duration) (*BlockAnalyzer, error) { + client, err := client_api.NewAPIClient(ctx, label, cliEndpoint, timeout) + if err != nil { + log.Errorf("could not create eth2 client: %s", err) + return &BlockAnalyzer{}, err } + return &BlockAnalyzer{ + ctx: ctx, + Eth2Provider: *client, + AttHistory: make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist), + log: log.WithField("label", label), + }, nil } -func (s *BlockAnalyzer) ListenBlocks() { - for { +func (b *BlockAnalyzer) ProcessNewBlock(slot phase0.Slot) error { - select { - case <-s.ctx.Done(): - log.Infof("context has died, closing block listener routine") - close(s.BlockChan) - return + randaoReveal := phase0.BLSSignature{} + graffiti := []byte("") + snapshot := time.Now() + block, err := b.Eth2Provider.Api.BeaconBlockProposal(b.ctx, slot, randaoReveal, graffiti) + blockTime := time.Since(snapshot).Seconds() + if err != nil { + return fmt.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) - case blockTask, ok := <-s.BlockChan: - if !ok { - log.Errorf("could not receive new block task") - } else { - log.Infof("Analyzing block from %s!", blockTask.Label) - } + } + for i := range b.AttHistory { + if i+32 < slot { // attestations can only reference 32 slots back + delete(b.AttHistory, i) // remove old entries from the map + } + } - default: + for i := range b.BlockRootHistory { + if i+64 < slot { // attestations can only reference 32 slots back + delete(b.BlockRootHistory, i) // remove old entries from the map } + } + metrics, err := b.BellatrixBlockMetrics(block.Bellatrix) + if err != nil { + return fmt.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) } + b.log.Infof("Block Duration: %f", blockTime) + b.log.Infof("Metrics: %+v", metrics) + + return nil } type BeaconBlockMetrics struct { - Score float64 + AttScore float64 + SyncScore float64 AttestationNum int64 NewVotes int64 CorrectSource int64 diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go new file mode 100644 index 0000000..2aa64c6 --- /dev/null +++ b/pkg/analysis/metrics.go @@ -0,0 +1,72 @@ +package analysis + +import ( + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/prysmaticlabs/go-bitfield" + "github.com/tdahar/block-scorer/pkg/utils" +) + +const ( + TIMELY_SOURCE_WEIGHT = 14 + TIMELY_TARGET_WEIGHT = 26 + TIMELY_HEAD_WEIGHT = 14 + PROPOSER_WEIGHT = 8 + WEIGHT_DENOMINATOR = 64 + SYNC_REWARD_WEIGHT = 2 +) + +// Map is attestation slot -> committee index -> validator committee index -> aggregate. +func (p *BlockAnalyzer) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (BeaconBlockMetrics, error) { + log = log.WithField("method", "bellatrix-block") + totalNewVotes := 0 + totalScore := 0 + + for _, attestation := range block.Body.Attestations { + newVotes := 0 + slot := attestation.Data.Slot + + if _, exists := p.AttHistory[slot]; !exists { + // add slot to map + p.AttHistory[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) + } + + committeIndex := attestation.Data.Index + if _, exists := p.AttHistory[slot][committeIndex]; !exists { + p.AttHistory[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) + } + + attestingIndices := attestation.AggregationBits.BitIndices() + + for _, idx := range attestingIndices { + if p.AttHistory[slot][committeIndex].BitAt(uint64(idx)) { + // already registered vote + continue + } + p.AttHistory[slot][committeIndex].SetBitAt(uint64(idx), true) + newVotes++ + } + score := 0 + if utils.IsCorrectSource(*attestation, *block) { + score += newVotes * TIMELY_SOURCE_WEIGHT + } + if utils.IsCorrectTarget(*attestation, *block, p.BlockRootHistory) { + score += newVotes * TIMELY_TARGET_WEIGHT + } + if utils.IsCorrectHead(*attestation, *block) { + score += newVotes * TIMELY_HEAD_WEIGHT + } + + totalNewVotes += newVotes + // denominator := (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) * WEIGHT_DENOMINATOR / PROPOSER_WEIGHT + totalScore += score / WEIGHT_DENOMINATOR + + } + + syncCommitteeScore := float64(block.Body.SyncAggregate.SyncCommitteeBits.Count()) * float64(SYNC_REWARD_WEIGHT) / float64(WEIGHT_DENOMINATOR) + + return BeaconBlockMetrics{ + AttScore: float64(totalScore), + SyncScore: float64(syncCommitteeScore), + }, nil +} From 0344135e6e39a08abb85f7d069246fae332c128b Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 18:57:49 +0200 Subject: [PATCH 07/62] Remove extra dependencies and methods --- pkg/app/service.go | 40 ++++++++++++++++++------------------- pkg/client_api/clientapi.go | 29 --------------------------- 2 files changed, 19 insertions(+), 50 deletions(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index b94aea3..ec451ff 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -8,9 +8,8 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" + "github.com/tdahar/block-scorer/pkg/analysis" "github.com/tdahar/block-scorer/pkg/chain_stats" - "github.com/tdahar/block-scorer/pkg/client_api" - "github.com/tdahar/block-scorer/pkg/score" ) var ( @@ -21,17 +20,16 @@ var ( ) type AppService struct { - ctx context.Context - Clients []*client_api.APIClient - ScoreService *score.BlockAnalyzer - initTime time.Time - ChainTime chain_stats.ChainTime - HeadSlot phase0.Slot + ctx context.Context + Analyzers []*analysis.BlockAnalyzer + initTime time.Time + ChainTime chain_stats.ChainTime + HeadSlot phase0.Slot } func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, error) { - clients := make([]*client_api.APIClient, 0) + analyzers := make([]*analysis.BlockAnalyzer, 0) for i := range bnEndpoints { if !strings.Contains(bnEndpoints[i], "/") { @@ -39,29 +37,28 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro } label := strings.Split(bnEndpoints[i], "/")[0] endpoint := strings.Split(bnEndpoints[i], "/")[1] - newClient, err := client_api.NewAPIClient(ctx, label, endpoint, time.Second*5) + newAnalyzer, err := analysis.NewBlockAnalyzer(ctx, label, endpoint, time.Second*5) if err != nil { log.Errorf("could not create client for endpoint: %s ", endpoint, err) continue } - clients = append(clients, newClient) + analyzers = append(analyzers, newAnalyzer) } - genesis, err := clients[0].Api.GenesisTime(ctx) + genesis, err := analyzers[0].Eth2Provider.Api.GenesisTime(ctx) if err != nil { return nil, fmt.Errorf("could not obtain genesis time: %s", err) } - headHeader, err := clients[0].Api.BeaconBlockHeader(ctx, "head") + headHeader, err := analyzers[0].Eth2Provider.Api.BeaconBlockHeader(ctx, "head") if err != nil { return nil, fmt.Errorf("could not obtain head block header: %s", err) } return &AppService{ - ctx: ctx, - Clients: clients, - initTime: time.Now(), - HeadSlot: headHeader.Header.Message.Slot, - ScoreService: score.NewBlockScorer(ctx), + ctx: ctx, + Analyzers: analyzers, + initTime: time.Now(), + HeadSlot: headHeader.Header.Message.Slot, // start 64 slots behind to create attestation history ChainTime: chain_stats.ChainTime{ GenesisTime: genesis, }, @@ -70,7 +67,7 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro func (s *AppService) Run() { log = log.WithField("routine", "main") - go s.ScoreService.ListenBlocks() + ticker := time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) for { @@ -84,13 +81,14 @@ func (s *AppService) Run() { s.HeadSlot++ log.Infof("Entered a new slot!: %d, time: %s", s.HeadSlot, time.Now()) + // reset ticker to next slot ticker = time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) // a new slot has begun, therefore execute all needed actions log.Tracef("Next Duration: %s", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1))).String()) - for _, client := range s.Clients { + for _, analyzer := range s.Analyzers { - go client.RequestBlock(s.HeadSlot, s.ScoreService.BlockChan) + go analyzer.ProcessNewBlock(s.HeadSlot) } default: } diff --git a/pkg/client_api/clientapi.go b/pkg/client_api/clientapi.go index 9070af5..19ec663 100644 --- a/pkg/client_api/clientapi.go +++ b/pkg/client_api/clientapi.go @@ -5,8 +5,6 @@ import ( "time" "github.com/attestantio/go-eth2-client/http" - "github.com/attestantio/go-eth2-client/spec" - "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/rs/zerolog" "github.com/sirupsen/logrus" ) @@ -49,30 +47,3 @@ func NewAPIClient(ctx context.Context, label string, cliEndpoint string, timeout func (p APIClient) String() string { return p.Label + "->" + p.Api.Address() } - -func (p APIClient) RequestBlock(slot phase0.Slot, resC chan *APIBlockAnswer) error { - log = log.WithField("process", "block-request") - randaoReveal := phase0.BLSSignature{} - graffiti := []byte("") - blockAnswer := APIBlockAnswer{ - Error: nil, - Label: p.Label, - } - log.Debugf("Asking for a new block...") - block, err := p.Api.BeaconBlockProposal(p.ctx, slot, randaoReveal, graffiti) - if err != nil { - log.Errorf("error requesting block from %s: %s", p.Label, err) - blockAnswer.Error = err - } - blockAnswer.Block = block - log.Tracef("Sending block answer for analysis...") - resC <- &blockAnswer - - return nil -} - -type APIBlockAnswer struct { - Block *spec.VersionedBeaconBlock - Label string - Error error -} From 7485f2b391d1ac8ff68059ecfa53320313aef941 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 21 Sep 2022 18:58:20 +0200 Subject: [PATCH 08/62] Move score as metrics --- pkg/analysis/score.go | 62 ------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 pkg/analysis/score.go diff --git a/pkg/analysis/score.go b/pkg/analysis/score.go deleted file mode 100644 index 291610f..0000000 --- a/pkg/analysis/score.go +++ /dev/null @@ -1,62 +0,0 @@ -package analysis - -import ( - "github.com/attestantio/go-eth2-client/spec/bellatrix" - "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/prysmaticlabs/go-bitfield" - "github.com/tdahar/block-scorer/pkg/utils" -) - -const ( - TIMELY_SOURCE_WEIGHT = 14 - TIMELY_TARGET_WEIGHT = 26 - TIMELY_HEAD_WEIGHT = 14 -) - -func BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (BeaconBlockMetrics, error) { - totalNewVotes := 0 - totalScore := 0 - // Map is attestation slot -> committee index -> validator committee index -> aggregate. - attested := make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist) - for _, attestation := range block.Body.Attestations { - newVotes := 0 - slot := attestation.Data.Slot - - if _, exists := attested[slot]; !exists { - // add slot to map - attested[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) - } - - committeIndex := attestation.Data.Index - if _, exists := attested[slot][committeIndex]; !exists { - attested[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) - } - - attestingIndices := attestation.AggregationBits.BitIndices() - - for _, idx := range attestingIndices { - if attested[slot][committeIndex].BitAt(uint64(idx)) { - // already registered vote - continue - } - attested[slot][committeIndex].SetBitAt(uint64(idx), true) - newVotes++ - } - score := 0 - if utils.IsCorrectHead(*attestation, *block) { - score += newVotes * TIMELY_HEAD_WEIGHT - } - - if utils.IsCorrectTarget(*attestation, *block) { - score += newVotes * TIMELY_TARGET_WEIGHT - } - - totalNewVotes += newVotes - totalScore += score - - } - - return BeaconBlockMetrics{ - Score: float64(totalScore), - }, nil -} From f27f2c7766403b57ec06f3cb47620ec886774640 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 22 Sep 2022 09:08:54 +0200 Subject: [PATCH 09/62] Added comments and changed score name to analyze --- main.go | 4 ++-- pkg/analysis/analyzer.go | 20 +++++++++++--------- pkg/analysis/metrics.go | 19 +++++++++---------- pkg/app/service.go | 14 +++++++++----- pkg/chain_stats/chaintime.go | 1 + pkg/cmd/{scorer_cmd.go => analyzer_cmd.go} | 9 ++++----- pkg/postgresql/block_score.go | 20 ++++++++++++-------- pkg/postgresql/service.go | 2 +- 8 files changed, 49 insertions(+), 40 deletions(-) rename pkg/cmd/{scorer_cmd.go => analyzer_cmd.go} (81%) diff --git a/main.go b/main.go index 3b47d3b..9723999 100644 --- a/main.go +++ b/main.go @@ -36,10 +36,10 @@ func main() { }, EnableBashCompletion: true, Commands: []*cli.Command{ - cmd.ScorerCommand, + cmd.AnalyzerCommand, }, } - // generate the crawler + // generate the block analyzer if err := app.RunContext(context.Background(), os.Args); err != nil { log.Errorf("error: %v\n", err) os.Exit(1) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 9ca64b9..26c0f0c 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -19,10 +19,10 @@ var ( type BlockAnalyzer struct { ctx context.Context - Eth2Provider client_api.APIClient - AttHistory map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist - BlockRootHistory map[phase0.Slot]phase0.Root - log *logrus.Entry + Eth2Provider client_api.APIClient // connection to the beacon node + AttHistory map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist // 32 slots of attestation per slot and committeeIndex + BlockRootHistory map[phase0.Slot]phase0.Root // 64 slots of roots + log *logrus.Entry // each analyzer has its own logger } func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, timeout time.Duration) (*BlockAnalyzer, error) { @@ -32,10 +32,11 @@ func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, tim return &BlockAnalyzer{}, err } return &BlockAnalyzer{ - ctx: ctx, - Eth2Provider: *client, - AttHistory: make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist), - log: log.WithField("label", label), + ctx: ctx, + Eth2Provider: *client, + AttHistory: make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist), + BlockRootHistory: make(map[phase0.Slot]phase0.Root), + log: log.WithField("label", label), }, nil } @@ -45,7 +46,7 @@ func (b *BlockAnalyzer) ProcessNewBlock(slot phase0.Slot) error { graffiti := []byte("") snapshot := time.Now() block, err := b.Eth2Provider.Api.BeaconBlockProposal(b.ctx, slot, randaoReveal, graffiti) - blockTime := time.Since(snapshot).Seconds() + blockTime := time.Since(snapshot).Seconds() // time to ask for block if err != nil { return fmt.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) @@ -62,6 +63,7 @@ func (b *BlockAnalyzer) ProcessNewBlock(slot phase0.Slot) error { } } + // for now we just have Bellatrix metrics, err := b.BellatrixBlockMetrics(block.Bellatrix) if err != nil { return fmt.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 2aa64c6..3a3d9b3 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -16,9 +16,8 @@ const ( SYNC_REWARD_WEIGHT = 2 ) -// Map is attestation slot -> committee index -> validator committee index -> aggregate. -func (p *BlockAnalyzer) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (BeaconBlockMetrics, error) { - log = log.WithField("method", "bellatrix-block") +func (b *BlockAnalyzer) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (BeaconBlockMetrics, error) { + // log = b.log.WithField("method", "bellatrix-block") // add extra log for function totalNewVotes := 0 totalScore := 0 @@ -26,31 +25,31 @@ func (p *BlockAnalyzer) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (Bea newVotes := 0 slot := attestation.Data.Slot - if _, exists := p.AttHistory[slot]; !exists { + if _, exists := b.AttHistory[slot]; !exists { // add slot to map - p.AttHistory[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) + b.AttHistory[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) } committeIndex := attestation.Data.Index - if _, exists := p.AttHistory[slot][committeIndex]; !exists { - p.AttHistory[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) + if _, exists := b.AttHistory[slot][committeIndex]; !exists { + b.AttHistory[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) } attestingIndices := attestation.AggregationBits.BitIndices() for _, idx := range attestingIndices { - if p.AttHistory[slot][committeIndex].BitAt(uint64(idx)) { + if b.AttHistory[slot][committeIndex].BitAt(uint64(idx)) { // already registered vote continue } - p.AttHistory[slot][committeIndex].SetBitAt(uint64(idx), true) + b.AttHistory[slot][committeIndex].SetBitAt(uint64(idx), true) newVotes++ } score := 0 if utils.IsCorrectSource(*attestation, *block) { score += newVotes * TIMELY_SOURCE_WEIGHT } - if utils.IsCorrectTarget(*attestation, *block, p.BlockRootHistory) { + if utils.IsCorrectTarget(*attestation, *block, b.BlockRootHistory) { score += newVotes * TIMELY_TARGET_WEIGHT } if utils.IsCorrectHead(*attestation, *block) { diff --git a/pkg/app/service.go b/pkg/app/service.go index ec451ff..b48a8c4 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -29,9 +29,10 @@ type AppService struct { func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, error) { - analyzers := make([]*analysis.BlockAnalyzer, 0) + analyzers := make([]*analysis.BlockAnalyzer, 0) // one analyzer per beacon node for i := range bnEndpoints { + // parse each beacon node endpoint if !strings.Contains(bnEndpoints[i], "/") { log.Errorf("incorrect format for endpoint: %s", bnEndpoints[i]) } @@ -44,11 +45,12 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro } analyzers = append(analyzers, newAnalyzer) } - + // get genesis time to calculate each slot time genesis, err := analyzers[0].Eth2Provider.Api.GenesisTime(ctx) if err != nil { return nil, fmt.Errorf("could not obtain genesis time: %s", err) } + // check the current chain head headHeader, err := analyzers[0].Eth2Provider.Api.BeaconBlockHeader(ctx, "head") if err != nil { return nil, fmt.Errorf("could not obtain head block header: %s", err) @@ -65,9 +67,11 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro }, nil } +// Main routine func (s *AppService) Run() { log = log.WithField("routine", "main") + // tick every slot start (12 seconds) ticker := time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) for { @@ -78,16 +82,16 @@ func (s *AppService) Run() { return case <-ticker: - + // we entered a new slot time s.HeadSlot++ log.Infof("Entered a new slot!: %d, time: %s", s.HeadSlot, time.Now()) // reset ticker to next slot ticker = time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) // a new slot has begun, therefore execute all needed actions - log.Tracef("Next Duration: %s", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1))).String()) + log.Tracef("Time until next slot tick: %s", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1))).String()) for _, analyzer := range s.Analyzers { - + // for each beacon node, get a new block and analyze it go analyzer.ProcessNewBlock(s.HeadSlot) } default: diff --git a/pkg/chain_stats/chaintime.go b/pkg/chain_stats/chaintime.go index dd4bd2a..5adf048 100644 --- a/pkg/chain_stats/chaintime.go +++ b/pkg/chain_stats/chaintime.go @@ -14,6 +14,7 @@ type ChainTime struct { GenesisTime time.Time } +// Calculate at which time a given slot happens func (c ChainTime) SlotTime(slot phase0.Slot) time.Time { return c.GenesisTime.Add(time.Duration(slot) * SLOT_DURATION * time.Second) } diff --git a/pkg/cmd/scorer_cmd.go b/pkg/cmd/analyzer_cmd.go similarity index 81% rename from pkg/cmd/scorer_cmd.go rename to pkg/cmd/analyzer_cmd.go index 07d7f30..5ddb5c0 100644 --- a/pkg/cmd/scorer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -10,10 +10,10 @@ import ( cli "github.com/urfave/cli/v2" ) -var ScorerCommand = &cli.Command{ +var AnalyzerCommand = &cli.Command{ Name: "block-scorer", Usage: "Receive Block proposals from clients and evaluate score", - Action: LaunchBlockScorer, + Action: LaunchBlockAnalyzer, Flags: []cli.Flag{ &cli.StringFlag{ Name: "bn-endpoints", @@ -24,13 +24,12 @@ var ScorerCommand = &cli.Command{ var QueryTimeout = 90 * time.Second -// CrawlAction is the function that is called when running `eth2`. -func LaunchBlockScorer(c *cli.Context) error { +func LaunchBlockAnalyzer(c *cli.Context) error { logLauncher := log.WithField( "module", "ScorerCommand", ) logLauncher.Info("parsing flags") - // check if a config file is set + // check if a beacon node is set if !c.IsSet("bn-endpoints") { return errors.New("bn endpoint not provided") } diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index 8feffa6..ab5b836 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -53,19 +53,19 @@ func (p *PostgresDBService) createScoreMetricsTable(ctx context.Context, pool *p return nil } -func (p *PostgresDBService) InsertNewScore(slot int, label string, score float64, duration float64, attMetrics AttestationMetrics) error { +func (p *PostgresDBService) InsertNewScore(slot int, label string, score float64, duration float64, blockMetrics BlockMetricsModel) error { _, err := p.psqlPool.Exec(p.ctx, InsertNewScore, slot, label, score, duration, - attMetrics.CorrectSource, - attMetrics.CorrectTarget, - attMetrics.CorrectHead, - attMetrics.Sync1Bits, - attMetrics.AttNum, - attMetrics.NewVotes) + blockMetrics.CorrectSource, + blockMetrics.CorrectTarget, + blockMetrics.CorrectHead, + blockMetrics.Sync1Bits, + blockMetrics.AttNum, + blockMetrics.NewVotes) if err != nil { return errors.Wrap(err, "error inserting row in score metrics table") @@ -73,7 +73,11 @@ func (p *PostgresDBService) InsertNewScore(slot int, label string, score float64 return nil } -type AttestationMetrics struct { +type BlockMetricsModel struct { + Slot int + Label string + Score float64 + Duration float64 CorrectSource int CorrectTarget int CorrectHead int diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index 60e65c6..c88df6c 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -34,7 +34,7 @@ type PostgresDBService struct { func ConnectToDB(ctx context.Context, url string) (*PostgresDBService, error) { mainCtx, cancel := context.WithCancel(ctx) // spliting the url to don't share any confidential information on logs - log.Infof("Conneting to postgres DB %s", url) + if strings.Contains(url, "@") { log.Debugf("Connecting to PostgresDB at %s", strings.Split(url, "@")[1]) } From a0e25bf66d3f04051a7d7f0ee12d9cab50bab8dc Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 22 Sep 2022 09:11:46 +0200 Subject: [PATCH 10/62] Add go-eth2-client changes --- go-eth2-client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-eth2-client b/go-eth2-client index fb3b635..44a58a6 160000 --- a/go-eth2-client +++ b/go-eth2-client @@ -1 +1 @@ -Subproject commit fb3b635a21fa102b167bb427631e7bb1d24672e6 +Subproject commit 44a58a628ad5dc63ed6a876b9ed923802144c0c0 From b0a949a3cb5c6dcd2091606f4c28427bf1c826f9 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 2 Nov 2022 13:51:28 +0100 Subject: [PATCH 11/62] Add postgres support --- pkg/postgresql/attestation_arrival.go | 60 ++++++++++++ pkg/postgresql/block_arrival.go | 54 +++++++++++ pkg/postgresql/block_score.go | 10 +- pkg/postgresql/service.go | 128 +++++++++++++++++++++++--- 4 files changed, 235 insertions(+), 17 deletions(-) create mode 100644 pkg/postgresql/attestation_arrival.go create mode 100644 pkg/postgresql/block_arrival.go diff --git a/pkg/postgresql/attestation_arrival.go b/pkg/postgresql/attestation_arrival.go new file mode 100644 index 0000000..51cfda7 --- /dev/null +++ b/pkg/postgresql/attestation_arrival.go @@ -0,0 +1,60 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + "time" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +var ( + CreateAttTable = ` + CREATE TABLE IF NOT EXISTS t_att_metrics( + f_label VARCHAR(100), + f_slot INT, + f_committee_index INT, + f_val_idx INT, + f_timestamp TIME, + CONSTRAINT PK_Attestation PRIMARY KEY (f_label,f_slot,f_committee_index,f_val_idx));` + + InsertNewAtt = ` + INSERT INTO t_att_metrics ( + f_label, + f_slot, + f_committee_index, + f_timestamp, + f_val_idx + ) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT DO NOTHING;` +) + +// in case the table did not exist +func (p *PostgresDBService) createAttMetricsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, CreateAttTable) + if err != nil { + return errors.Wrap(err, "error creating attestation metrics table") + } + return nil +} + +func (p *PostgresDBService) InsertNewAtt(slot int, label string, timestamp time.Time) error { + + _, err := p.psqlPool.Exec(p.ctx, InsertNewAtt, + slot, + label, + timestamp) + + if err != nil { + return errors.Wrap(err, "error inserting row in score metrics table") + } + return nil +} diff --git a/pkg/postgresql/block_arrival.go b/pkg/postgresql/block_arrival.go new file mode 100644 index 0000000..84cd075 --- /dev/null +++ b/pkg/postgresql/block_arrival.go @@ -0,0 +1,54 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + "time" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +var ( + CREATE_BLOCK_ARRIVAL_TABLE = ` + CREATE TABLE IF NOT EXISTS t_block_metrics( + f_slot INT, + f_label VARCHAR(100), + f_timestamp TIME, + CONSTRAINT PK_SlotAddr PRIMARY KEY (f_slot,f_label));` + + InsertNewBlock = ` + INSERT INTO t_score_metrics ( + f_slot, + f_label, + f_timestamp) + VALUES ($1, $2, $3);` +) + +// in case the table did not exist +func (p *PostgresDBService) createBlockMetricsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, CREATE_BLOCK_ARRIVAL_TABLE) + if err != nil { + return errors.Wrap(err, "error creating score metrics table") + } + return nil +} + +func (p *PostgresDBService) InsertNewBlock(slot int, label string, timestamp time.Time) error { + + _, err := p.psqlPool.Exec(p.ctx, InsertNewBlock, + slot, + label, + timestamp) + + if err != nil { + return errors.Wrap(err, "error inserting row in score metrics table") + } + return nil +} diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index ab5b836..76238f0 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -53,13 +53,13 @@ func (p *PostgresDBService) createScoreMetricsTable(ctx context.Context, pool *p return nil } -func (p *PostgresDBService) InsertNewScore(slot int, label string, score float64, duration float64, blockMetrics BlockMetricsModel) error { +func (p *PostgresDBService) InsertNewScore(blockMetrics BlockMetricsModel) error { _, err := p.psqlPool.Exec(p.ctx, InsertNewScore, - slot, - label, - score, - duration, + blockMetrics.Slot, + blockMetrics.Label, + blockMetrics.Score, + blockMetrics.Duration, blockMetrics.CorrectSource, blockMetrics.CorrectTarget, blockMetrics.CorrectHead, diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index c88df6c..84f6728 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -3,7 +3,11 @@ package postgresql import ( "context" "strings" + "sync" + "sync/atomic" + "time" + pgx_v4 "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -16,22 +20,26 @@ var ( log = logrus.WithField( "module", PsqlType, ) - MAX_BATCH_QUEUE = 300 + MAX_BATCH_QUEUE = 3000 + WRITE_CHAN_LENGTH = 400000 ) type PostgresDBService struct { // Control Variables - ctx context.Context - cancel context.CancelFunc - connectionUrl string // the url might not be necessary (better to remove it?¿) - psqlPool *pgxpool.Pool - - // Network DB Model + ctx context.Context + cancel context.CancelFunc + connectionUrl string // the url might not be necessary (better to remove it?¿) + psqlPool *pgxpool.Pool + WriteChan chan WriteTask + doneTasks chan interface{} + endProcess int32 + FinishSignalChan chan struct{} + workerNum int } // Connect to the PostgreSQL Database and get the multithread-proof connection // from the given url-composed credentials -func ConnectToDB(ctx context.Context, url string) (*PostgresDBService, error) { +func ConnectToDB(ctx context.Context, url string, workerNum int) (*PostgresDBService, error) { mainCtx, cancel := context.WithCancel(ctx) // spliting the url to don't share any confidential information on logs @@ -48,16 +56,22 @@ func ConnectToDB(ctx context.Context, url string) (*PostgresDBService, error) { // filter the type of network that we are filtering psqlDB := &PostgresDBService{ - ctx: mainCtx, - cancel: cancel, - connectionUrl: url, - psqlPool: psqlPool, + ctx: mainCtx, + cancel: cancel, + connectionUrl: url, + psqlPool: psqlPool, + WriteChan: make(chan WriteTask, WRITE_CHAN_LENGTH*workerNum), + doneTasks: make(chan interface{}, 1), + endProcess: 0, + FinishSignalChan: make(chan struct{}, 1), + workerNum: workerNum, } // init the psql db err = psqlDB.init(ctx, psqlDB.psqlPool) if err != nil { return psqlDB, errors.Wrap(err, "error initializing the tables of the psqldb") } + go psqlDB.runWriters() return psqlDB, err } @@ -72,5 +86,95 @@ func (p *PostgresDBService) init(ctx context.Context, pool *pgxpool.Pool) error return err } + err = p.createAttMetricsTable(ctx, pool) + if err != nil { + return err + } + return nil } + +func (p *PostgresDBService) DoneTasks() { + atomic.AddInt32(&p.endProcess, int32(1)) + log.Infof("Received finish signal") +} + +func (p *PostgresDBService) runWriters() { + var wgDBWriters sync.WaitGroup + log.Info("Launching Beacon State Writers") + log.Infof("Launching %d Beacon State Writers", p.workerNum) + for i := 0; i < p.workerNum; i++ { + wgDBWriters.Add(1) + go func(dbWriterID int) { + defer wgDBWriters.Done() + wlogWriter := log.WithField("DBWriter", dbWriterID) + writeBatch := pgx_v4.Batch{} + loop: + for { + + if p.endProcess >= 1 && len(p.WriteChan) == 0 { + wlogWriter.Info("finish detected, closing persister") + break loop + } + + select { + case task := <-p.WriteChan: + wlogWriter.Tracef("Received new write task") + writeBatch.Queue(task.QueryString, task.Params...) + + if writeBatch.Len() > MAX_BATCH_QUEUE { + wlogWriter.Debugf("Writing batch to database") + err := p.ExecuteBatch(writeBatch) + if err != nil { + wlogWriter.Errorf("Error processing batch", err.Error()) + } + writeBatch = pgx_v4.Batch{} + } + + case <-p.ctx.Done(): + wlogWriter.Info("shutdown detected, closing persister") + break loop + default: + } + + } + wlogWriter.Debugf("DB Writer finished...") + + }(i) + } + + wgDBWriters.Wait() + +} + +type WriteTask struct { + QueryString string + Params []interface{} +} + +func (p PostgresDBService) ExecuteBatch(batch pgx_v4.Batch) error { + + snapshot := time.Now() + tx, err := p.psqlPool.Begin(p.ctx) + if err != nil { + panic(err) + } + + batchResults := tx.SendBatch(p.ctx, &batch) + + var qerr error + var rows pgx_v4.Rows + for qerr == nil { + rows, qerr = batchResults.Query() + rows.Close() + } + if qerr.Error() != "no result" { + log.Errorf(qerr.Error()) + } + + // p.MonitorStruct.AddDBWrite(time.Since(snapshot).Seconds()) + log.Debugf("Batch process time: %f, batch size: %d", time.Since(snapshot).Seconds(), batch.Len()) + + return tx.Commit(p.ctx) + +} From 63c762276f590a3abccf68a578bc6ae9e3987e60 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 2 Nov 2022 13:52:04 +0100 Subject: [PATCH 12/62] Add logger utils --- pkg/utils/logger.go | 54 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 pkg/utils/logger.go diff --git a/pkg/utils/logger.go b/pkg/utils/logger.go new file mode 100644 index 0000000..53c1f13 --- /dev/null +++ b/pkg/utils/logger.go @@ -0,0 +1,54 @@ +package utils + +import ( + "io" + "os" + + "github.com/sirupsen/logrus" +) + +// App default configurations +var ( + ModName = "utils" + DefaultLoglvl = logrus.InfoLevel + DefaultLogOutput = os.Stdout + DefaultFormater = &logrus.TextFormatter{} +) + +// Select Log Level from string +func ParseLogLevel(lvl string) logrus.Level { + switch lvl { + case "trace": + return logrus.TraceLevel + case "debug": + return logrus.DebugLevel + case "info": + return logrus.InfoLevel + case "warn": + return logrus.WarnLevel + case "error": + return logrus.ErrorLevel + default: + return DefaultLoglvl + } +} + +// parse Formatter from string +func ParseLogOutput(lvl string) io.Writer { + switch lvl { + case "terminal": + return os.Stdout + default: + return DefaultLogOutput + } +} + +// parse Formatter from string +func ParseLogFormatter(lvl string) logrus.Formatter { + switch lvl { + case "text": + return &logrus.TextFormatter{} + default: + return DefaultFormater + } +} From 8573b1eec489744133fd928dca729aeb3edac52d Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 2 Nov 2022 13:53:01 +0100 Subject: [PATCH 13/62] Base repository that tracks blocks proposals and events (new blocks and attestations) --- pkg/analysis/additional_structs/epoch.go | 79 ++++++++++++++++++++ pkg/analysis/analyzer.go | 45 ++++++------ pkg/analysis/events.go | 77 ++++++++++++++++++++ pkg/analysis/history.go | 92 ++++++++++++++++++++++++ pkg/analysis/metrics.go | 37 ++++++---- pkg/app/service.go | 53 +++++++++++--- pkg/cmd/analyzer_cmd.go | 38 +++++++++- 7 files changed, 378 insertions(+), 43 deletions(-) create mode 100644 pkg/analysis/additional_structs/epoch.go create mode 100644 pkg/analysis/events.go create mode 100644 pkg/analysis/history.go diff --git a/pkg/analysis/additional_structs/epoch.go b/pkg/analysis/additional_structs/epoch.go new file mode 100644 index 0000000..8f7859a --- /dev/null +++ b/pkg/analysis/additional_structs/epoch.go @@ -0,0 +1,79 @@ +package additional_structs + +import ( + "context" + "fmt" + "strconv" + "sync" + + api "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/http" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/sirupsen/logrus" +) + +var ( + log = logrus.WithField( + "module", "Epoch Data") +) + +type EpochStructs struct { + mu sync.Mutex + Api *http.Service + CurrentBeaconCommittees []*api.BeaconCommittee + CurrentEpoch uint64 + PreviousBeaconCommittees []*api.BeaconCommittee + PreviousEpoch uint64 +} + +func NewEpochData(iApi *http.Service) EpochStructs { + + return EpochStructs{ + Api: iApi, + CurrentBeaconCommittees: make([]*api.BeaconCommittee, 0), + CurrentEpoch: 0, + PreviousBeaconCommittees: make([]*api.BeaconCommittee, 0), + PreviousEpoch: 0, + } +} + +func (e *EpochStructs) RequestNewBeaconCommittee(slot uint64) error { + epochCommittees, err := e.Api.BeaconCommittees(context.Background(), strconv.Itoa(int(slot))) + + if err != nil { + return fmt.Errorf("could not request beacon committees for epoch %d: %s", int(slot/32), err) + } + + // keep in mind we can only receive attestations to 32 blocks before + e.PreviousBeaconCommittees = e.CurrentBeaconCommittees + e.PreviousEpoch = e.CurrentEpoch + e.CurrentBeaconCommittees = epochCommittees + e.CurrentEpoch = uint64(slot / 32) + + return nil + +} + +func (e *EpochStructs) GetBeaconCommittee(slot uint64, index uint64) []phase0.ValidatorIndex { + log := log.WithField("routine", "epoch-structs") + e.mu.Lock() + // if the epoch requested is newer than the data we have + if slot/32 > e.CurrentEpoch { + log.Debugf("Requesting new beacon committee for %d", slot/32) + e.RequestNewBeaconCommittee(slot) + } + e.mu.Unlock() + + committeeList := e.PreviousBeaconCommittees + if slot/32 == e.CurrentEpoch { + committeeList = e.CurrentBeaconCommittees + } + + for _, item := range committeeList { + if item.Slot == phase0.Slot(slot) && item.Index == phase0.CommitteeIndex(index) { + return item.Validators + } + } + + return nil +} diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 26c0f0c..74d6d79 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -6,9 +6,12 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/jackc/pgx/v4" "github.com/prysmaticlabs/go-bitfield" "github.com/sirupsen/logrus" + "github.com/tdahar/block-scorer/pkg/analysis/additional_structs" "github.com/tdahar/block-scorer/pkg/client_api" + "github.com/tdahar/block-scorer/pkg/postgresql" ) var ( @@ -17,36 +20,45 @@ var ( "module", moduleName) ) -type BlockAnalyzer struct { +type ClientLiveData struct { ctx context.Context Eth2Provider client_api.APIClient // connection to the beacon node AttHistory map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist // 32 slots of attestation per slot and committeeIndex BlockRootHistory map[phase0.Slot]phase0.Root // 64 slots of roots log *logrus.Entry // each analyzer has its own logger + ProcessNewHead chan struct{} + DBClient *postgresql.PostgresDBService + writeBatch pgx.Batch + EpochData additional_structs.EpochStructs } -func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, timeout time.Duration) (*BlockAnalyzer, error) { +func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, timeout time.Duration, dbClient *postgresql.PostgresDBService) (*ClientLiveData, error) { client, err := client_api.NewAPIClient(ctx, label, cliEndpoint, timeout) if err != nil { log.Errorf("could not create eth2 client: %s", err) - return &BlockAnalyzer{}, err + return &ClientLiveData{}, err } - return &BlockAnalyzer{ + return &ClientLiveData{ ctx: ctx, Eth2Provider: *client, + DBClient: dbClient, AttHistory: make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist), BlockRootHistory: make(map[phase0.Slot]phase0.Root), log: log.WithField("label", label), + writeBatch: pgx.Batch{}, + EpochData: additional_structs.NewEpochData(client.Api), }, nil } -func (b *BlockAnalyzer) ProcessNewBlock(slot phase0.Slot) error { - +// Asks for a block proposal to the client and stores score in the database +func (b *ClientLiveData) ProcessNewBlock(slot phase0.Slot) error { + log := b.log.WithField("task", "generate-block") + log.Debugf("processing new block: %d\n", slot) randaoReveal := phase0.BLSSignature{} graffiti := []byte("") snapshot := time.Now() - block, err := b.Eth2Provider.Api.BeaconBlockProposal(b.ctx, slot, randaoReveal, graffiti) - blockTime := time.Since(snapshot).Seconds() // time to ask for block + block, err := b.Eth2Provider.Api.BeaconBlockProposal(b.ctx, slot, randaoReveal, graffiti) // ask for block proposal + blockTime := time.Since(snapshot).Seconds() // time to generate block if err != nil { return fmt.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) @@ -68,18 +80,11 @@ func (b *BlockAnalyzer) ProcessNewBlock(slot phase0.Slot) error { if err != nil { return fmt.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) } - b.log.Infof("Block Duration: %f", blockTime) - b.log.Infof("Metrics: %+v", metrics) + log.Infof("Block Generation Time: %f", blockTime) + log.Infof("Metrics: %+v", metrics) + b.DBClient.InsertNewScore(metrics) + // We block the update attestations as new head could impact attestations of the proposed block + b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations return nil } - -type BeaconBlockMetrics struct { - AttScore float64 - SyncScore float64 - AttestationNum int64 - NewVotes int64 - CorrectSource int64 - CorrectTarget int64 - CorrectHead int64 -} diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go new file mode 100644 index 0000000..30d7c07 --- /dev/null +++ b/pkg/analysis/events.go @@ -0,0 +1,77 @@ +package analysis + +import ( + "encoding/hex" + "time" + + api "github.com/attestantio/go-eth2-client/api/v1" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/tdahar/block-scorer/pkg/postgresql" +) + +func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { + timestamp := time.Now() + log := b.log.WithField("routine", "head-event") + + if event.Data == nil { + return + } + + data := event.Data.(*api.HeadEvent) + log.Debugf("Received a new head event") + + newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, hex.EncodeToString(data.Block[:])) + b.DBClient.InsertNewBlock(int(data.Slot), b.Eth2Provider.Label, timestamp) + <-b.ProcessNewHead + b.UpdateAttestations(*newBlock.Bellatrix.Message) + + if err != nil { + log.Errorf("could not retrieve the head block: %s", err) + } + +} + +func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { + timestamp := time.Now() + + log := b.log.WithField("routine", "attestation-event") + log.Debugf("Received a new event") + + if event.Data == nil { + log.Errorf("attestation event does not contain anything") + } + + data := event.Data.(*phase0.Attestation) + + log.Debugf("Initiating processing event in %f seconds", time.Since(timestamp).Seconds()) + beaconCommittee := b.EpochData.GetBeaconCommittee(uint64(data.Data.Slot), uint64(data.Data.Index)) + + if beaconCommittee == nil { + log.Errorf("could not retrieve beacon committee at slot %d", uint64(data.Data.Slot)) + } + attestingVals := make([]phase0.ValidatorIndex, 0) + + for _, bit := range data.AggregationBits.BitIndices() { + attestingVals = append(attestingVals, beaconCommittee[bit]) + } + log.Debugf("Initiating writing tasks event in %f seconds", time.Since(timestamp).Seconds()) + // create params to be written + baseParams := make([]interface{}, 0) + baseParams = append(baseParams, b.Eth2Provider.Label) + baseParams = append(baseParams, uint64(data.Data.Slot)) + baseParams = append(baseParams, uint64(data.Data.Index)) + baseParams = append(baseParams, timestamp) + + // for each attesting validator + for _, item := range attestingVals { + params := append(baseParams, uint64(item)) + writeTask := postgresql.WriteTask{ + QueryString: postgresql.InsertNewAtt, + Params: params, + } + b.DBClient.WriteChan <- writeTask // send task to be written + } + + log.Debugf("Finished processing event in %f seconds", time.Since(timestamp).Seconds()) + +} diff --git a/pkg/analysis/history.go b/pkg/analysis/history.go new file mode 100644 index 0000000..1d7758e --- /dev/null +++ b/pkg/analysis/history.go @@ -0,0 +1,92 @@ +package analysis + +import ( + "fmt" + "strings" + + "github.com/attestantio/go-eth2-client/spec/bellatrix" + "github.com/attestantio/go-eth2-client/spec/phase0" + "github.com/prysmaticlabs/go-bitfield" +) + +func (b *ClientLiveData) UpdateAttestations(block bellatrix.BeaconBlock) { + + for _, attestation := range block.Body.Attestations { + slot := attestation.Data.Slot + + if _, exists := b.AttHistory[slot]; !exists { + // add slot to map + b.AttHistory[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) + } + + committeIndex := attestation.Data.Index + if _, exists := b.AttHistory[slot][committeIndex]; !exists { + b.AttHistory[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) + } + + attestingIndices := attestation.AggregationBits.BitIndices() + + for _, idx := range attestingIndices { + if b.AttHistory[slot][committeIndex].BitAt(uint64(idx)) { + // already registered vote + continue + } + b.AttHistory[slot][committeIndex].SetBitAt(uint64(idx), true) + } + + } + +} + +func (b *ClientLiveData) BuildHistory() bool { + log := b.log.WithField("routine", "history-build") + currentHead, err := b.Eth2Provider.Api.BeaconBlockHeader(b.ctx, "head") + headSlot := currentHead.Header.Message.Slot + + if err != nil { + log.Panicf("could not retrieve current head: ", err) + + } + + if _, ok := b.BlockRootHistory[currentHead.Header.Message.Slot]; ok { + // at this point we have already filled the historical records + return true + } + + // at this point there are no historical records + + for i := headSlot; i >= headSlot-64; i-- { + if _, ok := b.BlockRootHistory[i]; ok { + // at this point we have already filled the historical records + return false // but we had to fill something + } + + log.Debugf("filling block history, slot: %d\n", i) + block, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, fmt.Sprintf("%d", i)) + + if err != nil { + if strings.Contains(err.Error(), "Could not find requested block") { + log.Debugf("Missed block!") + continue + } else { + log.Panicf("could not retrieve historical block at slot: %d: ", i, err) + } + } + if block == nil { + log.Debugf("Unknown error with block %d\n", i) + continue + } + root, err := block.Root() + if err != nil { + log.Panicf("could not retrieve block root from block %d: ", i, err) + } + b.BlockRootHistory[i] = root + + if i+32 >= headSlot { + b.UpdateAttestations(*block.Bellatrix.Message) + } + + } + + return false +} diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 3a3d9b3..f5f0214 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -4,6 +4,7 @@ import ( "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/go-bitfield" + "github.com/tdahar/block-scorer/pkg/postgresql" "github.com/tdahar/block-scorer/pkg/utils" ) @@ -16,33 +17,38 @@ const ( SYNC_REWARD_WEIGHT = 2 ) -func (b *BlockAnalyzer) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (BeaconBlockMetrics, error) { - // log = b.log.WithField("method", "bellatrix-block") // add extra log for function +func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (postgresql.BlockMetricsModel, error) { + // log := b.log.WithField("task", "bellatrix-block-score") // add extra log for function totalNewVotes := 0 totalScore := 0 - + attested := make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist) // for current block for _, attestation := range block.Body.Attestations { newVotes := 0 slot := attestation.Data.Slot - if _, exists := b.AttHistory[slot]; !exists { + if _, exists := attested[slot]; !exists { // add slot to map - b.AttHistory[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) + attested[slot] = make(map[phase0.CommitteeIndex]bitfield.Bitlist) } committeIndex := attestation.Data.Index - if _, exists := b.AttHistory[slot][committeIndex]; !exists { - b.AttHistory[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) + if _, exists := attested[slot][committeIndex]; !exists { + attested[slot][committeIndex] = bitfield.NewBitlist(attestation.AggregationBits.Len()) } attestingIndices := attestation.AggregationBits.BitIndices() for _, idx := range attestingIndices { if b.AttHistory[slot][committeIndex].BitAt(uint64(idx)) { - // already registered vote + // already registered vote in a previous block + continue + } + if attested[slot][committeIndex].BitAt(uint64(idx)) { + // already registered vote in a same block continue } - b.AttHistory[slot][committeIndex].SetBitAt(uint64(idx), true) + // we do not touch the history, as this is just a proposed block, but we do not know if it will be included in the chain + attested[slot][committeIndex].SetBitAt(uint64(idx), true) // register as attested in the current block newVotes++ } score := 0 @@ -62,10 +68,15 @@ func (b *BlockAnalyzer) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (Bea } - syncCommitteeScore := float64(block.Body.SyncAggregate.SyncCommitteeBits.Count()) * float64(SYNC_REWARD_WEIGHT) / float64(WEIGHT_DENOMINATOR) + // syncCommitteeScore := float64(block.Body.SyncAggregate.SyncCommitteeBits.Count()) * float64(SYNC_REWARD_WEIGHT) / float64(WEIGHT_DENOMINATOR) - return BeaconBlockMetrics{ - AttScore: float64(totalScore), - SyncScore: float64(syncCommitteeScore), + return postgresql.BlockMetricsModel{ + Slot: int(block.Slot), + Label: b.Eth2Provider.Label, + CorrectSource: 0, + CorrectTarget: 0, + CorrectHead: 0, + Score: float64(totalScore), + NewVotes: totalNewVotes, }, nil } diff --git a/pkg/app/service.go b/pkg/app/service.go index b48a8c4..e8065c1 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -10,6 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/tdahar/block-scorer/pkg/analysis" "github.com/tdahar/block-scorer/pkg/chain_stats" + "github.com/tdahar/block-scorer/pkg/postgresql" ) var ( @@ -21,24 +22,32 @@ var ( type AppService struct { ctx context.Context - Analyzers []*analysis.BlockAnalyzer + Analyzers []*analysis.ClientLiveData initTime time.Time ChainTime chain_stats.ChainTime HeadSlot phase0.Slot } -func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, error) { +func NewAppService(ctx context.Context, bnEndpoints []string, dbEndpooint string, dbWorkers int) (*AppService, error) { - analyzers := make([]*analysis.BlockAnalyzer, 0) // one analyzer per beacon node + dbClient, err := postgresql.ConnectToDB(ctx, dbEndpooint, dbWorkers) + + if err != nil { + log.Panicf("could not connect to database: %s", err) + } + + analyzers := make([]*analysis.ClientLiveData, 0) // one analyzer per beacon node for i := range bnEndpoints { // parse each beacon node endpoint if !strings.Contains(bnEndpoints[i], "/") { log.Errorf("incorrect format for endpoint: %s", bnEndpoints[i]) } - label := strings.Split(bnEndpoints[i], "/")[0] + // label := strings.Split(bnEndpoints[i], "/")[0] endpoint := strings.Split(bnEndpoints[i], "/")[1] - newAnalyzer, err := analysis.NewBlockAnalyzer(ctx, label, endpoint, time.Second*5) + // newAnalyzer, err := analysis.NewBlockAnalyzer(ctx, label, endpoint, time.Second*5) + newAnalyzer, err := analysis.NewBlockAnalyzer(ctx, bnEndpoints[i], endpoint, time.Second*5, dbClient) + if err != nil { log.Errorf("could not create client for endpoint: %s ", endpoint, err) continue @@ -46,6 +55,7 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro analyzers = append(analyzers, newAnalyzer) } // get genesis time to calculate each slot time + // Keep in mind first endpoint will be used as master genesis, err := analyzers[0].Eth2Provider.Api.GenesisTime(ctx) if err != nil { return nil, fmt.Errorf("could not obtain genesis time: %s", err) @@ -55,21 +65,46 @@ func NewAppService(ctx context.Context, bnEndpoints []string) (*AppService, erro if err != nil { return nil, fmt.Errorf("could not obtain head block header: %s", err) } - return &AppService{ ctx: ctx, Analyzers: analyzers, initTime: time.Now(), - HeadSlot: headHeader.Header.Message.Slot, // start 64 slots behind to create attestation history + HeadSlot: headHeader.Header.Message.Slot, ChainTime: chain_stats.ChainTime{ GenesisTime: genesis, }, }, nil } -// Main routine +// Main routine: build block history and block proposals every 12 seconds func (s *AppService) Run() { log = log.WithField("routine", "main") + historyBuilt := false + + for !historyBuilt { + historyBuilt = true + for _, item := range s.Analyzers { + ok := item.BuildHistory() + if !ok { + historyBuilt = false + } + } + } + + // Subscribe to events from each client + for _, item := range s.Analyzers { + err := item.Eth2Provider.Api.Events(s.ctx, []string{"head"}, item.HandleHeadEvent) // every new head + if err != nil { + log.Panicf("failed to subscribe to head events: %s, label: %s", err, item.Eth2Provider.Label) + } + + // every new attestation + err = item.Eth2Provider.Api.Events(s.ctx, []string{"attestation"}, item.HandleAttestationEvent) // every new head + if err != nil { + log.Panicf("failed to subscribe to attestation events: %s, label: %s", err, item.Eth2Provider.Label) + } + + } // tick every slot start (12 seconds) ticker := time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) @@ -89,11 +124,11 @@ func (s *AppService) Run() { ticker = time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) // a new slot has begun, therefore execute all needed actions log.Tracef("Time until next slot tick: %s", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1))).String()) - for _, analyzer := range s.Analyzers { // for each beacon node, get a new block and analyze it go analyzer.ProcessNewBlock(s.HeadSlot) } + default: } } diff --git a/pkg/cmd/analyzer_cmd.go b/pkg/cmd/analyzer_cmd.go index 5ddb5c0..f15ed71 100644 --- a/pkg/cmd/analyzer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -5,8 +5,10 @@ import ( "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/tdahar/block-scorer/pkg/app" + "github.com/tdahar/block-scorer/pkg/utils" cli "github.com/urfave/cli/v2" ) @@ -15,28 +17,62 @@ var AnalyzerCommand = &cli.Command{ Usage: "Receive Block proposals from clients and evaluate score", Action: LaunchBlockAnalyzer, Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "log-level", + Usage: "info,debug,warn", + DefaultText: "info", + }, &cli.StringFlag{ Name: "bn-endpoints", Usage: "beacon node endpoints (label/endpoint,label/endpoint)", DefaultText: "lh/localhost:5052", + }, + &cli.StringFlag{ + Name: "db-endpoint", + Usage: "postgresql database endpoint: postgresql://user:password@localhost:5432/beaconchain", + DefaultText: "lh/localhost:5052", + }, + &cli.StringFlag{ + Name: "db-workers", + Usage: "10", + DefaultText: "1", }}, } var QueryTimeout = 90 * time.Second func LaunchBlockAnalyzer(c *cli.Context) error { + dbWorkers := 1 logLauncher := log.WithField( "module", "ScorerCommand", ) + logLauncher.Info("parsing flags") + if !c.IsSet("log-level") { + logLauncher.Infof("Setting log level to Info") + logrus.SetLevel(logrus.InfoLevel) + } else { + logrus.SetLevel(utils.ParseLogLevel(c.String("log-level"))) + } // check if a beacon node is set if !c.IsSet("bn-endpoints") { return errors.New("bn endpoint not provided") } + if !c.IsSet("db-endpoint") { + return errors.New("db endpoint not provided") + } + + if !c.IsSet("db-workers") { + logLauncher.Warnf("no database workers configured, default is 1") + } else { + dbWorkers = c.Int("db-workers") + } + bnEndpoints := strings.Split(c.String("bn-endpoints"), ",") + dbEndpoint := c.String("db-endpoint") - service, err := app.NewAppService(c.Context, bnEndpoints) + service, err := app.NewAppService(c.Context, bnEndpoints, dbEndpoint, dbWorkers) if err != nil { log.Fatal("could not start app: %s", err.Error()) } From 54cd9d955c0f7606c63aa8996a72d09e6a0edd01 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 12:40:56 +0100 Subject: [PATCH 14/62] Add support for write channel --- go-eth2-client | 2 +- go.mod | 6 ++-- go.sum | 2 ++ pkg/postgresql/attestation_arrival.go | 32 +++++++------------- pkg/postgresql/block_arrival.go | 14 --------- pkg/postgresql/block_score.go | 42 ++++++++------------------- pkg/postgresql/missed_blocks.go | 38 ++++++++++++++++++++++++ pkg/postgresql/service.go | 12 +++++++- 8 files changed, 79 insertions(+), 69 deletions(-) create mode 100644 pkg/postgresql/missed_blocks.go diff --git a/go-eth2-client b/go-eth2-client index 44a58a6..f02f942 160000 --- a/go-eth2-client +++ b/go-eth2-client @@ -1 +1 @@ -Subproject commit 44a58a628ad5dc63ed6a876b9ed923802144c0c0 +Subproject commit f02f9429fcbdfbd5c4aa29857d5c58108dddfee8 diff --git a/go.mod b/go.mod index 9d13c27..22e55b1 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/urfave/cli/v2 v2.16.3 ) +require github.com/jackc/pgx v3.6.2+incompatible // indirect + require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.13.0 // indirect @@ -22,14 +24,14 @@ require ( github.com/jackc/pgproto3/v2 v2.3.1 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.12.0 // indirect - github.com/jackc/pgx/v4 v4.17.2 // indirect + github.com/jackc/pgx/v4 v4.17.2 github.com/jackc/puddle v1.3.0 // indirect github.com/klauspost/cpuid/v2 v2.0.11 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect + github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 github.com/r3labs/sse/v2 v2.7.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect diff --git a/go.sum b/go.sum index 7673a97..5c4c958 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,8 @@ github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrU github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= diff --git a/pkg/postgresql/attestation_arrival.go b/pkg/postgresql/attestation_arrival.go index 51cfda7..a3b91dc 100644 --- a/pkg/postgresql/attestation_arrival.go +++ b/pkg/postgresql/attestation_arrival.go @@ -8,7 +8,6 @@ This file together with the model, has all the needed methods to interact with t import ( "context" - "time" "github.com/jackc/pgx/v4/pgxpool" "github.com/pkg/errors" @@ -20,9 +19,12 @@ var ( f_label VARCHAR(100), f_slot INT, f_committee_index INT, - f_val_idx INT, - f_timestamp TIME, - CONSTRAINT PK_Attestation PRIMARY KEY (f_label,f_slot,f_committee_index,f_val_idx));` + f_signature TEXT, + f_source_root TEXT, + f_target_root TEXT, + f_head_root TEXT, + f_timestamp TIMESTAMP, + CONSTRAINT PK_Attestation PRIMARY KEY (f_label,f_slot,f_committee_index,f_signature));` InsertNewAtt = ` INSERT INTO t_att_metrics ( @@ -30,10 +32,11 @@ var ( f_slot, f_committee_index, f_timestamp, - f_val_idx - ) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT DO NOTHING;` + f_signature, + f_source_root, + f_target_root, + f_head_root) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8);` ) // in case the table did not exist @@ -45,16 +48,3 @@ func (p *PostgresDBService) createAttMetricsTable(ctx context.Context, pool *pgx } return nil } - -func (p *PostgresDBService) InsertNewAtt(slot int, label string, timestamp time.Time) error { - - _, err := p.psqlPool.Exec(p.ctx, InsertNewAtt, - slot, - label, - timestamp) - - if err != nil { - return errors.Wrap(err, "error inserting row in score metrics table") - } - return nil -} diff --git a/pkg/postgresql/block_arrival.go b/pkg/postgresql/block_arrival.go index 84cd075..cdad354 100644 --- a/pkg/postgresql/block_arrival.go +++ b/pkg/postgresql/block_arrival.go @@ -8,7 +8,6 @@ This file together with the model, has all the needed methods to interact with t import ( "context" - "time" "github.com/jackc/pgx/v4/pgxpool" "github.com/pkg/errors" @@ -39,16 +38,3 @@ func (p *PostgresDBService) createBlockMetricsTable(ctx context.Context, pool *p } return nil } - -func (p *PostgresDBService) InsertNewBlock(slot int, label string, timestamp time.Time) error { - - _, err := p.psqlPool.Exec(p.ctx, InsertNewBlock, - slot, - label, - timestamp) - - if err != nil { - return errors.Wrap(err, "error inserting row in score metrics table") - } - return nil -} diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index 76238f0..2458119 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -53,35 +53,17 @@ func (p *PostgresDBService) createScoreMetricsTable(ctx context.Context, pool *p return nil } -func (p *PostgresDBService) InsertNewScore(blockMetrics BlockMetricsModel) error { - - _, err := p.psqlPool.Exec(p.ctx, InsertNewScore, - blockMetrics.Slot, - blockMetrics.Label, - blockMetrics.Score, - blockMetrics.Duration, - blockMetrics.CorrectSource, - blockMetrics.CorrectTarget, - blockMetrics.CorrectHead, - blockMetrics.Sync1Bits, - blockMetrics.AttNum, - blockMetrics.NewVotes) - - if err != nil { - return errors.Wrap(err, "error inserting row in score metrics table") - } - return nil -} - type BlockMetricsModel struct { - Slot int - Label string - Score float64 - Duration float64 - CorrectSource int - CorrectTarget int - CorrectHead int - Sync1Bits int - AttNum int - NewVotes int + Slot int + Label string + Score float64 + Duration float64 + CorrectSource int + CorrectTarget int + CorrectHead int + Sync1Bits int + AttNum int + NewVotes int + AttesterSlashings int + ProposerSlashings int } diff --git a/pkg/postgresql/missed_blocks.go b/pkg/postgresql/missed_blocks.go new file mode 100644 index 0000000..dfadf50 --- /dev/null +++ b/pkg/postgresql/missed_blocks.go @@ -0,0 +1,38 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +var ( + CREATE_MISSED_BLOCKS_TABLE = ` + CREATE TABLE IF NOT EXISTS t_missed_blocks( + f_slot INT, + f_label TEXT, + CONSTRAINT PK_SlotAddr PRIMARY KEY (f_slot,f_label));` + + InsertNewMissedBlock = ` + INSERT INTO t_missed_blocks ( + f_slot, + f_label) + VALUES ($1, $2);` +) + +// in case the table did not exist +func (p *PostgresDBService) createMissedBlockMetricsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, CREATE_MISSED_BLOCKS_TABLE) + if err != nil { + return errors.Wrap(err, "error creating missed blocks metrics table") + } + return nil +} diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index 84f6728..028e861 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -20,7 +20,7 @@ var ( log = logrus.WithField( "module", PsqlType, ) - MAX_BATCH_QUEUE = 3000 + MAX_BATCH_QUEUE = 100 WRITE_CHAN_LENGTH = 400000 ) @@ -86,6 +86,16 @@ func (p *PostgresDBService) init(ctx context.Context, pool *pgxpool.Pool) error return err } + err = p.createBlockMetricsTable(ctx, pool) + if err != nil { + return err + } + + err = p.createMissedBlockMetricsTable(ctx, pool) + if err != nil { + return err + } + err = p.createAttMetricsTable(ctx, pool) if err != nil { return err From 92961dd992631222961aba26a44a2c08158ec53c Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 12:42:03 +0100 Subject: [PATCH 15/62] Add using channels, track missing blocks and track only attestations --- pkg/analysis/events.go | 90 ++++++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 30d7c07..a3fce6a 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -17,13 +17,40 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { return } - data := event.Data.(*api.HeadEvent) - log.Debugf("Received a new head event") + data := event.Data.(*api.HeadEvent) // cast to head event + log.Debugf("Received a new head event: slot %d", data.Slot) + // we only receive the block hash, get the new block newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, hex.EncodeToString(data.Block[:])) - b.DBClient.InsertNewBlock(int(data.Slot), b.Eth2Provider.Label, timestamp) + + // Track if there is any missing slot + if b.CurrentHeadSlot != 0 && // we are not at the beginning of the run + newBlock.Bellatrix.Message.Slot-phase0.Slot(b.CurrentHeadSlot) > 1 { // there a gap bigger than 1 with the new head + for i := b.CurrentHeadSlot; i < uint64(newBlock.Bellatrix.Message.Slot); i++ { + params := make([]interface{}, 2) + params = append(params, i) + params = append(params, b.Eth2Provider.Label) + writeTask := postgresql.WriteTask{ + QueryString: postgresql.InsertNewMissedBlock, + Params: params, + } + b.DBClient.WriteChan <- writeTask // store + } + } + params := make([]interface{}, 3) + params = append(params, int(data.Slot)) + params = append(params, b.Eth2Provider.Label) + params = append(params, timestamp) + writeTask := postgresql.WriteTask{ + QueryString: postgresql.InsertNewBlock, + Params: params, + } + b.DBClient.WriteChan <- writeTask // store + + // wait for the block proposal to be processed, otherwise the attestations could get mixed + // with the proposal <-b.ProcessNewHead - b.UpdateAttestations(*newBlock.Bellatrix.Message) + b.UpdateAttestations(*newBlock.Bellatrix.Message) // now update the attestations with the new head block in the chain if err != nil { log.Errorf("could not retrieve the head block: %s", err) @@ -31,6 +58,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { } +// When the node receives a new attestation, this function is tiggered func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { timestamp := time.Now() @@ -41,37 +69,49 @@ func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { log.Errorf("attestation event does not contain anything") } - data := event.Data.(*phase0.Attestation) + data := event.Data.(*phase0.Attestation) // cast + // With the beacon committee we can identify the attesting validators + // Will not track this for now + // beaconCommittee := b.EpochData.GetBeaconCommittee(uint64(data.Data.Slot), uint64(data.Data.Index)) - log.Debugf("Initiating processing event in %f seconds", time.Since(timestamp).Seconds()) - beaconCommittee := b.EpochData.GetBeaconCommittee(uint64(data.Data.Slot), uint64(data.Data.Index)) + // if beaconCommittee == nil { + // log.Errorf("could not retrieve beacon committee at slot %d", uint64(data.Data.Slot)) + // return + // } + // attestingVals := make([]phase0.ValidatorIndex, 0) - if beaconCommittee == nil { - log.Errorf("could not retrieve beacon committee at slot %d", uint64(data.Data.Slot)) - } - attestingVals := make([]phase0.ValidatorIndex, 0) + // for _, bit := range data.AggregationBits.BitIndices() { + // attestingVals = append(attestingVals, beaconCommittee[bit]) + // } - for _, bit := range data.AggregationBits.BitIndices() { - attestingVals = append(attestingVals, beaconCommittee[bit]) - } - log.Debugf("Initiating writing tasks event in %f seconds", time.Since(timestamp).Seconds()) - // create params to be written + // create params to be written, same for all validators (same attestation) baseParams := make([]interface{}, 0) baseParams = append(baseParams, b.Eth2Provider.Label) baseParams = append(baseParams, uint64(data.Data.Slot)) baseParams = append(baseParams, uint64(data.Data.Index)) baseParams = append(baseParams, timestamp) - - // for each attesting validator - for _, item := range attestingVals { - params := append(baseParams, uint64(item)) - writeTask := postgresql.WriteTask{ - QueryString: postgresql.InsertNewAtt, - Params: params, - } - b.DBClient.WriteChan <- writeTask // send task to be written + baseParams = append(baseParams, hex.EncodeToString(data.Signature[:])) + baseParams = append(baseParams, hex.EncodeToString(data.Data.Source.Root[:])) + baseParams = append(baseParams, hex.EncodeToString(data.Data.Target.Root[:])) + baseParams = append(baseParams, hex.EncodeToString(data.Data.BeaconBlockRoot[:])) + + // for each attesting validator, not use for now + // for _, item := range attestingVals { + // params := append(baseParams, uint64(item)) // append the validator index + // writeTask := postgresql.WriteTask{ + // QueryString: postgresql.InsertNewAtt, + // Params: params, + // } + // b.DBClient.WriteChan <- writeTask // send task to be written + // } + + writeTask := postgresql.WriteTask{ + QueryString: postgresql.InsertNewAtt, + Params: baseParams, } + b.DBClient.WriteChan <- writeTask // send task to be written + log.Debugf("Finished processing event in %f seconds", time.Since(timestamp).Seconds()) } From 771bee61b799492c3ffc968dba0ec46f83566e9a Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 12:42:56 +0100 Subject: [PATCH 16/62] Use database write channel --- pkg/analysis/analyzer.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 74d6d79..ff76802 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -82,7 +82,25 @@ func (b *ClientLiveData) ProcessNewBlock(slot phase0.Slot) error { } log.Infof("Block Generation Time: %f", blockTime) log.Infof("Metrics: %+v", metrics) - b.DBClient.InsertNewScore(metrics) + + // Store in DB + params := make([]interface{}, 0) + params = append(params, metrics.Label) + params = append(params, metrics.Score) + params = append(params, metrics.Duration) + params = append(params, metrics.CorrectSource) + params = append(params, metrics.CorrectTarget) + params = append(params, metrics.CorrectHead) + params = append(params, metrics.Sync1Bits) + params = append(params, metrics.AttNum) + params = append(params, metrics.NewVotes) + + writeTask := postgresql.WriteTask{ + QueryString: postgresql.InsertNewScore, + Params: params, + } + + b.DBClient.WriteChan <- writeTask // We block the update attestations as new head could impact attestations of the proposed block b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations From bd96348ccf14eb70cc47494f673c7e7b00690399 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 12:43:31 +0100 Subject: [PATCH 17/62] Add comments and change method name --- pkg/analysis/analyzer.go | 9 +++++---- pkg/analysis/history.go | 4 ++++ pkg/app/service.go | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index ff76802..daa0c20 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -6,7 +6,6 @@ import ( "time" "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/jackc/pgx/v4" "github.com/prysmaticlabs/go-bitfield" "github.com/sirupsen/logrus" "github.com/tdahar/block-scorer/pkg/analysis/additional_structs" @@ -28,8 +27,8 @@ type ClientLiveData struct { log *logrus.Entry // each analyzer has its own logger ProcessNewHead chan struct{} DBClient *postgresql.PostgresDBService - writeBatch pgx.Batch EpochData additional_structs.EpochStructs + CurrentHeadSlot uint64 } func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, timeout time.Duration, dbClient *postgresql.PostgresDBService) (*ClientLiveData, error) { @@ -45,20 +44,22 @@ func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, tim AttHistory: make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist), BlockRootHistory: make(map[phase0.Slot]phase0.Root), log: log.WithField("label", label), - writeBatch: pgx.Batch{}, EpochData: additional_structs.NewEpochData(client.Api), + CurrentHeadSlot: 0, }, nil } // Asks for a block proposal to the client and stores score in the database -func (b *ClientLiveData) ProcessNewBlock(slot phase0.Slot) error { +func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { log := b.log.WithField("task", "generate-block") log.Debugf("processing new block: %d\n", slot) + randaoReveal := phase0.BLSSignature{} graffiti := []byte("") snapshot := time.Now() block, err := b.Eth2Provider.Api.BeaconBlockProposal(b.ctx, slot, randaoReveal, graffiti) // ask for block proposal blockTime := time.Since(snapshot).Seconds() // time to generate block + if err != nil { return fmt.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) diff --git a/pkg/analysis/history.go b/pkg/analysis/history.go index 1d7758e..c03f0b0 100644 --- a/pkg/analysis/history.go +++ b/pkg/analysis/history.go @@ -9,6 +9,8 @@ import ( "github.com/prysmaticlabs/go-bitfield" ) +// This method receives a new head block and updates the attestation in the history +// So, when a block is to be proposed, we can check the history to identify new votes func (b *ClientLiveData) UpdateAttestations(block bellatrix.BeaconBlock) { for _, attestation := range block.Body.Attestations { @@ -38,6 +40,8 @@ func (b *ClientLiveData) UpdateAttestations(block bellatrix.BeaconBlock) { } +// This function is only called at the beginning of the run, so we build an initial attestation history +// to judge new block proposals func (b *ClientLiveData) BuildHistory() bool { log := b.log.WithField("routine", "history-build") currentHead, err := b.Eth2Provider.Api.BeaconBlockHeader(b.ctx, "head") diff --git a/pkg/app/service.go b/pkg/app/service.go index e8065c1..7620847 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -126,7 +126,7 @@ func (s *AppService) Run() { log.Tracef("Time until next slot tick: %s", time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot+1))).String()) for _, analyzer := range s.Analyzers { // for each beacon node, get a new block and analyze it - go analyzer.ProcessNewBlock(s.HeadSlot) + go analyzer.ProposeNewBlock(s.HeadSlot) } default: From 96f812e3fafd46865a4b83ff7ccfc3b5f14c9fa6 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 12:43:48 +0100 Subject: [PATCH 18/62] Add more metrics --- pkg/analysis/metrics.go | 85 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index f5f0214..69fef53 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -1,6 +1,8 @@ package analysis import ( + "sort" + "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/go-bitfield" @@ -17,10 +19,16 @@ const ( SYNC_REWARD_WEIGHT = 2 ) +// https://github.com/attestantio/vouch/blob/0c75ee8315dc4e5df85eb2aa09b4acc2b4436661/strategies/beaconblockproposal/best/score.go#L222 +// This function receives a new block proposal and ouputs a block score and metrics about the block func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (postgresql.BlockMetricsModel, error) { // log := b.log.WithField("task", "bellatrix-block-score") // add extra log for function totalNewVotes := 0 totalScore := 0 + attScore := 0 + totalCorrectSource := 0 + totalCorrectTarget := 0 + totalCorrectHead := 0 attested := make(map[phase0.Slot]map[phase0.CommitteeIndex]bitfield.Bitlist) // for current block for _, attestation := range block.Body.Attestations { newVotes := 0 @@ -54,29 +62,88 @@ func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (po score := 0 if utils.IsCorrectSource(*attestation, *block) { score += newVotes * TIMELY_SOURCE_WEIGHT + totalCorrectSource += newVotes } if utils.IsCorrectTarget(*attestation, *block, b.BlockRootHistory) { score += newVotes * TIMELY_TARGET_WEIGHT + totalCorrectTarget += newVotes } if utils.IsCorrectHead(*attestation, *block) { score += newVotes * TIMELY_HEAD_WEIGHT + totalCorrectHead += newVotes } totalNewVotes += newVotes // denominator := (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) * WEIGHT_DENOMINATOR / PROPOSER_WEIGHT - totalScore += score / WEIGHT_DENOMINATOR + attScore += score / WEIGHT_DENOMINATOR } - // syncCommitteeScore := float64(block.Body.SyncAggregate.SyncCommitteeBits.Count()) * float64(SYNC_REWARD_WEIGHT) / float64(WEIGHT_DENOMINATOR) + syncCommitteeScore := float64(block.Body.SyncAggregate.SyncCommitteeBits.Count()) * float64(SYNC_REWARD_WEIGHT) / float64(WEIGHT_DENOMINATOR) + + attesterSlashingScore, proposerSlashingScore := scoreSlashings(block.Body.AttesterSlashings, block.Body.ProposerSlashings) + + totalScore = attScore + int(syncCommitteeScore) + int(attesterSlashingScore) + int(proposerSlashingScore) return postgresql.BlockMetricsModel{ - Slot: int(block.Slot), - Label: b.Eth2Provider.Label, - CorrectSource: 0, - CorrectTarget: 0, - CorrectHead: 0, - Score: float64(totalScore), - NewVotes: totalNewVotes, + Slot: int(block.Slot), + Label: b.Eth2Provider.Label, + CorrectSource: totalCorrectSource, + CorrectTarget: totalCorrectTarget, + CorrectHead: totalCorrectHead, + Score: float64(totalScore), + NewVotes: totalNewVotes, + AttNum: len(block.Body.Attestations), + Sync1Bits: int(block.Body.SyncAggregate.SyncCommitteeBits.Count()), + AttesterSlashings: len(block.Body.AttesterSlashings), + ProposerSlashings: len(block.Body.ProposerSlashings), }, nil } + +// https://github.com/attestantio/vouch/blob/0c75ee8315dc4e5df85eb2aa09b4acc2b4436661/strategies/beaconblockproposal/best/score.go#L312 +func scoreSlashings(attesterSlashings []*phase0.AttesterSlashing, + proposerSlashings []*phase0.ProposerSlashing, +) (float64, float64) { + // Slashing reward will be at most MAX_EFFECTIVE_BALANCE/WHISTLEBLOWER_REWARD_QUOTIENT, + // which is 0.0625 Ether. + // Individual attestation reward at 250K validators will be around 23,000 GWei, or .000023 Ether. + // So we state that a single slashing event has the same weight as about 2,700 attestations. + slashingWeight := float64(2700) + + // Add proposer slashing scores. + proposerSlashingScore := float64(len(proposerSlashings)) * slashingWeight + + // Add attester slashing scores. + indicesSlashed := 0 + for _, slashing := range attesterSlashings { + indicesSlashed += len(intersection(slashing.Attestation1.AttestingIndices, slashing.Attestation2.AttestingIndices)) + } + attesterSlashingScore := slashingWeight * float64(indicesSlashed) + + return attesterSlashingScore, proposerSlashingScore +} + +// https://github.com/attestantio/vouch/blob/0c75ee8315dc4e5df85eb2aa09b4acc2b4436661/strategies/beaconblockproposal/best/score.go#L426 +// intersection returns a list of items common between the two sets. +func intersection(set1 []uint64, set2 []uint64) []uint64 { + sort.Slice(set1, func(i, j int) bool { return set1[i] < set1[j] }) + sort.Slice(set2, func(i, j int) bool { return set2[i] < set2[j] }) + res := make([]uint64, 0) + + set1Pos := 0 + set2Pos := 0 + for set1Pos < len(set1) && set2Pos < len(set2) { + switch { + case set1[set1Pos] < set2[set2Pos]: + set1Pos++ + case set2[set2Pos] < set1[set1Pos]: + set2Pos++ + default: + res = append(res, set1[set1Pos]) + set1Pos++ + set2Pos++ + } + } + + return res +} From f7ac1c36eb85ef8c22988fb439f692853edc3b01 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 12:44:04 +0100 Subject: [PATCH 19/62] Change binary file as per repository name change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1b0e872..94de68f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ MKDIR_P=mkdir -p GIT_SUBM=git submodule BIN_PATH=./build -BIN="./build/block-scorer" +BIN="./build/eth_cl_live_metrics" .PHONY: check dependencies build install clean From 9270a75249cd8bbba523dbfd9c716645201bac64 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 14:39:36 +0100 Subject: [PATCH 20/62] Changed constraint names to avoid collision --- pkg/postgresql/block_arrival.go | 4 ++-- pkg/postgresql/block_score.go | 2 +- pkg/postgresql/missed_blocks.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/postgresql/block_arrival.go b/pkg/postgresql/block_arrival.go index cdad354..e937a33 100644 --- a/pkg/postgresql/block_arrival.go +++ b/pkg/postgresql/block_arrival.go @@ -19,7 +19,7 @@ var ( f_slot INT, f_label VARCHAR(100), f_timestamp TIME, - CONSTRAINT PK_SlotAddr PRIMARY KEY (f_slot,f_label));` + CONSTRAINT PK_Block PRIMARY KEY (f_slot,f_label));` InsertNewBlock = ` INSERT INTO t_score_metrics ( @@ -34,7 +34,7 @@ func (p *PostgresDBService) createBlockMetricsTable(ctx context.Context, pool *p // create the tables _, err := pool.Exec(ctx, CREATE_BLOCK_ARRIVAL_TABLE) if err != nil { - return errors.Wrap(err, "error creating score metrics table") + return errors.Wrap(err, "error creating block arrival metrics table") } return nil } diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index 2458119..912093c 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -26,7 +26,7 @@ var ( f_sync_bits INT, f_att_num INT, f_new_votes INT, - CONSTRAINT PK_SlotAddr PRIMARY KEY (f_slot,f_label));` + CONSTRAINT PK_Score PRIMARY KEY (f_slot,f_label));` InsertNewScore = ` INSERT INTO t_score_metrics ( diff --git a/pkg/postgresql/missed_blocks.go b/pkg/postgresql/missed_blocks.go index dfadf50..a9e50a9 100644 --- a/pkg/postgresql/missed_blocks.go +++ b/pkg/postgresql/missed_blocks.go @@ -18,7 +18,7 @@ var ( CREATE TABLE IF NOT EXISTS t_missed_blocks( f_slot INT, f_label TEXT, - CONSTRAINT PK_SlotAddr PRIMARY KEY (f_slot,f_label));` + CONSTRAINT PK_MissedBlock PRIMARY KEY (f_slot,f_label));` InsertNewMissedBlock = ` INSERT INTO t_missed_blocks ( From 5f25d78c0a72fc6fbfaa3bc4a291d390d329c9cd Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 15:35:11 +0100 Subject: [PATCH 21/62] Fix TEXT types and do nothing on attestation conflict --- pkg/postgresql/attestation_arrival.go | 7 ++++--- pkg/postgresql/block_arrival.go | 4 ++-- pkg/postgresql/block_score.go | 2 +- pkg/postgresql/service.go | 4 ---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pkg/postgresql/attestation_arrival.go b/pkg/postgresql/attestation_arrival.go index a3b91dc..cbdfc5b 100644 --- a/pkg/postgresql/attestation_arrival.go +++ b/pkg/postgresql/attestation_arrival.go @@ -16,7 +16,7 @@ import ( var ( CreateAttTable = ` CREATE TABLE IF NOT EXISTS t_att_metrics( - f_label VARCHAR(100), + f_label TEXT, f_slot INT, f_committee_index INT, f_signature TEXT, @@ -24,7 +24,7 @@ var ( f_target_root TEXT, f_head_root TEXT, f_timestamp TIMESTAMP, - CONSTRAINT PK_Attestation PRIMARY KEY (f_label,f_slot,f_committee_index,f_signature));` + CONSTRAINT PK_Attestation PRIMARY KEY (f_label,f_slot,f_committee_index,f_signature));` InsertNewAtt = ` INSERT INTO t_att_metrics ( @@ -36,7 +36,8 @@ var ( f_source_root, f_target_root, f_head_root) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8);` + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT DO NOTHING;` ) // in case the table did not exist diff --git a/pkg/postgresql/block_arrival.go b/pkg/postgresql/block_arrival.go index e937a33..e04b70c 100644 --- a/pkg/postgresql/block_arrival.go +++ b/pkg/postgresql/block_arrival.go @@ -17,12 +17,12 @@ var ( CREATE_BLOCK_ARRIVAL_TABLE = ` CREATE TABLE IF NOT EXISTS t_block_metrics( f_slot INT, - f_label VARCHAR(100), + f_label TEXT, f_timestamp TIME, CONSTRAINT PK_Block PRIMARY KEY (f_slot,f_label));` InsertNewBlock = ` - INSERT INTO t_score_metrics ( + INSERT INTO t_block_metrics ( f_slot, f_label, f_timestamp) diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index 912093c..09068e3 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -17,7 +17,7 @@ var ( CREATE_SCORE_TABLE = ` CREATE TABLE IF NOT EXISTS t_score_metrics( f_slot INT, - f_label VARCHAR(100), + f_label TEXT, f_score FLOAT, f_duration FLOAT, f_correct_source INT, diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index 028e861..ec57cff 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -178,10 +178,6 @@ func (p PostgresDBService) ExecuteBatch(batch pgx_v4.Batch) error { rows, qerr = batchResults.Query() rows.Close() } - if qerr.Error() != "no result" { - log.Errorf(qerr.Error()) - } - // p.MonitorStruct.AddDBWrite(time.Since(snapshot).Seconds()) log.Debugf("Batch process time: %f, batch size: %d", time.Since(snapshot).Seconds(), batch.Len()) From 3f149b04e59d08cbe8a690e6cf45de423170922f Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 15:35:43 +0100 Subject: [PATCH 22/62] Fix comment --- pkg/app/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index 7620847..c11c17f 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -99,7 +99,7 @@ func (s *AppService) Run() { } // every new attestation - err = item.Eth2Provider.Api.Events(s.ctx, []string{"attestation"}, item.HandleAttestationEvent) // every new head + err = item.Eth2Provider.Api.Events(s.ctx, []string{"attestation"}, item.HandleAttestationEvent) // every new attestation if err != nil { log.Panicf("failed to subscribe to attestation events: %s, label: %s", err, item.Eth2Provider.Label) } From b2274465538753d488b11af235e55be30784449f Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 15:38:20 +0100 Subject: [PATCH 23/62] Fix format --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 94de68f..16993c5 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,9 @@ BIN="./build/eth_cl_live_metrics" build: $(GOCC) build -o $(BIN) - install: $(GOCC) install - dependencies: $(GIT_SUBM) update --init From 294ad618513aad386c331a54823ca09cc26f3812 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 16:04:25 +0100 Subject: [PATCH 24/62] Fix missing argument --- pkg/analysis/analyzer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index daa0c20..0325b42 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -86,6 +86,7 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { // Store in DB params := make([]interface{}, 0) + params = append(params, metrics.Slot) params = append(params, metrics.Label) params = append(params, metrics.Score) params = append(params, metrics.Duration) From e245ede5b578953b2efece93a90499d7d4beea90 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 16:09:01 +0100 Subject: [PATCH 25/62] Fix params array length --- pkg/analysis/events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index a3fce6a..c3d9460 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -27,7 +27,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { if b.CurrentHeadSlot != 0 && // we are not at the beginning of the run newBlock.Bellatrix.Message.Slot-phase0.Slot(b.CurrentHeadSlot) > 1 { // there a gap bigger than 1 with the new head for i := b.CurrentHeadSlot; i < uint64(newBlock.Bellatrix.Message.Slot); i++ { - params := make([]interface{}, 2) + params := make([]interface{}, 0) params = append(params, i) params = append(params, b.Eth2Provider.Label) writeTask := postgresql.WriteTask{ @@ -37,7 +37,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { b.DBClient.WriteChan <- writeTask // store } } - params := make([]interface{}, 3) + params := make([]interface{}, 0) params = append(params, int(data.Slot)) params = append(params, b.Eth2Provider.Label) params = append(params, timestamp) From 7dd90c954ee419fd0c9aa07d1e2e7c7a21e16e1f Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 3 Nov 2022 16:42:27 +0100 Subject: [PATCH 26/62] Add possibility to measure different metrics separately --- pkg/app/service.go | 46 ++++++++++++++++++++++++++++++++++------- pkg/cmd/analyzer_cmd.go | 20 +++++++++++++++++- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index c11c17f..6b57c9b 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/attestantio/go-eth2-client/spec/phase0" @@ -18,6 +19,8 @@ var ( log = logrus.WithField( "module", modName, ) + attestationMetric = "attestations" + proposalMetric = "proposals" ) type AppService struct { @@ -26,9 +29,10 @@ type AppService struct { initTime time.Time ChainTime chain_stats.ChainTime HeadSlot phase0.Slot + Metrics []string } -func NewAppService(ctx context.Context, bnEndpoints []string, dbEndpooint string, dbWorkers int) (*AppService, error) { +func NewAppService(ctx context.Context, bnEndpoints []string, dbEndpooint string, dbWorkers int, metrics []string) (*AppService, error) { dbClient, err := postgresql.ConnectToDB(ctx, dbEndpooint, dbWorkers) @@ -73,11 +77,45 @@ func NewAppService(ctx context.Context, bnEndpoints []string, dbEndpooint string ChainTime: chain_stats.ChainTime{ GenesisTime: genesis, }, + Metrics: metrics, }, nil } // Main routine: build block history and block proposals every 12 seconds func (s *AppService) Run() { + var wg sync.WaitGroup + for _, item := range s.Metrics { + if item == attestationMetric { + wg.Add(1) + s.RunAttestations() + } + + if item == proposalMetric { + wg.Add(1) + go s.RunMainRoutine(&wg) + } + } + + wg.Wait() + +} + +// Main routine: build block history and block proposals every 12 seconds +func (s *AppService) RunAttestations() { + + // Subscribe to events from each client + for _, item := range s.Analyzers { + err := item.Eth2Provider.Api.Events(s.ctx, []string{"attestation"}, item.HandleAttestationEvent) // every new head + if err != nil { + log.Panicf("failed to subscribe to head events: %s, label: %s", err, item.Eth2Provider.Label) + } + + } +} + +// Main routine: build block history and block proposals every 12 seconds +func (s *AppService) RunMainRoutine(wg *sync.WaitGroup) { + defer wg.Done() log = log.WithField("routine", "main") historyBuilt := false @@ -98,12 +136,6 @@ func (s *AppService) Run() { log.Panicf("failed to subscribe to head events: %s, label: %s", err, item.Eth2Provider.Label) } - // every new attestation - err = item.Eth2Provider.Api.Events(s.ctx, []string{"attestation"}, item.HandleAttestationEvent) // every new attestation - if err != nil { - log.Panicf("failed to subscribe to attestation events: %s, label: %s", err, item.Eth2Provider.Label) - } - } // tick every slot start (12 seconds) diff --git a/pkg/cmd/analyzer_cmd.go b/pkg/cmd/analyzer_cmd.go index f15ed71..f8d9a95 100644 --- a/pkg/cmd/analyzer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -36,6 +36,11 @@ var AnalyzerCommand = &cli.Command{ Name: "db-workers", Usage: "10", DefaultText: "1", + }, + &cli.StringFlag{ + Name: "metrics", + Usage: "proposals,attestations", + DefaultText: "proposals,attestations", }}, } @@ -43,6 +48,7 @@ var QueryTimeout = 90 * time.Second func LaunchBlockAnalyzer(c *cli.Context) error { dbWorkers := 1 + metrics := make([]string, 0) logLauncher := log.WithField( "module", "ScorerCommand", ) @@ -65,6 +71,18 @@ func LaunchBlockAnalyzer(c *cli.Context) error { if !c.IsSet("db-workers") { logLauncher.Warnf("no database workers configured, default is 1") + metrics = append(metrics, "proposals") + metrics = append(metrics, "attestations") + } else { + metricsInput := strings.Split(c.String("metrics"), ",") + + for _, item := range metricsInput { + metrics = append(metrics, item) + } + } + + if !c.IsSet("metrics") { + logLauncher.Warnf("no metrics configured, measuring all") } else { dbWorkers = c.Int("db-workers") } @@ -72,7 +90,7 @@ func LaunchBlockAnalyzer(c *cli.Context) error { bnEndpoints := strings.Split(c.String("bn-endpoints"), ",") dbEndpoint := c.String("db-endpoint") - service, err := app.NewAppService(c.Context, bnEndpoints, dbEndpoint, dbWorkers) + service, err := app.NewAppService(c.Context, bnEndpoints, dbEndpoint, dbWorkers, metrics) if err != nil { log.Fatal("could not start app: %s", err.Error()) } From 1594277e648a6b8f31ec8b31194bb6f08cf4cde7 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 08:53:00 +0100 Subject: [PATCH 27/62] Organize repo: new name and Makefile command --- .env-example | 5 +++++ Makefile | 18 +++++++++++++----- go.mod | 2 +- main.go | 8 ++++---- pkg/analysis/analyzer.go | 6 +++--- pkg/analysis/events.go | 2 +- pkg/analysis/metrics.go | 4 ++-- pkg/app/service.go | 6 +++--- pkg/cmd/analyzer_cmd.go | 8 ++++---- 9 files changed, 36 insertions(+), 23 deletions(-) create mode 100644 .env-example diff --git a/.env-example b/.env-example new file mode 100644 index 0000000..0e45375 --- /dev/null +++ b/.env-example @@ -0,0 +1,5 @@ +LIVE_METRICS_CMD="eth-cl-live-metrics" +LIVE_METRICS_LOG_LEVEL="debug" +LIVE_METRICS_BN_ENDPOINTS="prysm/localhost:3500,lh/localhost:5052" +LIVE_METRICS_DB_ENDPOINT="postgresql://db_user:db_password@db_ip:db_port/db_name" +LIVE_METRICS_DB_WORKERS="5" \ No newline at end of file diff --git a/Makefile b/Makefile index 16993c5..3f78a1f 100644 --- a/Makefile +++ b/Makefile @@ -7,14 +7,22 @@ BIN="./build/eth_cl_live_metrics" .PHONY: check dependencies build install clean -build: - $(GOCC) build -o $(BIN) +build: + $(GOCC) build -o $(BIN) install: - $(GOCC) install + $(GOCC) install dependencies: - $(GIT_SUBM) update --init + $(GIT_SUBM) update --init clean: - rm -r $(BIN_PATH) \ No newline at end of file + rm -r $(BIN_PATH) + +run: + $(BIN) $(LIVE_METRICS_CMD) \ + --log-level=${LIVE_METRICS_LOG_LEVEL} \ + --bn-endpoints=${LIVE_METRICS_BN_ENDPOINT} \ + --db-endpoint=${LIVE_METRICS_DB_URL} \ + --db-workers=${LIVE_METRICS_DB_WORKERS_NUM} \ + --metrics=${LIVE_METRICS_METRICS} \ No newline at end of file diff --git a/go.mod b/go.mod index 22e55b1..4ff79ee 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/tdahar/block-scorer +module github.com/tdahar/eth-cl-live-metrics go 1.17 diff --git a/main.go b/main.go index 9723999..243be18 100644 --- a/main.go +++ b/main.go @@ -6,13 +6,13 @@ import ( "os" "github.com/sirupsen/logrus" - "github.com/tdahar/block-scorer/pkg/cmd" + "github.com/tdahar/eth-cl-live-metrics/pkg/cmd" cli "github.com/urfave/cli/v2" ) var ( Version = "v1.0.0" - CliName = "Eth2 Block Scorer" + CliName = "Eth CL Live Metrics" log = logrus.WithField( "cli", "CliName", ) @@ -22,12 +22,12 @@ func main() { fmt.Println(CliName, Version) // Set the general log configurations for the entire tool - logrus.SetLevel(logrus.DebugLevel) + logrus.SetLevel(logrus.InfoLevel) app := &cli.App{ Name: CliName, Usage: "Tinny client that requests and processes the Beacon Block proposals for each client.", - UsageText: "block-scorer [commands] [arguments...]", + UsageText: "live-metrics [commands] [arguments...]", Authors: []*cli.Author{ { Name: "Tarun", diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 0325b42..32139c5 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -8,9 +8,9 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/go-bitfield" "github.com/sirupsen/logrus" - "github.com/tdahar/block-scorer/pkg/analysis/additional_structs" - "github.com/tdahar/block-scorer/pkg/client_api" - "github.com/tdahar/block-scorer/pkg/postgresql" + "github.com/tdahar/eth-cl-live-metrics/pkg/analysis/additional_structs" + "github.com/tdahar/eth-cl-live-metrics/pkg/client_api" + "github.com/tdahar/eth-cl-live-metrics/pkg/postgresql" ) var ( diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index c3d9460..589bbe8 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -6,7 +6,7 @@ import ( api "github.com/attestantio/go-eth2-client/api/v1" "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/tdahar/block-scorer/pkg/postgresql" + "github.com/tdahar/eth-cl-live-metrics/pkg/postgresql" ) func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 69fef53..239f1d5 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -6,8 +6,8 @@ import ( "github.com/attestantio/go-eth2-client/spec/bellatrix" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/prysmaticlabs/go-bitfield" - "github.com/tdahar/block-scorer/pkg/postgresql" - "github.com/tdahar/block-scorer/pkg/utils" + "github.com/tdahar/eth-cl-live-metrics/pkg/postgresql" + "github.com/tdahar/eth-cl-live-metrics/pkg/utils" ) const ( diff --git a/pkg/app/service.go b/pkg/app/service.go index 6b57c9b..dc9562a 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -9,9 +9,9 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/sirupsen/logrus" - "github.com/tdahar/block-scorer/pkg/analysis" - "github.com/tdahar/block-scorer/pkg/chain_stats" - "github.com/tdahar/block-scorer/pkg/postgresql" + "github.com/tdahar/eth-cl-live-metrics/pkg/analysis" + "github.com/tdahar/eth-cl-live-metrics/pkg/chain_stats" + "github.com/tdahar/eth-cl-live-metrics/pkg/postgresql" ) var ( diff --git a/pkg/cmd/analyzer_cmd.go b/pkg/cmd/analyzer_cmd.go index f8d9a95..2076536 100644 --- a/pkg/cmd/analyzer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -7,14 +7,14 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" - "github.com/tdahar/block-scorer/pkg/app" - "github.com/tdahar/block-scorer/pkg/utils" + "github.com/tdahar/eth-cl-live-metrics/pkg/app" + "github.com/tdahar/eth-cl-live-metrics/pkg/utils" cli "github.com/urfave/cli/v2" ) var AnalyzerCommand = &cli.Command{ - Name: "block-scorer", - Usage: "Receive Block proposals from clients and evaluate score", + Name: "live-metrics", + Usage: "Receive Block proposals from clients and evaluate score, as well as other metrics", Action: LaunchBlockAnalyzer, Flags: []cli.Flag{ &cli.StringFlag{ From d1d9d92b102c56d57327df96b3db8724fdf2a0db Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 09:58:37 +0100 Subject: [PATCH 28/62] Add duration to metrics --- pkg/analysis/analyzer.go | 2 +- pkg/analysis/metrics.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 32139c5..31eaa4f 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -77,7 +77,7 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { } // for now we just have Bellatrix - metrics, err := b.BellatrixBlockMetrics(block.Bellatrix) + metrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) if err != nil { return fmt.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) } diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 239f1d5..96b9ff1 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -21,7 +21,7 @@ const ( // https://github.com/attestantio/vouch/blob/0c75ee8315dc4e5df85eb2aa09b4acc2b4436661/strategies/beaconblockproposal/best/score.go#L222 // This function receives a new block proposal and ouputs a block score and metrics about the block -func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (postgresql.BlockMetricsModel, error) { +func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock, duration float64) (postgresql.BlockMetricsModel, error) { // log := b.log.WithField("task", "bellatrix-block-score") // add extra log for function totalNewVotes := 0 totalScore := 0 @@ -92,6 +92,7 @@ func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock) (po CorrectTarget: totalCorrectTarget, CorrectHead: totalCorrectHead, Score: float64(totalScore), + Duration: duration, NewVotes: totalNewVotes, AttNum: len(block.Body.Attestations), Sync1Bits: int(block.Body.SyncAggregate.SyncCommitteeBits.Count()), From e5ad11a009b774d5f9b7d583a32898e7b24a1fac Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 09:59:04 +0100 Subject: [PATCH 29/62] Fix trace and debug logs --- go.mod | 2 -- go.sum | 13 ++++++------- pkg/analysis/analyzer.go | 11 ++++++----- pkg/analysis/events.go | 6 +++--- pkg/postgresql/service.go | 7 +++---- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 4ff79ee..1ce8a0d 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,6 @@ require ( github.com/urfave/cli/v2 v2.16.3 ) -require github.com/jackc/pgx v3.6.2+incompatible // indirect - require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.13.0 // indirect diff --git a/go.sum b/go.sum index 5c4c958..64b87e7 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/attestantio/go-eth2-client v0.11.7 h1:mRCtVZYg/Tmp47mNkWKnro/Wj63+jj1vz9BWNQN8MYo= -github.com/attestantio/go-eth2-client v0.11.7/go.mod h1:zXL/BxC0cBBhxj+tP7QG7t9Ufoa8GwQLdlbvZRd9+dM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -53,6 +51,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -93,6 +92,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -170,6 +170,7 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -191,8 +192,6 @@ github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrU github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= -github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -233,6 +232,7 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -300,6 +300,7 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -315,9 +316,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/urfave/cli/v2 v2.16.3 h1:gHoFIwpPjoyIMbJp/VFd+/vuD0dAgFK4B6DpEMFJfQk= github.com/urfave/cli/v2 v2.16.3/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= @@ -355,7 +356,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -651,7 +651,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 31eaa4f..bfe6c36 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -2,7 +2,6 @@ package analysis import ( "context" - "fmt" "time" "github.com/attestantio/go-eth2-client/spec/phase0" @@ -50,7 +49,7 @@ func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, tim } // Asks for a block proposal to the client and stores score in the database -func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { +func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { log := b.log.WithField("task", "generate-block") log.Debugf("processing new block: %d\n", slot) @@ -61,7 +60,8 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { blockTime := time.Since(snapshot).Seconds() // time to generate block if err != nil { - return fmt.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) + log.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) + return } for i := range b.AttHistory { @@ -79,7 +79,8 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { // for now we just have Bellatrix metrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) if err != nil { - return fmt.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) + log.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) + return } log.Infof("Block Generation Time: %f", blockTime) log.Infof("Metrics: %+v", metrics) @@ -106,5 +107,5 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) error { // We block the update attestations as new head could impact attestations of the proposed block b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations - return nil + return } diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 589bbe8..9d8df97 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -18,7 +18,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { } data := event.Data.(*api.HeadEvent) // cast to head event - log.Debugf("Received a new head event: slot %d", data.Slot) + log.Tracef("Received a new head event: slot %d", data.Slot) // we only receive the block hash, get the new block newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, hex.EncodeToString(data.Block[:])) @@ -63,7 +63,7 @@ func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { timestamp := time.Now() log := b.log.WithField("routine", "attestation-event") - log.Debugf("Received a new event") + log.Tracef("Received a new event") if event.Data == nil { log.Errorf("attestation event does not contain anything") @@ -112,6 +112,6 @@ func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { b.DBClient.WriteChan <- writeTask // send task to be written - log.Debugf("Finished processing event in %f seconds", time.Since(timestamp).Seconds()) + log.Tracef("Finished processing event in %f seconds", time.Since(timestamp).Seconds()) } diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index ec57cff..9cab767 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -123,7 +123,7 @@ func (p *PostgresDBService) runWriters() { for { if p.endProcess >= 1 && len(p.WriteChan) == 0 { - wlogWriter.Info("finish detected, closing persister") + wlogWriter.Warnf("finish detected, closing persister") break loop } @@ -133,7 +133,7 @@ func (p *PostgresDBService) runWriters() { writeBatch.Queue(task.QueryString, task.Params...) if writeBatch.Len() > MAX_BATCH_QUEUE { - wlogWriter.Debugf("Writing batch to database") + wlogWriter.Tracef("Writing batch to database") err := p.ExecuteBatch(writeBatch) if err != nil { wlogWriter.Errorf("Error processing batch", err.Error()) @@ -178,8 +178,7 @@ func (p PostgresDBService) ExecuteBatch(batch pgx_v4.Batch) error { rows, qerr = batchResults.Query() rows.Close() } - // p.MonitorStruct.AddDBWrite(time.Since(snapshot).Seconds()) - log.Debugf("Batch process time: %f, batch size: %d", time.Since(snapshot).Seconds(), batch.Len()) + log.Tracef("Batch process time: %f, batch size: %d", time.Since(snapshot).Seconds(), batch.Len()) return tx.Commit(p.ctx) From 81a9e9f31a1c889ad3ff04c9f5f4385d31adab77 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 10:26:39 +0100 Subject: [PATCH 30/62] Fix Makefile with correct environment variables --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3f78a1f..d522b80 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,13 @@ +#!make GOCC=go MKDIR_P=mkdir -p GIT_SUBM=git submodule +include .env + BIN_PATH=./build BIN="./build/eth_cl_live_metrics" +LIVE_METRICS_CMD="live-metrics" .PHONY: check dependencies build install clean @@ -22,7 +26,7 @@ clean: run: $(BIN) $(LIVE_METRICS_CMD) \ --log-level=${LIVE_METRICS_LOG_LEVEL} \ - --bn-endpoints=${LIVE_METRICS_BN_ENDPOINT} \ - --db-endpoint=${LIVE_METRICS_DB_URL} \ - --db-workers=${LIVE_METRICS_DB_WORKERS_NUM} \ - --metrics=${LIVE_METRICS_METRICS} \ No newline at end of file + --bn-endpoints=${LIVE_METRICS_BN_ENDPOINTS} \ + --db-endpoint=${LIVE_METRICS_DB_ENDPOINT} \ + --db-workers=${LIVE_METRICS_DB_WORKERS} \ + --metrics=${LIVE_METRICS_METRICS} \ No newline at end of file From b5d65cfeed1840f4198a1b31824d94f1d3345272 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 11:27:04 +0100 Subject: [PATCH 31/62] Switch all scores to float --- pkg/analysis/metrics.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 96b9ff1..0c80632 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -24,8 +24,8 @@ const ( func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock, duration float64) (postgresql.BlockMetricsModel, error) { // log := b.log.WithField("task", "bellatrix-block-score") // add extra log for function totalNewVotes := 0 - totalScore := 0 - attScore := 0 + totalScore := float64(0) + attScore := float64(0) totalCorrectSource := 0 totalCorrectTarget := 0 totalCorrectHead := 0 @@ -75,7 +75,7 @@ func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock, dur totalNewVotes += newVotes // denominator := (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) * WEIGHT_DENOMINATOR / PROPOSER_WEIGHT - attScore += score / WEIGHT_DENOMINATOR + attScore += float64(score) / float64(WEIGHT_DENOMINATOR) } @@ -83,7 +83,7 @@ func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock, dur attesterSlashingScore, proposerSlashingScore := scoreSlashings(block.Body.AttesterSlashings, block.Body.ProposerSlashings) - totalScore = attScore + int(syncCommitteeScore) + int(attesterSlashingScore) + int(proposerSlashingScore) + totalScore = attScore + syncCommitteeScore + attesterSlashingScore + proposerSlashingScore return postgresql.BlockMetricsModel{ Slot: int(block.Slot), From 4af46dd791652fc6c1668126092df87309e86d3e Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 11:28:04 +0100 Subject: [PATCH 32/62] Return all scores to main routine --- pkg/analysis/metrics.go | 27 +++++++++++++++------------ pkg/postgresql/block_score.go | 27 +++++++++++++++------------ 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 0c80632..42b9905 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -86,18 +86,21 @@ func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock, dur totalScore = attScore + syncCommitteeScore + attesterSlashingScore + proposerSlashingScore return postgresql.BlockMetricsModel{ - Slot: int(block.Slot), - Label: b.Eth2Provider.Label, - CorrectSource: totalCorrectSource, - CorrectTarget: totalCorrectTarget, - CorrectHead: totalCorrectHead, - Score: float64(totalScore), - Duration: duration, - NewVotes: totalNewVotes, - AttNum: len(block.Body.Attestations), - Sync1Bits: int(block.Body.SyncAggregate.SyncCommitteeBits.Count()), - AttesterSlashings: len(block.Body.AttesterSlashings), - ProposerSlashings: len(block.Body.ProposerSlashings), + Slot: int(block.Slot), + Label: b.Eth2Provider.Label, + CorrectSource: totalCorrectSource, + CorrectTarget: totalCorrectTarget, + CorrectHead: totalCorrectHead, + Score: float64(totalScore), + Duration: duration, + NewVotes: totalNewVotes, + AttNum: len(block.Body.Attestations), + Sync1Bits: int(block.Body.SyncAggregate.SyncCommitteeBits.Count()), + AttesterSlashings: len(block.Body.AttesterSlashings), + ProposerSlashings: len(block.Body.ProposerSlashings), + ProposerSlashingScore: proposerSlashingScore, + AttesterSlashingScore: attScore, + SyncScore: syncCommitteeScore, }, nil } diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index 09068e3..56a7d17 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -54,16 +54,19 @@ func (p *PostgresDBService) createScoreMetricsTable(ctx context.Context, pool *p } type BlockMetricsModel struct { - Slot int - Label string - Score float64 - Duration float64 - CorrectSource int - CorrectTarget int - CorrectHead int - Sync1Bits int - AttNum int - NewVotes int - AttesterSlashings int - ProposerSlashings int + Slot int + Label string + Score float64 + Duration float64 + CorrectSource int + CorrectTarget int + CorrectHead int + Sync1Bits int + AttNum int + NewVotes int + AttesterSlashings int + ProposerSlashings int + ProposerSlashingScore float64 + AttesterSlashingScore float64 + SyncScore float64 } From e1b73eaf9b5a5eaeb91ee30669baf1fc91e3be01 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 4 Nov 2022 11:31:12 +0100 Subject: [PATCH 33/62] Add extra fields to database --- pkg/analysis/analyzer.go | 5 +++++ pkg/postgresql/block_score.go | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index bfe6c36..e6dbb55 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -97,6 +97,11 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { params = append(params, metrics.Sync1Bits) params = append(params, metrics.AttNum) params = append(params, metrics.NewVotes) + params = append(params, metrics.AttesterSlashings) + params = append(params, metrics.ProposerSlashings) + params = append(params, metrics.ProposerSlashingScore) + params = append(params, metrics.AttesterSlashingScore) + params = append(params, metrics.SyncScore) writeTask := postgresql.WriteTask{ QueryString: postgresql.InsertNewScore, diff --git a/pkg/postgresql/block_score.go b/pkg/postgresql/block_score.go index 56a7d17..124194c 100644 --- a/pkg/postgresql/block_score.go +++ b/pkg/postgresql/block_score.go @@ -26,6 +26,11 @@ var ( f_sync_bits INT, f_att_num INT, f_new_votes INT, + f_attester_slashings INT, + f_proposer_slashings INT, + f_proposer_slashing_score FLOAT, + f_attester_slashing_score FLOAT, + f_sync_score FLOAT, CONSTRAINT PK_Score PRIMARY KEY (f_slot,f_label));` InsertNewScore = ` @@ -39,8 +44,13 @@ var ( f_correct_head, f_sync_bits, f_att_num, - f_new_votes) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);` + f_new_votes, + f_attester_slashings, + f_proposer_slashings, + f_proposer_slashing_score, + f_attester_slashing_score, + f_sync_score) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15);` ) // in case the table did not exist From c6808bf7d2931cfdff79f52d1b2a80db9ff48e13 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 9 Nov 2022 09:10:32 +0100 Subject: [PATCH 34/62] Add error and trace messages --- pkg/postgresql/service.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index 9cab767..0514e79 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -139,6 +139,8 @@ func (p *PostgresDBService) runWriters() { wlogWriter.Errorf("Error processing batch", err.Error()) } writeBatch = pgx_v4.Batch{} + } else { + log.Tracef("%d pending tasks to persist", MAX_BATCH_QUEUE-writeBatch.Len()) } case <-p.ctx.Done(): @@ -178,6 +180,10 @@ func (p PostgresDBService) ExecuteBatch(batch pgx_v4.Batch) error { rows, qerr = batchResults.Query() rows.Close() } + if qerr.Error() != "no result" { + log.Errorf(qerr.Error()) + } + log.Tracef("Batch process time: %f, batch size: %d", time.Since(snapshot).Seconds(), batch.Len()) return tx.Commit(p.ctx) From adaefdf08772713ab6f26acb81d870c5bd26fcbf Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 9 Nov 2022 09:15:01 +0100 Subject: [PATCH 35/62] Fix saved score --- pkg/analysis/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analysis/metrics.go b/pkg/analysis/metrics.go index 42b9905..2a7868d 100644 --- a/pkg/analysis/metrics.go +++ b/pkg/analysis/metrics.go @@ -99,7 +99,7 @@ func (b *ClientLiveData) BellatrixBlockMetrics(block *bellatrix.BeaconBlock, dur AttesterSlashings: len(block.Body.AttesterSlashings), ProposerSlashings: len(block.Body.ProposerSlashings), ProposerSlashingScore: proposerSlashingScore, - AttesterSlashingScore: attScore, + AttesterSlashingScore: attesterSlashingScore, SyncScore: syncCommitteeScore, }, nil } From e770f5e0c48ad36a45f7946134a54b9cd7397b9e Mon Sep 17 00:00:00 2001 From: Tarun Date: Mon, 5 Dec 2022 08:45:40 +0000 Subject: [PATCH 36/62] Added logs for activated services --- pkg/app/service.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/app/service.go b/pkg/app/service.go index dc9562a..db233a8 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -86,11 +86,13 @@ func (s *AppService) Run() { var wg sync.WaitGroup for _, item := range s.Metrics { if item == attestationMetric { + log.Infof("initiating attestation events monitoring") wg.Add(1) s.RunAttestations() } if item == proposalMetric { + log.Infof("initiating block proposal monitoring") wg.Add(1) go s.RunMainRoutine(&wg) } From fa18c189387cff28f2489a671a13d77e9dbb39ab Mon Sep 17 00:00:00 2001 From: Tarun Date: Mon, 5 Dec 2022 08:53:07 +0000 Subject: [PATCH 37/62] Add dynamic batch len --- pkg/app/service.go | 15 +++++++++++++-- pkg/postgresql/service.go | 6 ++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index db233a8..12578ae 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -32,9 +32,20 @@ type AppService struct { Metrics []string } -func NewAppService(ctx context.Context, bnEndpoints []string, dbEndpooint string, dbWorkers int, metrics []string) (*AppService, error) { +func NewAppService(ctx context.Context, + bnEndpoints []string, + dbEndpooint string, + dbWorkers int, + metrics []string) (*AppService, error) { + + batchLen := len(bnEndpoints) + for _, item := range metrics { + if item == attestationMetric { + batchLen = 100 + } + } - dbClient, err := postgresql.ConnectToDB(ctx, dbEndpooint, dbWorkers) + dbClient, err := postgresql.ConnectToDB(ctx, dbEndpooint, dbWorkers, batchLen) if err != nil { log.Panicf("could not connect to database: %s", err) diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index 9cab767..d84b148 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -35,11 +35,12 @@ type PostgresDBService struct { endProcess int32 FinishSignalChan chan struct{} workerNum int + maxBatchQueue int } // Connect to the PostgreSQL Database and get the multithread-proof connection // from the given url-composed credentials -func ConnectToDB(ctx context.Context, url string, workerNum int) (*PostgresDBService, error) { +func ConnectToDB(ctx context.Context, url string, workerNum int, batchLen int) (*PostgresDBService, error) { mainCtx, cancel := context.WithCancel(ctx) // spliting the url to don't share any confidential information on logs @@ -65,6 +66,7 @@ func ConnectToDB(ctx context.Context, url string, workerNum int) (*PostgresDBSer endProcess: 0, FinishSignalChan: make(chan struct{}, 1), workerNum: workerNum, + maxBatchQueue: batchLen, } // init the psql db err = psqlDB.init(ctx, psqlDB.psqlPool) @@ -132,7 +134,7 @@ func (p *PostgresDBService) runWriters() { wlogWriter.Tracef("Received new write task") writeBatch.Queue(task.QueryString, task.Params...) - if writeBatch.Len() > MAX_BATCH_QUEUE { + if writeBatch.Len() > p.maxBatchQueue { wlogWriter.Tracef("Writing batch to database") err := p.ExecuteBatch(writeBatch) if err != nil { From 855533a2098e8c0a0f8faf815a712744a1e4abca Mon Sep 17 00:00:00 2001 From: Tarun Date: Mon, 5 Dec 2022 12:29:10 +0000 Subject: [PATCH 38/62] Add exit signal controlled --- pkg/app/service.go | 44 +++++++++++++++++++++++++++++---------- pkg/cmd/analyzer_cmd.go | 25 +++++++++++++++++++++- pkg/postgresql/service.go | 18 +++++++++------- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index 12578ae..cd852a7 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/attestantio/go-eth2-client/spec/phase0" @@ -24,20 +25,24 @@ var ( ) type AppService struct { - ctx context.Context - Analyzers []*analysis.ClientLiveData - initTime time.Time - ChainTime chain_stats.ChainTime - HeadSlot phase0.Slot - Metrics []string + ctx context.Context + cancel context.CancelFunc + Analyzers []*analysis.ClientLiveData + initTime time.Time + ChainTime chain_stats.ChainTime + HeadSlot phase0.Slot + Metrics []string + finishTasks int32 + DBClient *postgresql.PostgresDBService } -func NewAppService(ctx context.Context, +func NewAppService(pCtx context.Context, bnEndpoints []string, dbEndpooint string, dbWorkers int, metrics []string) (*AppService, error) { + ctx, cancel := context.WithCancel(pCtx) batchLen := len(bnEndpoints) for _, item := range metrics { if item == attestationMetric { @@ -82,18 +87,21 @@ func NewAppService(ctx context.Context, } return &AppService{ ctx: ctx, + cancel: cancel, Analyzers: analyzers, initTime: time.Now(), HeadSlot: headHeader.Header.Message.Slot, ChainTime: chain_stats.ChainTime{ GenesisTime: genesis, }, - Metrics: metrics, + Metrics: metrics, + DBClient: dbClient, }, nil } // Main routine: build block history and block proposals every 12 seconds func (s *AppService) Run() { + defer s.cancel() var wg sync.WaitGroup for _, item := range s.Metrics { if item == attestationMetric { @@ -153,13 +161,19 @@ func (s *AppService) RunMainRoutine(wg *sync.WaitGroup) { // tick every slot start (12 seconds) ticker := time.After(time.Until(s.ChainTime.SlotTime(phase0.Slot(s.HeadSlot + 1)))) - +loop: for { + if s.finishTasks > 0 { + log.Infof("closing main routine") + s.Analyzers[0].DBClient.DoneTasks() // all the analyzers have the same db client + break loop + } select { case <-s.ctx.Done(): - - return + log.Infof("closing main routine") + s.DBClient.DoneTasks() // all the analyzers have the same db client + break loop case <-ticker: // we entered a new slot time @@ -177,4 +191,12 @@ func (s *AppService) RunMainRoutine(wg *sync.WaitGroup) { default: } } + log.Infof("finished") +} + +func (s *AppService) Close() { + log.Info("Sudden closed detected, closing Live Metrics") + atomic.AddInt32(&s.finishTasks, int32(1)) + s.DBClient.WgDBWriter.Wait() + s.cancel() } diff --git a/pkg/cmd/analyzer_cmd.go b/pkg/cmd/analyzer_cmd.go index 2076536..7124160 100644 --- a/pkg/cmd/analyzer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -1,7 +1,10 @@ package cmd import ( + "os" + "os/signal" "strings" + "syscall" "time" "github.com/pkg/errors" @@ -94,7 +97,27 @@ func LaunchBlockAnalyzer(c *cli.Context) error { if err != nil { log.Fatal("could not start app: %s", err.Error()) } - service.Run() + + procDoneC := make(chan struct{}) + sigtermC := make(chan os.Signal) + + signal.Notify(sigtermC, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, syscall.SIGTERM) + + go func() { + service.Run() + procDoneC <- struct{}{} + }() + + select { + case <-sigtermC: + logLauncher.Info("Sudden shutdown detected, controlled shutdown of the cli triggered") + service.Close() + + case <-procDoneC: + logLauncher.Info("Process successfully finish!") + } + close(sigtermC) + close(procDoneC) return nil } diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index d84b148..9c6e351 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -36,6 +36,7 @@ type PostgresDBService struct { FinishSignalChan chan struct{} workerNum int maxBatchQueue int + WgDBWriter sync.WaitGroup } // Connect to the PostgreSQL Database and get the multithread-proof connection @@ -67,6 +68,7 @@ func ConnectToDB(ctx context.Context, url string, workerNum int, batchLen int) ( FinishSignalChan: make(chan struct{}, 1), workerNum: workerNum, maxBatchQueue: batchLen, + WgDBWriter: sync.WaitGroup{}, } // init the psql db err = psqlDB.init(ctx, psqlDB.psqlPool) @@ -77,10 +79,6 @@ func ConnectToDB(ctx context.Context, url string, workerNum int, batchLen int) ( return psqlDB, err } -// Close the connection with the PostgreSQL -func (p *PostgresDBService) Close() { -} - func (p *PostgresDBService) init(ctx context.Context, pool *pgxpool.Pool) error { // create the tables err := p.createScoreMetricsTable(ctx, pool) @@ -112,13 +110,12 @@ func (p *PostgresDBService) DoneTasks() { } func (p *PostgresDBService) runWriters() { - var wgDBWriters sync.WaitGroup log.Info("Launching Beacon State Writers") log.Infof("Launching %d Beacon State Writers", p.workerNum) for i := 0; i < p.workerNum; i++ { - wgDBWriters.Add(1) + p.WgDBWriter.Add(1) go func(dbWriterID int) { - defer wgDBWriters.Done() + defer p.WgDBWriter.Done() wlogWriter := log.WithField("DBWriter", dbWriterID) writeBatch := pgx_v4.Batch{} loop: @@ -155,8 +152,13 @@ func (p *PostgresDBService) runWriters() { }(i) } - wgDBWriters.Wait() + p.WgDBWriter.Wait() + p.Close() + +} +func (p PostgresDBService) Close() { + p.psqlPool.Close() } type WriteTask struct { From d1f58ecbd900548b80ac4c3d394bebd77902cccc Mon Sep 17 00:00:00 2001 From: Tarun Date: Mon, 5 Dec 2022 13:00:34 +0000 Subject: [PATCH 39/62] Fix typo --- pkg/app/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index cd852a7..eec03cb 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -166,7 +166,7 @@ loop: if s.finishTasks > 0 { log.Infof("closing main routine") - s.Analyzers[0].DBClient.DoneTasks() // all the analyzers have the same db client + s.DBClient.DoneTasks() // all the analyzers have the same db client break loop } select { From caa6c788c84867922c86ea08f46f1247b982645f Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 16 Nov 2022 09:40:37 +0100 Subject: [PATCH 40/62] Add Reorg Events tracking --- pkg/analysis/events.go | 32 ++++++++++++++++++++++++-- pkg/postgresql/reorg.go | 47 +++++++++++++++++++++++++++++++++++++++ pkg/postgresql/service.go | 5 +++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 pkg/postgresql/reorg.go diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 9d8df97..33332fe 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -18,7 +18,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { } data := event.Data.(*api.HeadEvent) // cast to head event - log.Tracef("Received a new head event: slot %d", data.Slot) + log.Debugf("Received a new event: slot %d", data.Slot) // we only receive the block hash, get the new block newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, hex.EncodeToString(data.Block[:])) @@ -63,13 +63,13 @@ func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { timestamp := time.Now() log := b.log.WithField("routine", "attestation-event") - log.Tracef("Received a new event") if event.Data == nil { log.Errorf("attestation event does not contain anything") } data := event.Data.(*phase0.Attestation) // cast + log.Debugf("Received a new event: %s", data.Signature) // With the beacon committee we can identify the attesting validators // Will not track this for now // beaconCommittee := b.EpochData.GetBeaconCommittee(uint64(data.Data.Slot), uint64(data.Data.Index)) @@ -115,3 +115,31 @@ func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { log.Tracef("Finished processing event in %f seconds", time.Since(timestamp).Seconds()) } + +func (b *ClientLiveData) HandleReOrgEvent(event *api.Event) { + timestamp := time.Now() + log := b.log.WithField("routine", "reorg-event") + + if event.Data == nil { + return + } + + data := event.Data.(*api.ChainReorgEvent) // cast to head event + log.Debugf("New Reorg Evenet") + + baseParams := make([]interface{}, 0) + baseParams = append(baseParams, b.Eth2Provider.Label) + baseParams = append(baseParams, uint64(data.Slot)) + baseParams = append(baseParams, hex.EncodeToString(data.OldHeadBlock[:])) + baseParams = append(baseParams, hex.EncodeToString(data.NewHeadBlock[:])) + baseParams = append(baseParams, uint64(data.Depth)) + baseParams = append(baseParams, timestamp) + + writeTask := postgresql.WriteTask{ + QueryString: postgresql.InsertNewReorg, + Params: baseParams, + } + + b.DBClient.WriteChan <- writeTask // send task to be written + +} diff --git a/pkg/postgresql/reorg.go b/pkg/postgresql/reorg.go new file mode 100644 index 0000000..13faf97 --- /dev/null +++ b/pkg/postgresql/reorg.go @@ -0,0 +1,47 @@ +package postgresql + +/* + +This file together with the model, has all the needed methods to interact with the epoch_metrics table of the database + +*/ + +import ( + "context" + + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" +) + +var ( + CreateReorgTable = ` + CREATE TABLE IF NOT EXISTS t_reorg_metrics( + f_label TEXT, + f_slot INT, + f_old_head TEXT, + f_new_head TEXT, + f_depth INT, + f_timestamp TIMESTAMP, + CONSTRAINT PK_Reorg PRIMARY KEY (f_label,f_slot));` + + InsertNewReorg = ` + INSERT INTO t_reorg_metrics ( + f_label, + f_slot, + f_old_head, + f_new_head, + f_depth, + f_timestamp) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT DO NOTHING;` +) + +// in case the table did not exist +func (p *PostgresDBService) createReorgMetricsTable(ctx context.Context, pool *pgxpool.Pool) error { + // create the tables + _, err := pool.Exec(ctx, CreateReorgTable) + if err != nil { + return errors.Wrap(err, "error creating reorgs metrics table") + } + return nil +} diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index f2e2c54..1d0ad25 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -101,6 +101,11 @@ func (p *PostgresDBService) init(ctx context.Context, pool *pgxpool.Pool) error return err } + err = p.createReorgMetricsTable(ctx, pool) + if err != nil { + return err + } + return nil } From 7af1aefe0abb8fc0e4b020a0d7823eee35f2ece2 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 16 Nov 2022 09:40:54 +0100 Subject: [PATCH 41/62] Add reorg subscription --- pkg/app/service.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/app/service.go b/pkg/app/service.go index eec03cb..d89d4c9 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -22,6 +22,7 @@ var ( ) attestationMetric = "attestations" proposalMetric = "proposals" + reorgMetric = "reorgs" ) type AppService struct { @@ -110,6 +111,11 @@ func (s *AppService) Run() { s.RunAttestations() } + if item == reorgMetric { + wg.Add(1) + s.RunReOrgs() + } + if item == proposalMetric { log.Infof("initiating block proposal monitoring") wg.Add(1) @@ -134,6 +140,19 @@ func (s *AppService) RunAttestations() { } } +// Main routine: build block history and block proposals every 12 seconds +func (s *AppService) RunReOrgs() { + + // Subscribe to events from each client + for _, item := range s.Analyzers { + err := item.Eth2Provider.Api.Events(s.ctx, []string{"chain_reorg"}, item.HandleAttestationEvent) // every new head + if err != nil { + log.Panicf("failed to subscribe to reorg events: %s, label: %s", err, item.Eth2Provider.Label) + } + + } +} + // Main routine: build block history and block proposals every 12 seconds func (s *AppService) RunMainRoutine(wg *sync.WaitGroup) { defer wg.Done() From edf1e2be30bd95b57b615a95a65c25585976580b Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 16 Nov 2022 12:48:07 +0100 Subject: [PATCH 42/62] Fix logger --- pkg/analysis/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 33332fe..9173889 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -69,7 +69,7 @@ func (b *ClientLiveData) HandleAttestationEvent(event *api.Event) { } data := event.Data.(*phase0.Attestation) // cast - log.Debugf("Received a new event: %s", data.Signature) + log.Debugf("Received a new event: slot %d, committee: %d", uint64(data.Data.Slot), uint64(data.Data.Index)) // With the beacon committee we can identify the attesting validators // Will not track this for now // beaconCommittee := b.EpochData.GetBeaconCommittee(uint64(data.Data.Slot), uint64(data.Data.Index)) From 96f41d81b58595a7bc4a6cf5a18d46c6e01ee8de Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 09:49:36 +0000 Subject: [PATCH 43/62] Update libraries --- go.mod | 14 +++++++++++++- go.sum | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1ce8a0d..e0d4dc2 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,18 @@ require ( github.com/urfave/cli/v2 v2.16.3 ) +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect +) + require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.13.0 // indirect @@ -34,7 +46,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 64b87e7..3f90813 100644 --- a/go.sum +++ b/go.sum @@ -42,9 +42,11 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -78,9 +80,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -121,6 +125,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -245,6 +250,7 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -270,19 +276,27 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= @@ -426,12 +440,14 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -634,6 +650,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= From ec2d06e3dd884830b477b0f38e663ed064de12b3 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 09:56:54 +0000 Subject: [PATCH 44/62] Update libraries --- go.mod | 12 +----------- go.sum | 22 +++------------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index e0d4dc2..347f978 100644 --- a/go.mod +++ b/go.mod @@ -10,17 +10,7 @@ require ( github.com/urfave/cli/v2 v2.16.3 ) -require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect -) +require github.com/google/go-cmp v0.5.8 // indirect require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index 3f90813..01acd0d 100644 --- a/go.sum +++ b/go.sum @@ -42,11 +42,9 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -80,11 +78,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -125,7 +121,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -140,8 +135,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -250,7 +246,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -276,27 +271,19 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= @@ -438,8 +425,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -447,7 +434,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -650,8 +636,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= From 884e907679018f0547b383aacfacb31b180ebc7e Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 14:53:22 +0000 Subject: [PATCH 45/62] Fix head events bug --- pkg/analysis/events.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 9173889..72f98af 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -2,6 +2,7 @@ package analysis import ( "encoding/hex" + "fmt" "time" api "github.com/attestantio/go-eth2-client/api/v1" @@ -18,15 +19,15 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { } data := event.Data.(*api.HeadEvent) // cast to head event - log.Debugf("Received a new event: slot %d", data.Slot) + log.Infof("Received a new event: slot %d", data.Slot) // we only receive the block hash, get the new block - newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, hex.EncodeToString(data.Block[:])) + newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, fmt.Sprintf("%#x", data.Block)) // Track if there is any missing slot if b.CurrentHeadSlot != 0 && // we are not at the beginning of the run - newBlock.Bellatrix.Message.Slot-phase0.Slot(b.CurrentHeadSlot) > 1 { // there a gap bigger than 1 with the new head - for i := b.CurrentHeadSlot; i < uint64(newBlock.Bellatrix.Message.Slot); i++ { + data.Slot-phase0.Slot(b.CurrentHeadSlot) > 1 { // there a gap bigger than 1 with the new head + for i := b.CurrentHeadSlot + 1; i < uint64(data.Slot); i++ { params := make([]interface{}, 0) params = append(params, i) params = append(params, b.Eth2Provider.Label) From 780eba24040a565224a8d4e34aaaa602ecb183bd Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 14:55:23 +0000 Subject: [PATCH 46/62] Always store score --- pkg/analysis/analyzer.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index e6dbb55..3262ce7 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -59,11 +59,6 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { block, err := b.Eth2Provider.Api.BeaconBlockProposal(b.ctx, slot, randaoReveal, graffiti) // ask for block proposal blockTime := time.Since(snapshot).Seconds() // time to generate block - if err != nil { - log.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) - return - - } for i := range b.AttHistory { if i+32 < slot { // attestations can only reference 32 slots back delete(b.AttHistory, i) // remove old entries from the map @@ -76,14 +71,24 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { } } - // for now we just have Bellatrix - metrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) + metrics := postgresql.BlockMetricsModel{ + Slot: int(slot), + Label: b.Eth2Provider.Label, + Score: -1, + } if err != nil { - log.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) - return + log.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) + + } else { + // for now we just have Bellatrix + metrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) + if err != nil { + log.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) + return + } + log.Infof("Block Generation Time: %f", blockTime) + log.Infof("Metrics: %+v", metrics) } - log.Infof("Block Generation Time: %f", blockTime) - log.Infof("Metrics: %+v", metrics) // Store in DB params := make([]interface{}, 0) @@ -112,5 +117,4 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { // We block the update attestations as new head could impact attestations of the proposed block b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations - return } From a99e651b8294efec0d0f9f42b34a556380ee874f Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 14:55:54 +0000 Subject: [PATCH 47/62] Fix use same variable --- pkg/analysis/analyzer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 3262ce7..ccc5af1 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -81,7 +81,7 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { } else { // for now we just have Bellatrix - metrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) + metrics, err = b.BellatrixBlockMetrics(block.Bellatrix, blockTime) if err != nil { log.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) return From 6bfbf6335c14ae7a455bbff25d1931defc536fd3 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 14:56:50 +0000 Subject: [PATCH 48/62] Fix bug in postgresql --- pkg/postgresql/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/postgresql/service.go b/pkg/postgresql/service.go index 1d0ad25..6685943 100644 --- a/pkg/postgresql/service.go +++ b/pkg/postgresql/service.go @@ -136,7 +136,7 @@ func (p *PostgresDBService) runWriters() { wlogWriter.Tracef("Received new write task") writeBatch.Queue(task.QueryString, task.Params...) - if writeBatch.Len() > p.maxBatchQueue { + if writeBatch.Len() >= p.maxBatchQueue { wlogWriter.Tracef("Writing batch to database") err := p.ExecuteBatch(writeBatch) if err != nil { @@ -144,7 +144,7 @@ func (p *PostgresDBService) runWriters() { } writeBatch = pgx_v4.Batch{} } else { - log.Tracef("%d pending tasks to persist", MAX_BATCH_QUEUE-writeBatch.Len()) + log.Tracef("%d pending tasks to persist", p.maxBatchQueue-writeBatch.Len()) } case <-p.ctx.Done(): From 36baba79bacaa3dd1242609ad7d6f99717b12f9b Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 15:04:46 +0000 Subject: [PATCH 49/62] Fix bug in constructor --- pkg/analysis/analyzer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index ccc5af1..f37fd12 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -45,6 +45,7 @@ func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, tim log: log.WithField("label", label), EpochData: additional_structs.NewEpochData(client.Api), CurrentHeadSlot: 0, + ProcessNewHead: make(chan struct{}), }, nil } From 27299e62720c46d3f58082ac3ebf1b4690195906 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 15:05:23 +0000 Subject: [PATCH 50/62] Reorganize error checking --- pkg/analysis/events.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 72f98af..9a0321e 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -20,10 +20,19 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { data := event.Data.(*api.HeadEvent) // cast to head event log.Infof("Received a new event: slot %d", data.Slot) - + <-b.ProcessNewHead // wait for the block proposal to be done // we only receive the block hash, get the new block newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, fmt.Sprintf("%#x", data.Block)) + if newBlock == nil { + log.Errorf("the block is not available: %d", data.Slot) + return + } + if err != nil || newBlock == nil { + log.Errorf("could not request new block: %s", err) + return + } + // Track if there is any missing slot if b.CurrentHeadSlot != 0 && // we are not at the beginning of the run data.Slot-phase0.Slot(b.CurrentHeadSlot) > 1 { // there a gap bigger than 1 with the new head @@ -38,6 +47,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { b.DBClient.WriteChan <- writeTask // store } } + b.CurrentHeadSlot = uint64(data.Slot) params := make([]interface{}, 0) params = append(params, int(data.Slot)) params = append(params, b.Eth2Provider.Label) @@ -50,12 +60,8 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { // wait for the block proposal to be processed, otherwise the attestations could get mixed // with the proposal - <-b.ProcessNewHead - b.UpdateAttestations(*newBlock.Bellatrix.Message) // now update the attestations with the new head block in the chain - if err != nil { - log.Errorf("could not retrieve the head block: %s", err) - } + b.UpdateAttestations(*newBlock.Bellatrix.Message) // now update the attestations with the new head block in the chain } From 5543f8fe903b25b673a3de4c997c7642e0a02bf4 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 17:34:57 +0000 Subject: [PATCH 51/62] Add log --- pkg/analysis/history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analysis/history.go b/pkg/analysis/history.go index c03f0b0..af78258 100644 --- a/pkg/analysis/history.go +++ b/pkg/analysis/history.go @@ -12,7 +12,7 @@ import ( // This method receives a new head block and updates the attestation in the history // So, when a block is to be proposed, we can check the history to identify new votes func (b *ClientLiveData) UpdateAttestations(block bellatrix.BeaconBlock) { - + log.Tracef("updating attestations using block: %d", block.Slot) for _, attestation := range block.Body.Attestations { slot := attestation.Data.Slot From b404af176465c7523507c0fefeaa57cd8255e9d5 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 08:59:50 +0000 Subject: [PATCH 52/62] Do not return on error --- pkg/analysis/analyzer.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index f37fd12..2be4333 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -82,13 +82,15 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { } else { // for now we just have Bellatrix - metrics, err = b.BellatrixBlockMetrics(block.Bellatrix, blockTime) + newMetrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) if err != nil { log.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) - return + } else { + metrics = newMetrics + log.Infof("Block Generation Time: %f", blockTime) + log.Infof("Metrics: %+v", metrics) } - log.Infof("Block Generation Time: %f", blockTime) - log.Infof("Metrics: %+v", metrics) + } // Store in DB From 2e08b3534dbcefe234ee6e131ffafcc707cdbde3 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 09:29:36 +0000 Subject: [PATCH 53/62] Remove control chan --- pkg/analysis/analyzer.go | 2 +- pkg/analysis/events.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 2be4333..47f86a3 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -119,5 +119,5 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { b.DBClient.WriteChan <- writeTask // We block the update attestations as new head could impact attestations of the proposed block - b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations + // b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations } diff --git a/pkg/analysis/events.go b/pkg/analysis/events.go index 9a0321e..a46a009 100644 --- a/pkg/analysis/events.go +++ b/pkg/analysis/events.go @@ -20,7 +20,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { data := event.Data.(*api.HeadEvent) // cast to head event log.Infof("Received a new event: slot %d", data.Slot) - <-b.ProcessNewHead // wait for the block proposal to be done + // <-b.ProcessNewHead // wait for the block proposal to be done // we only receive the block hash, get the new block newBlock, err := b.Eth2Provider.Api.SignedBeaconBlock(b.ctx, fmt.Sprintf("%#x", data.Block)) @@ -32,6 +32,7 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { log.Errorf("could not request new block: %s", err) return } + b.UpdateAttestations(*newBlock.Bellatrix.Message) // now update the attestations with the new head block in the chain // Track if there is any missing slot if b.CurrentHeadSlot != 0 && // we are not at the beginning of the run @@ -61,8 +62,6 @@ func (b *ClientLiveData) HandleHeadEvent(event *api.Event) { // wait for the block proposal to be processed, otherwise the attestations could get mixed // with the proposal - b.UpdateAttestations(*newBlock.Bellatrix.Message) // now update the attestations with the new head block in the chain - } // When the node receives a new attestation, this function is tiggered From dc35e8f7243ec827e172d6a4dfdb46eac7776390 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 09:41:41 +0000 Subject: [PATCH 54/62] Add metrics exporter --- pkg/exporter/exporter.go | 107 +++++++++++++++++++++++++++++++++++++++ pkg/exporter/runner.go | 40 +++++++++++++++ pkg/exporter/service.go | 102 +++++++++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 pkg/exporter/exporter.go create mode 100644 pkg/exporter/runner.go create mode 100644 pkg/exporter/service.go diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go new file mode 100644 index 0000000..cec9d30 --- /dev/null +++ b/pkg/exporter/exporter.go @@ -0,0 +1,107 @@ +package exporter + +import ( + "context" + "time" + + "github.com/pkg/errors" +) + +var ( + // Assiciated with the Status of the exporter + ReadyStatus = "ready" + InitializedStatus = "initialized" + RunningStatus = "running" + ClosedStatus = "closed" +) + +type MetricsExporter struct { + // control variables + ctx context.Context + cancel context.CancelFunc + // TODO: add metrics to the export (time¿?) + name string // name the defines the exporter (Example Peer-Prometheus-Exporter) + status string + details string + + interval time.Duration + + initFunc func() // Initialization of the exporter + runFunc func() // function that will be executed in the running loop (the func needs to run a go routine) + closeFunc func() // might be empty if nothing is required +} + +func NewMetricsExporter(ctx context.Context, + name string, + details string, + initf func(), + runf func(), + closef func(), + interval time.Duration) (*MetricsExporter, error) { + // check if all the necesaty parameters where given + if len(name) <= 0 { + return nil, errors.New("no name was provided" + name) + } + // Extract context from the given one + mainctx, cancel := context.WithCancel(ctx) + + metricsExporter := &MetricsExporter{ + ctx: mainctx, + cancel: cancel, + name: name, + status: ReadyStatus, // Hardcoded to the ready status, Everytime a new metrics is created Run it + details: details, + interval: interval, + initFunc: initf, + runFunc: runf, + closeFunc: closef, + } + return metricsExporter, nil +} + +func (e MetricsExporter) Name() string { + return e.name +} + +func (e MetricsExporter) Details() string { + return e.details +} + +func (e MetricsExporter) Status() string { + return e.status +} + +func (e *MetricsExporter) Init() { + // Init loop for each of the Exporters + log.Infof("initializing exporter %s", e.name) + e.initFunc() + e.status = InitializedStatus +} + +func (e *MetricsExporter) Run() { + // Init loop for each of the Exporters + log.Infof("running exporter %s", e.name) + e.status = RunningStatus + // generate the ticker for the periodic metrics export + ticker := time.NewTicker(e.interval) + // run the export loop in a go routine (until ctx dies) + go func() { + for { + select { + case <-ticker.C: + // execute the routine designed + e.runFunc() + case <-e.ctx.Done(): + e.closeFunc() + return + } + } + }() +} + +func (e *MetricsExporter) Close() { + // Init loop for each of the Exporters + log.Infof("closing exporter %s", e.name) + e.cancel() + e.status = ClosedStatus +} diff --git a/pkg/exporter/runner.go b/pkg/exporter/runner.go new file mode 100644 index 0000000..ef43828 --- /dev/null +++ b/pkg/exporter/runner.go @@ -0,0 +1,40 @@ +package exporter + +import ( + "fmt" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + + // TODO: Just hardcoded, move to config + ExposedPort string = "9080" + EndpointUrl string = "metrics" + + MetricLoopInterval time.Duration = 15 * time.Second +) + +type PrometheusRunner struct { + ExposePort string + EndpointUrl string + RefreshInterval time.Duration +} + +func NewPrometheusRunner() PrometheusRunner { + return PrometheusRunner{ + ExposePort: ExposedPort, + EndpointUrl: EndpointUrl, + } +} + +func (c *PrometheusRunner) Start() error { + http.Handle("/"+c.EndpointUrl, promhttp.Handler()) + go func() { + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", c.ExposePort), nil)) + }() + + return nil +} diff --git a/pkg/exporter/service.go b/pkg/exporter/service.go new file mode 100644 index 0000000..0306f8e --- /dev/null +++ b/pkg/exporter/service.go @@ -0,0 +1,102 @@ +package exporter + +import ( + "context" + "time" + + "github.com/sirupsen/logrus" +) + +var ( + NewExporterCheckInterval = 2 * time.Minute + ModuleName = "EXPORTER" + log = logrus.WithField( + "module", ModuleName, + ) +) + +type ExporterService struct { + // TODO: adding new Exporters to the Service might bring up race conditions + // control variables + ctx context.Context + cancel context.CancelFunc + + PrometheusRunner PrometheusRunner + + ExporterRoutines map[string]Exporter + // TODO: Check if we need anything else +} + +// Basic unit of metrics export +type Exporter interface { + Name() string + Init() + Run() + Close() + Status() string + Details() string +} + +// Creates and initialized the exporter service +// where any module can create a dedicated exporter +func NewExporterService(ctx context.Context) *ExporterService { + mainctx, cancel := context.WithCancel(ctx) + + exporterSrv := &ExporterService{ + ctx: mainctx, + cancel: cancel, + PrometheusRunner: NewPrometheusRunner(), + ExporterRoutines: make(map[string]Exporter), + } + return exporterSrv +} + +func (s *ExporterService) Run() { + log.Info("running the exporter service") + // start the prometheus endpoint + s.PrometheusRunner.Start() + // run the exporters check in a go routine (until shutdown) + ticker := time.NewTicker(NewExporterCheckInterval) + go func() { + // iteration loop + for { + select { + case <-s.ctx.Done(): + log.Info("context of exporter was close, closing exporting service") + return + case <-ticker.C: + for _, export := range s.ExporterRoutines { + // check if the metrics are up exporting + // run them otherwise + status := export.Status() + // Check if the exporter is in ready state to initialize it + // and run it + if status == "ready" { + // get details of the + log.Debug(export.Details()) + export.Init() + export.Run() + log.Debug(export.Status()) + } + } + } + } + }() + +} + +// add new exporter to the list of exporters +func (s *ExporterService) AddNewExporter(exptr Exporter) { + log.Infof("adding exporter %s to the exporter service", exptr.Name()) + // TODO: - might generate a race condition + s.ExporterRoutines[exptr.Name()] = exptr +} + +// Close all the running metrics exposers +func (s *ExporterService) Close() { + log.Infof("closing %d metrics exporters", len(s.ExporterRoutines)) + for name, export := range s.ExporterRoutines { + log.Infof("closing exporter %s", name) + export.Close() + } +} From f859e3334183717ea0ecf7cb4cf459bc752bec97 Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 09:42:10 +0000 Subject: [PATCH 55/62] Add prometheus metrics from app --- pkg/app/prometheus_exporter.go | 55 ++++++++++++++++++++++++++++++++++ pkg/app/prometheus_metrics.go | 16 ++++++++++ 2 files changed, 71 insertions(+) create mode 100644 pkg/app/prometheus_exporter.go create mode 100644 pkg/app/prometheus_metrics.go diff --git a/pkg/app/prometheus_exporter.go b/pkg/app/prometheus_exporter.go new file mode 100644 index 0000000..75e9799 --- /dev/null +++ b/pkg/app/prometheus_exporter.go @@ -0,0 +1,55 @@ +package app + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/tdahar/eth-cl-live-metrics/pkg/exporter" +) + +// ServeMetrics: +// This method will serve the global peerstore values to the +// local prometheus instance. +func (s *AppService) ServeMetrics() { + // Generate new ticker + + // generate the Prometheus exporter + exptr, _ := exporter.NewMetricsExporter( + s.ctx, + "Client-Metrics-Prometheus", + "Expose in Prometheus the metrics of the clients", + s.initClientsPrometheusMetrics, + s.runClientsPrometheusMetrics, + func() {}, + exporter.MetricLoopInterval, + ) + // add the new exptr to the ExporterService + s.ExporterService.AddNewExporter(exptr) + +} + +func (s *AppService) initClientsPrometheusMetrics() { + // register variables + prometheus.MustRegister(ProposalsUp) +} +func (s *AppService) runClientsPrometheusMetrics() { + ticker := time.NewTicker(exporter.MetricLoopInterval) + // routine to loop + go func() { + for { + select { + case <-ticker.C: + + for _, item := range s.Analyzers { + ProposalsUp.WithLabelValues(item.Eth2Provider.Label).Set(float64(item.Monitoring.ProposalStatus)) + } + + case <-s.ctx.Done(): + log.Info("Closing the prometheus metrics export service") + // closing the routine in a ordened way + ticker.Stop() + return + } + } + }() +} diff --git a/pkg/app/prometheus_metrics.go b/pkg/app/prometheus_metrics.go new file mode 100644 index 0000000..7ae1c97 --- /dev/null +++ b/pkg/app/prometheus_metrics.go @@ -0,0 +1,16 @@ +package app + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// List of metrics that we are going to export +var ( + ProposalsUp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "clients", + Name: "proposals-up", + Help: "Block Proposals up", + }, + []string{"controlclients"}, + ) +) From a800db462ddcf01d1a87cb36a3604359a16cf36f Mon Sep 17 00:00:00 2001 From: Tarun Date: Wed, 7 Dec 2022 09:43:07 +0000 Subject: [PATCH 56/62] Add exporter service --- pkg/analysis/analyzer.go | 10 ++++++++++ pkg/app/service.go | 20 +++++++++++--------- pkg/cmd/analyzer_cmd.go | 4 ++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/pkg/analysis/analyzer.go b/pkg/analysis/analyzer.go index 47f86a3..8785190 100644 --- a/pkg/analysis/analyzer.go +++ b/pkg/analysis/analyzer.go @@ -28,6 +28,7 @@ type ClientLiveData struct { DBClient *postgresql.PostgresDBService EpochData additional_structs.EpochStructs CurrentHeadSlot uint64 + Monitoring MonitoringMetrics } func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, timeout time.Duration, dbClient *postgresql.PostgresDBService) (*ClientLiveData, error) { @@ -46,6 +47,7 @@ func NewBlockAnalyzer(ctx context.Context, label string, cliEndpoint string, tim EpochData: additional_structs.NewEpochData(client.Api), CurrentHeadSlot: 0, ProcessNewHead: make(chan struct{}), + Monitoring: MonitoringMetrics{}, }, nil } @@ -79,13 +81,16 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { } if err != nil { log.Errorf("error requesting block from %s: %s", b.Eth2Provider.Label, err) + b.Monitoring.ProposalStatus = 0 } else { // for now we just have Bellatrix newMetrics, err := b.BellatrixBlockMetrics(block.Bellatrix, blockTime) if err != nil { log.Errorf("error analyzing block from %s: %s", b.Eth2Provider.Label, err) + b.Monitoring.ProposalStatus = 0 } else { + b.Monitoring.ProposalStatus = 1 metrics = newMetrics log.Infof("Block Generation Time: %f", blockTime) log.Infof("Metrics: %+v", metrics) @@ -115,9 +120,14 @@ func (b *ClientLiveData) ProposeNewBlock(slot phase0.Slot) { QueryString: postgresql.InsertNewScore, Params: params, } + b.Monitoring.ProposalStatus = 1 b.DBClient.WriteChan <- writeTask // We block the update attestations as new head could impact attestations of the proposed block // b.ProcessNewHead <- struct{}{} // Allow the new head to update attestations } + +type MonitoringMetrics struct { + ProposalStatus int +} diff --git a/pkg/app/service.go b/pkg/app/service.go index d89d4c9..797dbff 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -12,6 +12,7 @@ import ( "github.com/sirupsen/logrus" "github.com/tdahar/eth-cl-live-metrics/pkg/analysis" "github.com/tdahar/eth-cl-live-metrics/pkg/chain_stats" + "github.com/tdahar/eth-cl-live-metrics/pkg/exporter" "github.com/tdahar/eth-cl-live-metrics/pkg/postgresql" ) @@ -26,15 +27,16 @@ var ( ) type AppService struct { - ctx context.Context - cancel context.CancelFunc - Analyzers []*analysis.ClientLiveData - initTime time.Time - ChainTime chain_stats.ChainTime - HeadSlot phase0.Slot - Metrics []string - finishTasks int32 - DBClient *postgresql.PostgresDBService + ctx context.Context + cancel context.CancelFunc + Analyzers []*analysis.ClientLiveData + initTime time.Time + ChainTime chain_stats.ChainTime + HeadSlot phase0.Slot + Metrics []string + finishTasks int32 + DBClient *postgresql.PostgresDBService + ExporterService exporter.ExporterService } func NewAppService(pCtx context.Context, diff --git a/pkg/cmd/analyzer_cmd.go b/pkg/cmd/analyzer_cmd.go index 7124160..3c9d298 100644 --- a/pkg/cmd/analyzer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/tdahar/eth-cl-live-metrics/pkg/app" + "github.com/tdahar/eth-cl-live-metrics/pkg/exporter" "github.com/tdahar/eth-cl-live-metrics/pkg/utils" cli "github.com/urfave/cli/v2" ) @@ -93,6 +94,9 @@ func LaunchBlockAnalyzer(c *cli.Context) error { bnEndpoints := strings.Split(c.String("bn-endpoints"), ",") dbEndpoint := c.String("db-endpoint") + exportService := exporter.NewExporterService(c.Context) + go exportService.Run() + service, err := app.NewAppService(c.Context, bnEndpoints, dbEndpoint, dbWorkers, metrics) if err != nil { log.Fatal("could not start app: %s", err.Error()) From bef71032129418610da33dd37acdf20087ffb87b Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 10:16:46 +0000 Subject: [PATCH 57/62] Update libraries --- go.mod | 12 ++++++++++++ go.sum | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/go.mod b/go.mod index 347f978..fa405ed 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.17 require ( github.com/attestantio/go-eth2-client v0.11.7 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.14.0 github.com/rs/zerolog v1.28.0 github.com/sirupsen/logrus v1.9.0 github.com/urfave/cli/v2 v2.16.3 @@ -12,6 +13,17 @@ require ( require github.com/google/go-cmp v0.5.8 // indirect +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect +) + require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.13.0 // indirect diff --git a/go.sum b/go.sum index 01acd0d..a1ade65 100644 --- a/go.sum +++ b/go.sum @@ -42,9 +42,11 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -78,9 +80,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -121,6 +125,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -246,6 +251,7 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -271,19 +277,27 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4= github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= @@ -434,6 +448,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -444,6 +459,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -491,6 +507,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -636,6 +653,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= From 8f80a7b999599bf34acd612e6bdb2f51c7a58e45 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 10:17:45 +0000 Subject: [PATCH 58/62] Change metrics interval --- pkg/exporter/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/exporter/service.go b/pkg/exporter/service.go index 0306f8e..6d1e33b 100644 --- a/pkg/exporter/service.go +++ b/pkg/exporter/service.go @@ -8,7 +8,7 @@ import ( ) var ( - NewExporterCheckInterval = 2 * time.Minute + NewExporterCheckInterval = 15 * time.Second ModuleName = "EXPORTER" log = logrus.WithField( "module", ModuleName, From 999a854aa4af5f0d0914f289862316d10f2c8f53 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 10:19:21 +0000 Subject: [PATCH 59/62] Launch exporter service --- pkg/cmd/analyzer_cmd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/analyzer_cmd.go b/pkg/cmd/analyzer_cmd.go index 3c9d298..8435e67 100644 --- a/pkg/cmd/analyzer_cmd.go +++ b/pkg/cmd/analyzer_cmd.go @@ -95,9 +95,9 @@ func LaunchBlockAnalyzer(c *cli.Context) error { dbEndpoint := c.String("db-endpoint") exportService := exporter.NewExporterService(c.Context) - go exportService.Run() + exportService.Run() - service, err := app.NewAppService(c.Context, bnEndpoints, dbEndpoint, dbWorkers, metrics) + service, err := app.NewAppService(c.Context, bnEndpoints, dbEndpoint, dbWorkers, metrics, exportService) if err != nil { log.Fatal("could not start app: %s", err.Error()) } From 268bb6cdf9842453b82a18b04e61ac1391cddea6 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 10:21:27 +0000 Subject: [PATCH 60/62] Add metrics to service --- pkg/app/service.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/app/service.go b/pkg/app/service.go index 797dbff..f185582 100644 --- a/pkg/app/service.go +++ b/pkg/app/service.go @@ -36,14 +36,15 @@ type AppService struct { Metrics []string finishTasks int32 DBClient *postgresql.PostgresDBService - ExporterService exporter.ExporterService + ExporterService *exporter.ExporterService } func NewAppService(pCtx context.Context, bnEndpoints []string, dbEndpooint string, dbWorkers int, - metrics []string) (*AppService, error) { + metrics []string, + exporterService *exporter.ExporterService) (*AppService, error) { ctx, cancel := context.WithCancel(pCtx) batchLen := len(bnEndpoints) @@ -97,13 +98,17 @@ func NewAppService(pCtx context.Context, ChainTime: chain_stats.ChainTime{ GenesisTime: genesis, }, - Metrics: metrics, - DBClient: dbClient, + Metrics: metrics, + DBClient: dbClient, + ExporterService: exporterService, }, nil } // Main routine: build block history and block proposals every 12 seconds func (s *AppService) Run() { + + s.ServeMetrics() + defer s.cancel() var wg sync.WaitGroup for _, item := range s.Metrics { From 0630c1aa6800db59a3ced8f8913429c156a32feb Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 10:21:48 +0000 Subject: [PATCH 61/62] Change prometheus metrics name --- pkg/app/prometheus_metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/app/prometheus_metrics.go b/pkg/app/prometheus_metrics.go index 7ae1c97..6f3a863 100644 --- a/pkg/app/prometheus_metrics.go +++ b/pkg/app/prometheus_metrics.go @@ -8,9 +8,9 @@ import ( var ( ProposalsUp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "clients", - Name: "proposals-up", + Name: "proposals_up", Help: "Block Proposals up", }, - []string{"controlclients"}, + []string{"controldist"}, ) ) From 7c75cf90679b07c1600846c7a4b51a57fcf01390 Mon Sep 17 00:00:00 2001 From: Tarun Date: Fri, 9 Dec 2022 10:22:04 +0000 Subject: [PATCH 62/62] Add metrics loggers --- pkg/app/prometheus_exporter.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/app/prometheus_exporter.go b/pkg/app/prometheus_exporter.go index 75e9799..bac8add 100644 --- a/pkg/app/prometheus_exporter.go +++ b/pkg/app/prometheus_exporter.go @@ -33,15 +33,17 @@ func (s *AppService) initClientsPrometheusMetrics() { prometheus.MustRegister(ProposalsUp) } func (s *AppService) runClientsPrometheusMetrics() { + log := log.WithField("routine", "prometheus-metrics") ticker := time.NewTicker(exporter.MetricLoopInterval) // routine to loop go func() { for { select { case <-ticker.C: - + log.Infof("prometheus summary:") for _, item := range s.Analyzers { ProposalsUp.WithLabelValues(item.Eth2Provider.Label).Set(float64(item.Monitoring.ProposalStatus)) + log.Infof("Endpoint: %s, status: %d", item.Eth2Provider.Label, item.Monitoring.ProposalStatus) } case <-s.ctx.Done():