diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 57c2e5062..d813b5557 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,6 @@ jobs: strategy: matrix: go_version: - - ~1.16 - ^1.17 steps: - name: Checkout @@ -83,7 +82,7 @@ jobs: - name: Run linter uses: golangci/golangci-lint-action@v2 with: - version: v1.43.0 + version: v1.44.2 docker: name: Docker diff --git a/.golangci.toml b/.golangci.toml index 97f84a529..353b1d9da 100644 --- a/.golangci.toml +++ b/.golangci.toml @@ -9,4 +9,4 @@ format = "colored-line-number" [linters] enable-all = true -disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct"] +disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"] diff --git a/Makefile b/Makefile index c54a228ee..354cdec74 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) IMAGE_NAME := mtg APP_NAME := $(IMAGE_NAME) -GOLANGCI_LINT_VERSION := v1.43.0 +GOLANGCI_LINT_VERSION := v1.44.2 VERSION_GO := $(shell go version) VERSION_DATE := $(shell date -Ru) diff --git a/README.md b/README.md index 1b8a433a1..0740b7585 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mtg -Highly-opionated (ex-bullshit-free) MTPROTO proxy for +Highly-opinionated (ex-bullshit-free) MTPROTO proxy for [Telegram](https://telegram.org/). [![CI](https://github.com/9seconds/mtg/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/9seconds/mtg/actions/workflows/ci.yaml) @@ -224,6 +224,16 @@ $ mtg generate-secret --hex google.com ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d ``` +equivalent commands with docker: + +```console +$ docker run --rm nineseconds/mtg:2 generate-secret google.com +7ibaERuTSGPH1RdztfYnN4tnb29nbGUuY29t + +$ docker run --rm nineseconds/mtg:2 generate-secret --hex google.com +ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d +``` + This secret is a keystone for a proxy and your password for a client. You need to keep it secured. @@ -324,7 +334,7 @@ $ sudo systemctl start mtg or you can run a docker image ```console -docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --restart=unless-stopped nineseconds/mtg:2 +docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --name mtg-proxy --restart=unless-stopped nineseconds/mtg:2 ``` where _443_ is a host port (a port you want to connect to from a @@ -353,6 +363,12 @@ $ mtg access /etc/mtg.toml } ``` +or if you are using docker: + +```console +$ docker exec mtg-proxy /mtg access /config.toml +``` + ## Metrics Out of the box, mtg works with @@ -367,6 +383,7 @@ Here goes a list of metrics with their types but without a prefix. | client_connections | gauge | `ip_family` | Count of processing client connections. | | telegram_connections | gauge | `telegram_ip`, `dc` | Count of connections to Telegram servers. | | domain_fronting_connections | gauge | `ip_family` | Count of connections to fronting domain. | +| iplist_size | gauge | `ip_list` | A size of either allowlist or blocklist in use. | | telegram_traffic | counter | `telegram_ip`, `dc`, `direction` | Count of bytes, transmitted to/from Telegram. | | domain_fronting_traffic | counter | `direction` | Count of bytes, transmitted to/from fronting domain. | | domain_fronting | counter | – | Count of domain fronting events. | @@ -382,3 +399,4 @@ Tag meaning: | dc | | A number of the Telegram DC for a connection. | | telegram_ip | | IP address of the Telegram server. | | direction | `to_client`, `from_client` | A direction of the traffic flow. | +| ip_list | `allowlist`, `blocklist` | A type of the IP list. | diff --git a/events/event_stream.go b/events/event_stream.go index cc971a6cc..c847bf29c 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -102,6 +102,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob observer.EventConcurrencyLimited(typedEvt) case mtglib.EventReplayAttack: observer.EventReplayAttack(typedEvt) + case mtglib.EventIPListSize: + observer.EventIPListSize(typedEvt) } } } diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 6989c77a1..da55f2425 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -204,6 +204,27 @@ func (suite *EventStreamTestSuite) TestEventReplayAttack() { time.Sleep(100 * time.Millisecond) } +func (suite *EventStreamTestSuite) TestEventIPListSize() { + evt := mtglib.NewEventIPListSize(10, true) + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventIPListSize", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught, ok := args.Get(0).(mtglib.EventIPListSize) + + suite.True(ok) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + suite.Equal(evt.Size, caught.Size) + suite.Equal(evt.IsBlockList, caught.IsBlockList) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + func (suite *EventStreamTestSuite) TearDownTest() { suite.stream.Shutdown() suite.ctxCancel() diff --git a/events/init.go b/events/init.go index f754ede13..94804dcc4 100644 --- a/events/init.go +++ b/events/init.go @@ -53,6 +53,9 @@ type Observer interface { // EventReplayAttack reacts on incoming mtglib.EventReplayAttack event. EventReplayAttack(mtglib.EventReplayAttack) + // EventIPListSize reacts on incoming mtglib.EventIPListSize + EventIPListSize(mtglib.EventIPListSize) + // Shutdown stop observer. Default event stream guarantees: // 1. If shutdown is executed, it is executed only once // 2. Observer won't receieve any new message after this diff --git a/events/init_test.go b/events/init_test.go index 1b2e0cd9f..3fc4ff0a4 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -41,6 +41,10 @@ func (o *ObserverMock) EventReplayAttack(evt mtglib.EventReplayAttack) { o.Called(evt) } +func (o *ObserverMock) EventIPListSize(evt mtglib.EventIPListSize) { + o.Called(evt) +} + func (o *ObserverMock) Shutdown() { o.Called() } diff --git a/events/multi_observer.go b/events/multi_observer.go index eeeacae81..2b529cde9 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -130,6 +130,21 @@ func (m multiObserver) EventReplayAttack(evt mtglib.EventReplayAttack) { wg.Wait() } +func (m multiObserver) EventIPListSize(evt mtglib.EventIPListSize) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventIPListSize(evt) + }(v) + } + + wg.Wait() +} + func (m multiObserver) Shutdown() { for _, v := range m.observers { v.Shutdown() diff --git a/events/noop.go b/events/noop.go index 0582d9f69..68ee0734e 100644 --- a/events/noop.go +++ b/events/noop.go @@ -25,6 +25,7 @@ func (n noopObserver) EventFinish(_ mtglib.EventFinish) func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} func (n noopObserver) EventReplayAttack(_ mtglib.EventReplayAttack) {} +func (n noopObserver) EventIPListSize(_ mtglib.EventIPListSize) {} func (n noopObserver) Shutdown() {} // NewNoopObserver creates an observer which discards each message. diff --git a/events/noop_test.go b/events/noop_test.go index 369a0610f..ebd04a9ad 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -27,6 +27,7 @@ func (suite *NoopTestSuite) SetupSuite() { "concurrency-limited": mtglib.NewEventConcurrencyLimited(), "ip-blacklisted": mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")), "replay-attack": mtglib.NewEventReplayAttack("connID"), + "ip-list-size": mtglib.NewEventIPListSize(10, true), } suite.ctx = context.Background() } @@ -65,6 +66,8 @@ func (suite *NoopTestSuite) TestObserver() { observer.EventIPBlocklisted(typedEvt) case mtglib.EventReplayAttack: observer.EventReplayAttack(typedEvt) + case mtglib.EventIPListSize: + observer.EventIPListSize(typedEvt) } }) } diff --git a/example.config.toml b/example.config.toml index b4862f679..bdec7b73f 100644 --- a/example.config.toml +++ b/example.config.toml @@ -170,7 +170,7 @@ download-concurrency = 2 # You can provider links here (starts with https:// or http://) or # path to a local file, but in this case it should be absolute. urls = [ - # "https://iplists.firehol.org/files/firehol_level1.netset", + "https://iplists.firehol.org/files/firehol_level1.netset", # "/local.file" ] # How often do we need to update a blocklist set. diff --git a/go.mod b/go.mod index ce52f2a4d..a245cd250 100644 --- a/go.mod +++ b/go.mod @@ -4,33 +4,35 @@ go 1.17 require ( github.com/OneOfOne/xxhash v1.2.8 - github.com/alecthomas/kong v0.2.19 - github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a + github.com/alecthomas/kong v0.5.0 + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/d4l3k/messagediff v1.2.1 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gotd/td v0.34.0 github.com/jarcoal/httpmock v1.0.8 - github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e github.com/mccutchen/go-httpbin v1.1.1 - github.com/panjf2000/ants/v2 v2.4.7 + github.com/panjf2000/ants/v2 v2.4.8 github.com/pelletier/go-toml v1.9.4 - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.12.1 github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rs/zerolog v1.26.0 + github.com/rs/zerolog v1.26.1 github.com/smira/go-statsd v1.3.2 github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 - golang.org/x/crypto v0.0.0-20211202192323-5770296d904e + golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect - golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 + golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 google.golang.org/protobuf v1.27.1 // indirect ) -require github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b +require ( + github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a + github.com/yl2chen/cidranger v1.0.2 +) require ( github.com/beorn7/perks v1.0.1 // indirect @@ -44,7 +46,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf // indirect + github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8 // indirect github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 6e70dd9e7..b4021d651 100644 --- a/go.sum +++ b/go.sum @@ -37,15 +37,17 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/alecthomas/kong v0.2.19 h1:qBDfByO5XgWUXyNB4D6OOhGh5Z1eNOwWayDPQJFNWdc= -github.com/alecthomas/kong v0.2.19/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ= +github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE= +github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -163,14 +165,13 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.4.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e h1:1wAVuGu1c+lsdaOPQN+9xoP9+gaIMJV6H0ehGc+K5iA= -github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -191,10 +192,11 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/panjf2000/ants/v2 v2.4.7 h1:MZnw2JRyTJxFwtaMtUJcwE618wKD04POWk2gwwP4E2M= -github.com/panjf2000/ants/v2 v2.4.7/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= +github.com/panjf2000/ants/v2 v2.4.8 h1:JgTbolX6K6RreZ4+bfctI0Ifs+3mrE5BIHudQxUDQ9k= +github.com/panjf2000/ants/v2 v2.4.8/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= @@ -209,8 +211,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -231,8 +234,8 @@ github.com/quasilyte/go-ruleguard/dsl v0.3.2/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQP github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE= -github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -245,21 +248,23 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf h1:7PflaKRtU4np/epFxRXlFhlzLXZzKFrH5/I4so5Ove0= github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf/go.mod h1:CLUSJbazqETbaR+i0YAhXBICV9TrKH93pziccMhmhpM= -github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b h1:6J/38A0Xmdnjacfie0Udams7OP/GdoExyTipKwuQWjY= -github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b/go.mod h1:7NloQcrxaZYKURWph5HLxVDlIwMHJXCPkeWPtpftsIg= +github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8 h1:iYc+JnXtzv6sdMx9Q7OTKkDAn7FhDPDogcjeSfEQcLY= +github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8/go.mod h1:CLUSJbazqETbaR+i0YAhXBICV9TrKH93pziccMhmhpM= +github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a h1:BOqgJ4jku0LHPDoR51RD8Mxmo0LHxCzJT/M9MemYdHo= +github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a/go.mod h1:7NloQcrxaZYKURWph5HLxVDlIwMHJXCPkeWPtpftsIg= github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe h1:gMWxZxBFRAXqoGkwkYlPX2zvyyKNWJpxOxCrjqJkm5A= github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe/go.mod h1:WgqbSEmUYSjEV3B1qmee/PpP2NYEz4bL9/+mF1ma+s4= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -285,8 +290,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -350,6 +356,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= @@ -411,8 +418,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index a53e1d72b..ae06a80a9 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -1,6 +1,7 @@ package cli import ( + "context" "fmt" "net" "net/url" @@ -85,7 +86,10 @@ func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache { ) } -func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Network) (mtglib.IPBlocklist, error) { +func makeIPBlocklist(conf config.ListConfig, + logger mtglib.Logger, + ntw mtglib.Network, + updateCallback ipblocklist.FireholUpdateCallback) (mtglib.IPBlocklist, error) { if !conf.Enabled.Get(false) { return ipblocklist.NewNoop(), nil } @@ -105,11 +109,14 @@ func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Ne ntw, conf.DownloadConcurrency.Get(1), remoteURLs, - localFiles) + localFiles, + updateCallback) if err != nil { return nil, fmt.Errorf("incorrect parameters for firehol: %w", err) } + go firehol.Run(conf.UpdateEach.Get(ipblocklist.DefaultFireholUpdateEach)) + return firehol, nil } @@ -157,12 +164,23 @@ func runProxy(conf *config.Config, version string) error { // nolint: funlen logger.BindJSON("configuration", conf.String()).Debug("configuration") + eventStream, err := makeEventStream(conf, logger) + if err != nil { + return fmt.Errorf("cannot build event stream: %w", err) + } + ntw, err := makeNetwork(conf, version) if err != nil { return fmt.Errorf("cannot build network: %w", err) } - blocklist, err := makeIPBlocklist(conf.Defense.Blocklist, logger, ntw) + blocklist, err := makeIPBlocklist( + conf.Defense.Blocklist, + logger.Named("blocklist"), + ntw, + func(ctx context.Context, size int) { + eventStream.Send(ctx, mtglib.NewEventIPListSize(size, true)) + }) if err != nil { return fmt.Errorf("cannot build ip blocklist: %w", err) } @@ -170,19 +188,20 @@ func runProxy(conf *config.Config, version string) error { // nolint: funlen var whitelist mtglib.IPBlocklist if conf.Defense.Allowlist.Enabled.Get(false) { - whlist, err := makeIPBlocklist(conf.Defense.Allowlist, logger, ntw) + whlist, err := makeIPBlocklist( + conf.Defense.Allowlist, + logger.Named("allowlist"), + ntw, + func(ctx context.Context, size int) { + eventStream.Send(ctx, mtglib.NewEventIPListSize(size, false)) + }) if err != nil { - return fmt.Errorf("cannot build ip blocklist: %w", err) + return fmt.Errorf("cannot build ip allowlist: %w", err) } whitelist = whlist } - eventStream, err := makeEventStream(conf, logger) - if err != nil { - return fmt.Errorf("cannot build event stream: %w", err) - } - opts := mtglib.ProxyOpts{ Logger: logger, Network: ntw, diff --git a/internal/testlib/mtglib_network_mock.go b/internal/testlib/mtglib_network_mock.go index 97bd69a54..a6d08bfdb 100644 --- a/internal/testlib/mtglib_network_mock.go +++ b/internal/testlib/mtglib_network_mock.go @@ -15,16 +15,16 @@ type MtglibNetworkMock struct { func (m *MtglibNetworkMock) Dial(network, address string) (essentials.Conn, error) { args := m.Called(network, address) - return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck + return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert } func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) { args := m.Called(ctx, network, address) - return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck + return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert } func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client { - return m.Called(dialFunc).Get(0).(*http.Client) + return m.Called(dialFunc).Get(0).(*http.Client) // nolint: forcetypeassert } diff --git a/internal/testlib/net_conn_mock.go b/internal/testlib/net_conn_mock.go index 476fe0923..f4c455b9f 100644 --- a/internal/testlib/net_conn_mock.go +++ b/internal/testlib/net_conn_mock.go @@ -36,11 +36,11 @@ func (n *EssentialsConnMock) CloseWrite() error { } func (n *EssentialsConnMock) LocalAddr() net.Addr { - return n.Called().Get(0).(net.Addr) + return n.Called().Get(0).(net.Addr) // nolint: forcetypeassert } func (n *EssentialsConnMock) RemoteAddr() net.Addr { - return n.Called().Get(0).(net.Addr) + return n.Called().Get(0).(net.Addr) // nolint: forcetypeassert } func (n *EssentialsConnMock) SetDeadline(t time.Time) error { diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 41726a71b..d6fa3e883 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -12,17 +12,20 @@ import ( "github.com/9seconds/mtg/v2/ipblocklist/files" "github.com/9seconds/mtg/v2/mtglib" - "github.com/kentik/patricia" - "github.com/kentik/patricia/bool_tree" "github.com/panjf2000/ants/v2" + "github.com/yl2chen/cidranger" ) -const ( - fireholIPv4DefaultCIDR = 32 - fireholIPv6DefaultCIDR = 128 +var ( + fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) + + fireholIPv4DefaultCIDR = net.CIDRMask(32, 32) // nolint: gomnd + fireholIPv6DefaultCIDR = net.CIDRMask(128, 128) // nolint: gomnd ) -var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) +// FireholUpdateCallback defines a signature of the callback that has to be +// execute when ip list is updated. +type FireholUpdateCallback func(context.Context, int) // Firehol is IPBlocklist which uses lists from FireHOL: // https://iplists.firehol.org/ @@ -43,11 +46,12 @@ type Firehol struct { logger mtglib.Logger updateMutex sync.RWMutex + updateCallback FireholUpdateCallback + ranger cidranger.Ranger + blocklists []files.File workerPool *ants.Pool - treeV4 *bool_tree.TreeV4 - treeV6 *bool_tree.TreeV6 } // Shutdown stop a background update process. @@ -64,11 +68,12 @@ func (f *Firehol) Contains(ip net.IP) bool { f.updateMutex.RLock() defer f.updateMutex.RUnlock() - if ip4 := ip.To4(); ip4 != nil { - return f.containsIPv4(ip4) + ok, err := f.ranger.Contains(ip) + if err != nil { + f.logger.BindStr("ip", ip.String()).DebugError("Cannot check if ip is present", err) } - return f.containsIPv6(ip.To16()) + return ok && err == nil } // Run starts a background update process. @@ -103,26 +108,6 @@ func (f *Firehol) Run(updateEach time.Duration) { } } -func (f *Firehol) containsIPv4(addr net.IP) bool { - ip := patricia.NewIPv4AddressFromBytes(addr, 32) // nolint: gomnd - - if ok, _ := f.treeV4.FindDeepestTag(ip); ok { - return true - } - - return false -} - -func (f *Firehol) containsIPv6(addr net.IP) bool { - ip := patricia.NewIPv6Address(addr, 128) // nolint: gomnd - - if ok, _ := f.treeV6.FindDeepestTag(ip); ok { - return true - } - - return false -} - func (f *Firehol) update() { ctx, cancel := context.WithCancel(f.ctx) defer cancel() @@ -130,9 +115,8 @@ func (f *Firehol) update() { wg := &sync.WaitGroup{} wg.Add(len(f.blocklists)) - treeMutex := &sync.Mutex{} - v4tree := bool_tree.NewTreeV4() - v6tree := bool_tree.NewTreeV6() + mutex := &sync.Mutex{} + ranger := cidranger.NewPCTrieRanger() for _, v := range f.blocklists { go func(file files.File) { @@ -149,7 +133,7 @@ func (f *Firehol) update() { defer fileContent.Close() - if err := f.updateFromFile(treeMutex, v4tree, v6tree, bufio.NewScanner(fileContent)); err != nil { + if err := f.updateFromFile(mutex, ranger, bufio.NewScanner(fileContent)); err != nil { logger.WarningError("update has failed", err) } }(v) @@ -160,15 +144,17 @@ func (f *Firehol) update() { f.updateMutex.Lock() defer f.updateMutex.Unlock() - f.treeV4 = v4tree - f.treeV6 = v6tree + f.ranger = ranger - f.logger.Info("blocklist was updated") + if f.updateCallback != nil { + f.updateCallback(ctx, ranger.Len()) + } + + f.logger.Info("ip list was updated") } func (f *Firehol) updateFromFile(mutex sync.Locker, - v4tree *bool_tree.TreeV4, - v6tree *bool_tree.TreeV6, + ranger cidranger.Ranger, scanner *bufio.Scanner) error { for scanner.Scan() { text := scanner.Text() @@ -179,12 +165,18 @@ func (f *Firehol) updateFromFile(mutex sync.Locker, continue } - ip, cidr, err := f.updateParseLine(text) + ipnet, err := f.updateParseLine(text) if err != nil { return fmt.Errorf("cannot parse a line: %w", err) } - f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree) + mutex.Lock() + err = ranger.Insert(cidranger.NewBasicRangerEntry(*ipnet)) + mutex.Unlock() + + if err != nil { + return fmt.Errorf("cannot insert %v into ranger: %w", ipnet, err) + } } if scanner.Err() != nil { @@ -194,38 +186,26 @@ func (f *Firehol) updateFromFile(mutex sync.Locker, return nil } -func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) { - _, ipnet, err := net.ParseCIDR(text) - if err != nil { - ipaddr := net.ParseIP(text) - if ipaddr == nil { - return nil, 0, fmt.Errorf("incorrect ip address %s", text) - } - - ip4 := ipaddr.To4() - if ip4 != nil { - return ip4, fireholIPv4DefaultCIDR, nil - } - - return ipaddr.To16(), fireholIPv6DefaultCIDR, nil +func (f *Firehol) updateParseLine(text string) (*net.IPNet, error) { + if _, ipnet, err := net.ParseCIDR(text); err == nil { + return ipnet, nil } - ones, _ := ipnet.Mask.Size() - - return ipnet.IP, uint(ones), nil -} + ipaddr := net.ParseIP(text) + if ipaddr == nil { + return nil, fmt.Errorf("incorrect ip address %s", text) + } -func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, - mutex sync.Locker, - v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) { - mutex.Lock() - defer mutex.Unlock() + mask := fireholIPv4DefaultCIDR - if ip.To4() != nil { - v4tree.Set(patricia.NewIPv4AddressFromBytes(ip, cidr), true) - } else { - v6tree.Set(patricia.NewIPv6Address(ip, cidr), true) + if ipaddr.To4() == nil { + mask = fireholIPv6DefaultCIDR } + + return &net.IPNet{ + IP: ipaddr, + Mask: mask, + }, nil } // NewFirehol creates a new instance of FireHOL IP blocklist. @@ -235,7 +215,8 @@ func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, func NewFirehol(logger mtglib.Logger, network mtglib.Network, downloadConcurrency uint, urls []string, - localFiles []string) (*Firehol, error) { + localFiles []string, + updateCallback FireholUpdateCallback) (*Firehol, error) { blocklists := []files.File{} for _, v := range localFiles { @@ -258,12 +239,13 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, blocklists = append(blocklists, file) } - return NewFireholFromFiles(logger, downloadConcurrency, blocklists) + return NewFireholFromFiles(logger, downloadConcurrency, blocklists, updateCallback) } func NewFireholFromFiles(logger mtglib.Logger, downloadConcurrency uint, - blocklists []files.File) (*Firehol, error) { + blocklists []files.File, + updateCallback FireholUpdateCallback) (*Firehol, error) { if downloadConcurrency == 0 { downloadConcurrency = DefaultFireholDownloadConcurrency } @@ -272,12 +254,12 @@ func NewFireholFromFiles(logger mtglib.Logger, ctx, cancel := context.WithCancel(context.Background()) return &Firehol{ - ctx: ctx, - ctxCancel: cancel, - logger: logger.Named("firehol"), - treeV4: bool_tree.NewTreeV4(), - treeV6: bool_tree.NewTreeV6(), - workerPool: workerPool, - blocklists: blocklists, + ctx: ctx, + ctxCancel: cancel, + logger: logger.Named("firehol"), + ranger: cidranger.NewPCTrieRanger(), + workerPool: workerPool, + blocklists: blocklists, + updateCallback: updateCallback, }, nil } diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go index 9d26c6d33..416d43e82 100644 --- a/ipblocklist/firehol_test.go +++ b/ipblocklist/firehol_test.go @@ -67,7 +67,8 @@ func (suite *FireholTestSuite) TearDownSuite() { func (suite *FireholTestSuite) TestLocalFail() { blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), suite.networkMock, 2, - nil, []string{filepath.Join("testdata", "broken_ipset.ipset")}) + nil, []string{filepath.Join("testdata", "broken_ipset.ipset")}, + nil) suite.NoError(err) @@ -85,7 +86,8 @@ func (suite *FireholTestSuite) TestLocalFail() { func (suite *FireholTestSuite) TestLocalOk() { blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), suite.networkMock, 2, - nil, []string{filepath.Join("testdata", "good_ipset.ipset")}) + nil, []string{filepath.Join("testdata", "good_ipset.ipset")}, + nil) suite.NoError(err) @@ -103,7 +105,7 @@ func (suite *FireholTestSuite) TestLocalOk() { func (suite *FireholTestSuite) TestRemoteFail() { blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), suite.networkMock, 2, - []string{"https://google.com"}, nil) + []string{"https://google.com"}, nil, nil) suite.NoError(err) @@ -127,7 +129,7 @@ func (suite *FireholTestSuite) TestMixed() { suite.httpServer.URL, }, []string{ filepath.Join("testdata", "good_ipset.ipset"), - }) + }, nil) suite.NoError(err) diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go index 2df31cb24..40da20fc6 100644 --- a/ipblocklist/noop.go +++ b/ipblocklist/noop.go @@ -2,13 +2,16 @@ package ipblocklist import ( "net" + "time" "github.com/9seconds/mtg/v2/mtglib" ) type noop struct{} -func (n noop) Contains(ip net.IP) bool { return false } +func (n noop) Contains(ip net.IP) bool { return false } +func (n noop) Run(updateEach time.Duration) {} +func (n noop) Shutdown() {} // NewNoop returns a dummy ipblocklist which allows all incoming // connections. diff --git a/ipblocklist/noop_test.go b/ipblocklist/noop_test.go index f5bb346e7..31c632745 100644 --- a/ipblocklist/noop_test.go +++ b/ipblocklist/noop_test.go @@ -17,6 +17,13 @@ func (suite *NoopTestSuite) TestOp() { suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) } +func (suite *NoopTestSuite) TestRun() { + blocklist := ipblocklist.NewNoop() + + blocklist.Run(0) + blocklist.Shutdown() +} + func TestNoop(t *testing.T) { t.Parallel() suite.Run(t, &NoopTestSuite{}) diff --git a/mtglib/events.go b/mtglib/events.go index 1bc6fb79e..fdc387bd4 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -92,6 +92,15 @@ type EventReplayAttack struct { eventBase } +// EventIPListSize is emitted when mtg updates a contents of the ip lists: +// allowlist or blocklist. +type EventIPListSize struct { + eventBase + + Size int + IsBlockList bool +} + // NewEventStart creates a new EventStart event. func NewEventStart(streamID string, remoteIP net.IP) EventStart { return EventStart{ @@ -176,3 +185,14 @@ func NewEventReplayAttack(streamID string) EventReplayAttack { }, } } + +// NewEventIPListSize creates a new EventIPListSize event. +func NewEventIPListSize(size int, isBlockList bool) EventIPListSize { + return EventIPListSize{ + eventBase: eventBase{ + timestamp: time.Now(), + }, + Size: size, + IsBlockList: isBlockList, + } +} diff --git a/mtglib/events_test.go b/mtglib/events_test.go index bb88b0a59..56da9793b 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -69,6 +69,15 @@ func (suite *EventsTestSuite) TestEventReplayAttack() { suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } +func (suite *EventsTestSuite) TestEventIPListSize() { + evt := mtglib.NewEventIPListSize(10, false) + + suite.Empty(evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) + suite.Equal(10, evt.Size) + suite.False(evt.IsBlockList) +} + func TestEvents(t *testing.T) { t.Parallel() suite.Run(t, &EventsTestSuite{}) diff --git a/mtglib/init.go b/mtglib/init.go index 2f34ff719..a245935ba 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -176,6 +176,12 @@ type IPBlocklist interface { // Contains checks if given IP address belongs to this blocklist If. // it is, a connection is terminated . Contains(net.IP) bool + + // Run starts a background update procedure for a blocklist + Run(time.Duration) + + // Shutdown stops a blocklist. It is assumed that none will access it after. + Shutdown() } // Event is a data structure which is populated during mtg request diff --git a/mtglib/internal/faketls/pools.go b/mtglib/internal/faketls/pools.go index 52288b1be..e8dbcd27d 100644 --- a/mtglib/internal/faketls/pools.go +++ b/mtglib/internal/faketls/pools.go @@ -12,7 +12,7 @@ var bytesBufferPool = sync.Pool{ } func acquireBytesBuffer() *bytes.Buffer { - return bytesBufferPool.Get().(*bytes.Buffer) + return bytesBufferPool.Get().(*bytes.Buffer) // nolint: forcetypeassert } func releaseBytesBuffer(b *bytes.Buffer) { diff --git a/mtglib/internal/faketls/record/pools.go b/mtglib/internal/faketls/record/pools.go index 16fa81fbe..e51c10cad 100644 --- a/mtglib/internal/faketls/record/pools.go +++ b/mtglib/internal/faketls/record/pools.go @@ -11,7 +11,7 @@ var recordPool = sync.Pool{ } func AcquireRecord() *Record { - return recordPool.Get().(*Record) + return recordPool.Get().(*Record) // nolint: forcetypeassert } func ReleaseRecord(r *Record) { diff --git a/mtglib/internal/obfuscated2/pools.go b/mtglib/internal/obfuscated2/pools.go index 811c540df..fd4a3da1d 100644 --- a/mtglib/internal/obfuscated2/pools.go +++ b/mtglib/internal/obfuscated2/pools.go @@ -21,7 +21,7 @@ var ( ) func acquireSha256Hasher() hash.Hash { - return sha256HasherPool.Get().(hash.Hash) + return sha256HasherPool.Get().(hash.Hash) // nolint: forcetypeassert } func releaseSha256Hasher(h hash.Hash) { @@ -30,7 +30,7 @@ func releaseSha256Hasher(h hash.Hash) { } func acquireBytesBuffer() *bytes.Buffer { - return bytesBufferPool.Get().(*bytes.Buffer) + return bytesBufferPool.Get().(*bytes.Buffer) // nolint: forcetypeassert } func releaseBytesBuffer(buf *bytes.Buffer) { diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go index 89719bb48..5321c9ae8 100644 --- a/mtglib/internal/obfuscated2/server_handshake_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -69,7 +69,7 @@ func (suite *ServerHandshakeTestSuite) TestSendToTelegram() { Once(). Run(func(args mock.Arguments) { message := make([]byte, len(messageToTelegram)) - suite.decryptor.XORKeyStream(message, args.Get(0).([]byte)) + suite.decryptor.XORKeyStream(message, args.Get(0).([]byte)) // nolint: forcetypeassert suite.Equal(messageToTelegram, message) }) @@ -89,7 +89,7 @@ func (suite *ServerHandshakeTestSuite) TestRecieveFromTelegram() { Run(func(args mock.Arguments) { message := make([]byte, len(messageFromTelegram)) suite.encryptor.XORKeyStream(message, messageFromTelegram) - copy(args.Get(0).([]byte), message) + copy(args.Get(0).([]byte), message) // nolint: forcetypeassert }) n, err := suite.proxyConn.Read(buffer) diff --git a/mtglib/internal/relay/pools.go b/mtglib/internal/relay/pools.go index 7b9371e2a..b8536811c 100644 --- a/mtglib/internal/relay/pools.go +++ b/mtglib/internal/relay/pools.go @@ -11,7 +11,7 @@ var copyBufferPool = sync.Pool{ } func acquireCopyBuffer() *[]byte { - return copyBufferPool.Get().(*[]byte) + return copyBufferPool.Get().(*[]byte) // nolint: forcetypeassert } func releaseCopyBuffer(buf *[]byte) { diff --git a/mtglib/internal/telegram/init.go b/mtglib/internal/telegram/init.go index 448120bcd..78874dafa 100644 --- a/mtglib/internal/telegram/init.go +++ b/mtglib/internal/telegram/init.go @@ -2,10 +2,13 @@ package telegram import ( "context" + "errors" "github.com/9seconds/mtg/v2/essentials" ) +var errNoAddresses = errors.New("no addresses") + type preferIP uint8 const ( diff --git a/mtglib/internal/telegram/telegram.go b/mtglib/internal/telegram/telegram.go index 395a9b7a6..926c7a00e 100644 --- a/mtglib/internal/telegram/telegram.go +++ b/mtglib/internal/telegram/telegram.go @@ -28,10 +28,9 @@ func (t Telegram) Dial(ctx context.Context, dc int) (essentials.Conn, error) { addresses = append(t.pool.getV6(dc), t.pool.getV4(dc)...) } - var ( - conn essentials.Conn - err error - ) + var conn essentials.Conn + + err := errNoAddresses for _, v := range addresses { conn, err = t.dialer.DialContext(ctx, v.network, v.address) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 294d55d2a..86e882f9c 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -106,7 +106,7 @@ func (p *Proxy) Serve(listener net.Listener) error { // nolint: cyclop } } - ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP + ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP // nolint: forcetypeassert logger := p.logger.BindStr("ip", ipAddr.String()) if p.whitelist != nil && !p.whitelist.Contains(ipAddr) { @@ -144,6 +144,12 @@ func (p *Proxy) Shutdown() { p.ctxCancel() p.streamWaitGroup.Wait() p.workerPool.Release() + + if p.whitelist != nil { + p.whitelist.Shutdown() + } + + p.blocklist.Shutdown() } func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { @@ -249,7 +255,10 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { } p.eventStream.Send(ctx, - NewEventConnectedToDC(ctx.streamID, conn.RemoteAddr().(*net.TCPAddr).IP, ctx.dc)) + NewEventConnectedToDC(ctx.streamID, + conn.RemoteAddr().(*net.TCPAddr).IP, // nolint: forcetypeassert + ctx.dc), + ) return nil } @@ -310,7 +319,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { pool, err := ants.NewPoolWithFunc(opts.getConcurrency(), func(arg interface{}) { - proxy.ServeConn(arg.(essentials.Conn)) + proxy.ServeConn(arg.(essentials.Conn)) // nolint: forcetypeassert }, ants.WithLogger(opts.getLogger("ants")), ants.WithNonblocking(true)) diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go index e1f231971..81752f239 100644 --- a/mtglib/stream_context.go +++ b/mtglib/stream_context.go @@ -49,7 +49,7 @@ func (s *streamContext) Close() { } func (s *streamContext) ClientIP() net.IP { - return s.clientConn.RemoteAddr().(*net.TCPAddr).IP + return s.clientConn.RemoteAddr().(*net.TCPAddr).IP // nolint: forcetypeassert } func newStreamContext(ctx context.Context, logger Logger, clientConn essentials.Conn) *streamContext { diff --git a/network/default.go b/network/default.go index 50855d989..e2a5ff0ab 100644 --- a/network/default.go +++ b/network/default.go @@ -36,7 +36,7 @@ func (d *defaultDialer) DialContext(ctx context.Context, network, address string return nil, fmt.Errorf("cannot set socket options: %w", err) } - return conn.(essentials.Conn), nil + return conn.(essentials.Conn), nil // nolint: forcetypeassert } // NewDefaultDialer build a new dialer which dials bypassing proxies diff --git a/network/init_internal_test.go b/network/init_internal_test.go index 6e63532fd..0b6e4a9df 100644 --- a/network/init_internal_test.go +++ b/network/init_internal_test.go @@ -14,11 +14,11 @@ type DialerMock struct { func (d *DialerMock) Dial(network, address string) (essentials.Conn, error) { args := d.Called(network, address) - return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck + return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert } func (d *DialerMock) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) { args := d.Called(ctx, network, address) - return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck + return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert } diff --git a/network/init_test.go b/network/init_test.go index 6e79a48db..c5e265129 100644 --- a/network/init_test.go +++ b/network/init_test.go @@ -22,13 +22,13 @@ type DialerMock struct { func (d *DialerMock) Dial(network, address string) (essentials.Conn, error) { args := d.Called(network, address) - return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck + return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert } func (d *DialerMock) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) { args := d.Called(ctx, network, address) - return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck + return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert } type HTTPServerTestSuite struct { diff --git a/network/sockopts.go b/network/sockopts.go index da99d0974..938155b5d 100644 --- a/network/sockopts.go +++ b/network/sockopts.go @@ -10,13 +10,13 @@ import ( // // bufferSize setting is deprecated and ignored. func SetClientSocketOptions(conn net.Conn, bufferSize int) error { - return setCommonSocketOptions(conn.(*net.TCPConn)) + return setCommonSocketOptions(conn.(*net.TCPConn)) // nolint: forcetypeassert } // SetServerSocketOptions tunes a TCP socket that represents a connection to // remote server like Telegram or fronting domain (but not end user). func SetServerSocketOptions(conn net.Conn, bufferSize int) error { - return setCommonSocketOptions(conn.(*net.TCPConn)) + return setCommonSocketOptions(conn.(*net.TCPConn)) // nolint: forcetypeassert } func setCommonSocketOptions(conn *net.TCPConn) error { diff --git a/stats/init.go b/stats/init.go index 2154c605b..1bf33dfb6 100644 --- a/stats/init.go +++ b/stats/init.go @@ -89,6 +89,13 @@ const ( // Type: counter MetricReplayAttacks = "replay_attacks" + // MetricIPListSize defines a metric for the size of the the ip list. + // + // Type: gauge + // Tags: + // ip_list | 'allowlist' or 'blocklist' + MetricIPListSize = "iplist_size" + // TagIPFamily defines a name of the 'ip_family' tag and all values. TagIPFamily = "ip_family" @@ -114,4 +121,13 @@ const ( // TagDirectionFromClient defines that traffic is sent from a client to // Telegram. TagDirectionFromClient = "from_client" + + // TagIPList defines a name of the 'ip_list' and all values. + TagIPList = "ip_list" + + // TagIPListAllow defines a value of 'ip_list' of allowlist. + TagIPListAllow = "allowlist" + + // TagIPListBlock defines a value of 'ip_list' of blocklist. + TagIPListBlock = "blocklist" ) diff --git a/stats/pools.go b/stats/pools.go index b9b625569..6c81d20b8 100644 --- a/stats/pools.go +++ b/stats/pools.go @@ -11,7 +11,7 @@ var streamInfoPool = sync.Pool{ } func acquireStreamInfo() *streamInfo { - return streamInfoPool.Get().(*streamInfo) + return streamInfoPool.Get().(*streamInfo) // nolint: forcetypeassert } func releaseStreamInfo(info *streamInfo) { diff --git a/stats/prometheus.go b/stats/prometheus.go index 97c6e973a..17882df81 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -118,6 +118,15 @@ func (p prometheusProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { p.factory.metricReplayAttacks.Inc() } +func (p prometheusProcessor) EventIPListSize(evt mtglib.EventIPListSize) { + tag := TagIPListBlock + if !evt.IsBlockList { + tag = TagIPListAllow + } + + p.factory.metricIPListSize.WithLabelValues(tag).Set(float64(evt.Size)) +} + func (p prometheusProcessor) Shutdown() { for k, v := range p.streams { releaseStreamInfo(v) @@ -137,6 +146,7 @@ type PrometheusFactory struct { metricClientConnections *prometheus.GaugeVec metricTelegramConnections *prometheus.GaugeVec metricDomainFrontingConnections *prometheus.GaugeVec + metricIPListSize *prometheus.GaugeVec metricTelegramTraffic *prometheus.CounterVec metricDomainFrontingTraffic *prometheus.CounterVec @@ -197,6 +207,11 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint Name: MetricDomainFrontingConnections, Help: "A number of connections which talk to front domain.", }, []string{TagIPFamily}), + metricIPListSize: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricPrefix, + Name: MetricIPListSize, + Help: "A size of the ip list (blocklist or allowlist)", + }, []string{TagIPList}), metricTelegramTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metricPrefix, @@ -234,6 +249,7 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint registry.MustRegister(factory.metricClientConnections) registry.MustRegister(factory.metricTelegramConnections) registry.MustRegister(factory.metricDomainFrontingConnections) + registry.MustRegister(factory.metricIPListSize) registry.MustRegister(factory.metricTelegramTraffic) registry.MustRegister(factory.metricDomainFrontingTraffic) diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index 6e44c0b25..e5300950c 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -169,6 +169,18 @@ func (suite *PrometheusTestSuite) TestEventReplayAttack() { suite.Contains(data, `mtg_replay_attacks 1`) } +func (suite *PrometheusTestSuite) TestEventIPListSize() { + suite.prometheus.EventIPListSize(mtglib.NewEventIPListSize(10, false)) + suite.prometheus.EventIPListSize(mtglib.NewEventIPListSize(3, true)) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_iplist_size{ip_list="allowlist"} 10`) + suite.Contains(data, `mtg_iplist_size{ip_list="blocklist"} 3`) +} + func TestPrometheus(t *testing.T) { t.Parallel() suite.Run(t, &PrometheusTestSuite{}) diff --git a/stats/statsd.go b/stats/statsd.go index fcf8191ae..bd3f6fbe6 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -121,6 +121,15 @@ func (s statsdProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { s.client.Incr(MetricReplayAttacks, 1) } +func (s statsdProcessor) EventIPListSize(evt mtglib.EventIPListSize) { + tag := TagIPListBlock + if !evt.IsBlockList { + tag = TagIPListAllow + } + + s.client.Gauge(MetricIPListSize, int64(evt.Size), statsd.StringTag(TagIPList, tag)) +} + func (s statsdProcessor) Shutdown() { events := make([]mtglib.EventFinish, 0, len(s.streams)) diff --git a/stats/statsd_test.go b/stats/statsd_test.go index d810c426a..f3eaafd9d 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -196,6 +196,22 @@ func (suite *StatsdTestSuite) TestEventReplayAttack() { suite.Equal("mtg.replay_attacks:1|c", suite.statsdServer.String()) } +func (suite *StatsdTestSuite) TestEventIPListSizeAllowlist() { + suite.statsd.EventIPListSize(mtglib.NewEventIPListSize(10, false)) + + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), "mtg.iplist_size:10|g") + suite.Contains(suite.statsdServer.String(), "allowlist") +} + +func (suite *StatsdTestSuite) TestEventIPListSizeBlocklist() { + suite.statsd.EventIPListSize(mtglib.NewEventIPListSize(10, true)) + + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), "mtg.iplist_size:10|g") + suite.Contains(suite.statsdServer.String(), "blocklist") +} + func TestStatsd(t *testing.T) { t.Parallel() suite.Run(t, &StatsdTestSuite{})