From 782dd67183891c7a7fc3e6382d6e641466b6c534 Mon Sep 17 00:00:00 2001 From: Ayuhito Date: Tue, 10 Sep 2024 16:24:48 +0300 Subject: [PATCH 1/2] fix(core): improve user agent coverage --- core/go.mod | 8 ++++---- core/go.sum | 16 ++++++++-------- core/services/event.go | 31 +++++++++++++------------------ 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/core/go.mod b/core/go.mod index a59a5f3..9f9d590 100644 --- a/core/go.mod +++ b/core/go.mod @@ -13,7 +13,7 @@ require ( github.com/marcboeker/go-duckdb v1.7.1 github.com/medama-io/go-referrer-parser v0.0.0-20240903120234-0a63376371c3 github.com/medama-io/go-timezone-country v0.0.0-20240903121643-db228bdc5dc1 - github.com/medama-io/go-useragent v1.0.0 + github.com/medama-io/go-useragent v1.0.1 github.com/ncruces/go-sqlite3 v0.18.2 github.com/ogen-go/ogen v1.4.0 github.com/rs/cors v1.11.1 @@ -44,7 +44,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7 // indirect + github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/maruel/natural v1.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -68,12 +68,12 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.27.0 // indirect - golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/core/go.sum b/core/go.sum index c8389e8..8bdbc50 100644 --- a/core/go.sum +++ b/core/go.sum @@ -60,8 +60,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7 h1:5RK988zAqB3/AN3opGfRpoQgAVqr6/A5+qRTi67VUZY= -github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= +github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/marcboeker/go-duckdb v1.7.1 h1:m9/nKfP7cG9AptcQ95R1vfacRuhtrZE5pZF8BPUb/Iw= github.com/marcboeker/go-duckdb v1.7.1/go.mod h1:2oV8BZv88S16TKGKM+Lwd0g7DX84x0jMxjTInThC8Is= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -78,8 +78,8 @@ github.com/medama-io/go-referrer-parser v0.0.0-20240903120234-0a63376371c3 h1:6/ github.com/medama-io/go-referrer-parser v0.0.0-20240903120234-0a63376371c3/go.mod h1:Zng9ySjx7KXIpvVqT/mZbYfKE39CkyS/aQR4kXdJuG0= github.com/medama-io/go-timezone-country v0.0.0-20240903121643-db228bdc5dc1 h1:/Q1ZWbdGSRpExJRlQZybxwxXa6u4lYH7K/OLD9t/d8M= github.com/medama-io/go-timezone-country v0.0.0-20240903121643-db228bdc5dc1/go.mod h1:Wq7lg5D0ZdQ3bHnzOTKsb1YGlxm/l82OVA4aIbAA5w4= -github.com/medama-io/go-useragent v1.0.0 h1:AEtLVKtUV1PiZ4u/zCkRLHBahs/crkoFw/YLSU3Rb8w= -github.com/medama-io/go-useragent v1.0.0/go.mod h1:H9GYWth4IN8vAFZh5LeARza7VwM4jK9uk7Tb9huVzLw= +github.com/medama-io/go-useragent v1.0.1 h1:staHGaZKIRpGI7sPVnyjYVT0SMRlQmp0L21rKYusi8Y= +github.com/medama-io/go-useragent v1.0.1/go.mod h1:H9GYWth4IN8vAFZh5LeARza7VwM4jK9uk7Tb9huVzLw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/ncruces/go-sqlite3 v0.18.2 h1:m7QXhBWIwXsp84HE11t+ze0n1v3LRU+zGFg4uHjBeFA= @@ -151,8 +151,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= -golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= @@ -202,8 +202,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= diff --git a/core/services/event.go b/core/services/event.go index 6c660a0..7e22cb0 100644 --- a/core/services/event.go +++ b/core/services/event.go @@ -210,20 +210,24 @@ func (h *Handler) PostEventHit(ctx context.Context, req api.EventHit, _params ap countryName, err := h.timezoneCountryMap.GetCountry(req.EventLoad.T.Value) if err != nil { log.Debug().Err(err).Msg("hit: failed to get country name from timezone") - countryName = Unknown - } - if countryName == "" { unknownCounter++ if unknownCounter >= IsBotThreshold { return &api.PostEventHitNoContent{}, nil } + + countryName = Unknown } // Get users language from Accept-Language header languages, _, err := language.ParseAcceptLanguage(reqBody.Header.Get("Accept-Language")) if err != nil { log.Debug().Err(err).Msg("hit: failed to parse accept language header") + + unknownCounter++ + if unknownCounter >= IsBotThreshold { + return &api.PostEventHitNoContent{}, nil + } } // Get the first language from the list which is the most preferred and convert it to a language name @@ -236,16 +240,10 @@ func (h *Handler) PostEventHit(ctx context.Context, req api.EventHit, _params ap languageDialect = display.English.Tags().Name(languages[0]) } - if languageBase == Unknown { - unknownCounter++ - if unknownCounter >= IsBotThreshold { - return &api.PostEventHitNoContent{}, nil - } - } - // Parse referrer URL and remove any query parameters or self-referencing // hostnames. referrerHost := "" + referrerGroup := "" if req.EventLoad.R.Value != "" { referrer, err := url.Parse(req.EventLoad.R.Value) if err != nil { @@ -253,20 +251,17 @@ func (h *Handler) PostEventHit(ctx context.Context, req api.EventHit, _params ap return ErrBadRequest(err), nil } + referrerHost = referrer.Hostname() + // If the referrer hostname is the same as the current hostname, we // want to remove it. - referrerHost = referrer.Hostname() - if referrerHost == hostname { + if referrerHost != hostname { + referrerGroup = h.referrer.Parse(referrerHost) + } else { referrerHost = "" } } - referrerGroup := "" - if referrerHost != "" { - // Get the referrer group from the referrer URL. - referrerGroup = h.referrer.Parse(referrerHost) - } - // Get utm source, medium, and campaigm from URL query parameters. queries := req.EventLoad.U.Query() utmSource := queries.Get("utm_source") From a8385bacc0a9899af566c908d330a0b6aeb45ccb Mon Sep 17 00:00:00 2001 From: Ayuhito Date: Tue, 10 Sep 2024 17:26:28 +0300 Subject: [PATCH 2/2] cI: pin linter version --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5533e51..db1df11 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: - name: Lint core uses: golangci/golangci-lint-action@v6 with: - version: latest + version: v1.60 working-directory: core - name: Lint dashboard