Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(core): improve user agent coverage #144

Merged
merged 2 commits into from
Sep 10, 2024
Merged

fix(core): improve user agent coverage #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
782dd67
fix(core): improve user agent coverage
ayuhito Sep 10, 2024
a8385ba
cI: pin linter version
ayuhito Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
- name: Lint core
uses: golangci/golangci-lint-action@v6
with:
version: latest
version: v1.60
working-directory: core

- name: Lint dashboard
Expand Down
8 changes: 4 additions & 4 deletions core/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
github.com/marcboeker/go-duckdb v1.7.1
github.com/medama-io/go-referrer-parser v0.0.0-20240903120234-0a63376371c3
github.com/medama-io/go-timezone-country v0.0.0-20240903121643-db228bdc5dc1
github.com/medama-io/go-useragent v1.0.0
github.com/medama-io/go-useragent v1.0.1
github.com/ncruces/go-sqlite3 v0.18.2
github.com/ogen-go/ogen v1.4.0
github.com/rs/cors v1.11.1
Expand Down Expand Up @@ -44,7 +44,7 @@ require (
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7 // indirect
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
github.com/maruel/natural v1.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
Expand All @@ -68,12 +68,12 @@ require (
github.com/zeebo/xxh3 v1.0.2 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/tools v0.24.0 // indirect
golang.org/x/tools v0.25.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
16 changes: 8 additions & 8 deletions core/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7 h1:5RK988zAqB3/AN3opGfRpoQgAVqr6/A5+qRTi67VUZY=
github.com/lufia/plan9stats v0.0.0-20240819163618-b1d8f4d146e7/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0=
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
github.com/marcboeker/go-duckdb v1.7.1 h1:m9/nKfP7cG9AptcQ95R1vfacRuhtrZE5pZF8BPUb/Iw=
github.com/marcboeker/go-duckdb v1.7.1/go.mod h1:2oV8BZv88S16TKGKM+Lwd0g7DX84x0jMxjTInThC8Is=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
Expand All @@ -78,8 +78,8 @@ github.com/medama-io/go-referrer-parser v0.0.0-20240903120234-0a63376371c3 h1:6/
github.com/medama-io/go-referrer-parser v0.0.0-20240903120234-0a63376371c3/go.mod h1:Zng9ySjx7KXIpvVqT/mZbYfKE39CkyS/aQR4kXdJuG0=
github.com/medama-io/go-timezone-country v0.0.0-20240903121643-db228bdc5dc1 h1:/Q1ZWbdGSRpExJRlQZybxwxXa6u4lYH7K/OLD9t/d8M=
github.com/medama-io/go-timezone-country v0.0.0-20240903121643-db228bdc5dc1/go.mod h1:Wq7lg5D0ZdQ3bHnzOTKsb1YGlxm/l82OVA4aIbAA5w4=
github.com/medama-io/go-useragent v1.0.0 h1:AEtLVKtUV1PiZ4u/zCkRLHBahs/crkoFw/YLSU3Rb8w=
github.com/medama-io/go-useragent v1.0.0/go.mod h1:H9GYWth4IN8vAFZh5LeARza7VwM4jK9uk7Tb9huVzLw=
github.com/medama-io/go-useragent v1.0.1 h1:staHGaZKIRpGI7sPVnyjYVT0SMRlQmp0L21rKYusi8Y=
github.com/medama-io/go-useragent v1.0.1/go.mod h1:H9GYWth4IN8vAFZh5LeARza7VwM4jK9uk7Tb9huVzLw=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/ncruces/go-sqlite3 v0.18.2 h1:m7QXhBWIwXsp84HE11t+ze0n1v3LRU+zGFg4uHjBeFA=
Expand Down Expand Up @@ -151,8 +151,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk=
golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
Expand Down Expand Up @@ -202,8 +202,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE=
golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
Expand Down
31 changes: 13 additions & 18 deletions core/services/event.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,20 +210,24 @@ func (h *Handler) PostEventHit(ctx context.Context, req api.EventHit, _params ap
countryName, err := h.timezoneCountryMap.GetCountry(req.EventLoad.T.Value)
if err != nil {
log.Debug().Err(err).Msg("hit: failed to get country name from timezone")
countryName = Unknown
}

if countryName == "" {
unknownCounter++
if unknownCounter >= IsBotThreshold {
return &api.PostEventHitNoContent{}, nil
}

countryName = Unknown
}

// Get users language from Accept-Language header
languages, _, err := language.ParseAcceptLanguage(reqBody.Header.Get("Accept-Language"))
if err != nil {
log.Debug().Err(err).Msg("hit: failed to parse accept language header")

unknownCounter++
if unknownCounter >= IsBotThreshold {
return &api.PostEventHitNoContent{}, nil
}
}

// Get the first language from the list which is the most preferred and convert it to a language name
Expand All @@ -236,37 +240,28 @@ func (h *Handler) PostEventHit(ctx context.Context, req api.EventHit, _params ap
languageDialect = display.English.Tags().Name(languages[0])
}

if languageBase == Unknown {
unknownCounter++
if unknownCounter >= IsBotThreshold {
return &api.PostEventHitNoContent{}, nil
}
}

// Parse referrer URL and remove any query parameters or self-referencing
// hostnames.
referrerHost := ""
referrerGroup := ""
if req.EventLoad.R.Value != "" {
referrer, err := url.Parse(req.EventLoad.R.Value)
if err != nil {
log.Warn().Err(err).Msg("hit: failed to parse referrer URL")
return ErrBadRequest(err), nil
}

referrerHost = referrer.Hostname()

// If the referrer hostname is the same as the current hostname, we
// want to remove it.
referrerHost = referrer.Hostname()
if referrerHost == hostname {
if referrerHost != hostname {
referrerGroup = h.referrer.Parse(referrerHost)
} else {
referrerHost = ""
}
}

referrerGroup := ""
if referrerHost != "" {
// Get the referrer group from the referrer URL.
referrerGroup = h.referrer.Parse(referrerHost)
}

// Get utm source, medium, and campaigm from URL query parameters.
queries := req.EventLoad.U.Query()
utmSource := queries.Get("utm_source")
Expand Down
Loading