Skip to content

Commit

Permalink
Merge branch 'AdguardTeam:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
hoang-rio authored Sep 4, 2024
2 parents 8d6d440 + 76344f9 commit 6b40366
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 26 deletions.
11 changes: 8 additions & 3 deletions scripts/blocked-services/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
package main

import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/url"
"os"
Expand All @@ -14,10 +16,13 @@ import (
"text/template"
"time"

"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/logutil/slogutil"
)

func main() {
ctx := context.Background()
l := slogutil.New(nil)

urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/services.json"
if v, ok := os.LookupEnv("URL"); ok {
urlStr = v
Expand All @@ -33,7 +38,7 @@ func main() {

resp, err := c.Get(urlStr)
check(err)
defer log.OnCloserError(resp.Body, log.ERROR)
defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError)

if resp.StatusCode != http.StatusOK {
panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
Expand Down Expand Up @@ -64,7 +69,7 @@ func main() {
0o644,
)
check(err)
defer log.OnCloserError(f, log.ERROR)
defer slogutil.CloseAndLog(ctx, l, f, slog.LevelError)

err = tmpl.Execute(f, hlSvcs)
check(err)
Expand Down
44 changes: 28 additions & 16 deletions scripts/translations/download.go
Original file line number Diff line number Diff line change
@@ -1,25 +1,26 @@
package main

import (
"context"
"flag"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"

"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/ioutil"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/logutil/slogutil"
)

// download and save all translations.
func (c *twoskyClient) download() (err error) {
func (c *twoskyClient) download(ctx context.Context, l *slog.Logger) (err error) {
var numWorker int

flagSet := flag.NewFlagSet("download", flag.ExitOnError)
Expand Down Expand Up @@ -50,7 +51,7 @@ func (c *twoskyClient) download() (err error) {

for range numWorker {
wg.Add(1)
go downloadWorker(wg, failed, client, uriCh)
go downloadWorker(ctx, l, wg, failed, client, uriCh)
}

for _, lang := range c.langs {
Expand All @@ -62,13 +63,13 @@ func (c *twoskyClient) download() (err error) {
close(uriCh)
wg.Wait()

printFailedLocales(failed)
printFailedLocales(ctx, l, failed)

return nil
}

// printFailedLocales prints sorted list of failed downloads, if any.
func printFailedLocales(failed *sync.Map) {
func printFailedLocales(ctx context.Context, l *slog.Logger, failed *sync.Map) {
keys := []string{}
failed.Range(func(k, _ any) bool {
s, ok := k.(string)
Expand All @@ -86,12 +87,14 @@ func printFailedLocales(failed *sync.Map) {
}

slices.Sort(keys)
log.Info("failed locales: %s", strings.Join(keys, " "))
l.InfoContext(ctx, "failed", "locales", keys)
}

// downloadWorker downloads translations by received urls and saves them.
// Where failed is a map for storing failed downloads.
func downloadWorker(
ctx context.Context,
l *slog.Logger,
wg *sync.WaitGroup,
failed *sync.Map,
client *http.Client,
Expand All @@ -103,22 +106,26 @@ func downloadWorker(
q := uri.Query()
code := q.Get("language")

err := saveToFile(client, uri, code)
err := saveToFile(ctx, l, client, uri, code)
if err != nil {
log.Error("download: worker: %s", err)
l.ErrorContext(ctx, "download worker", slogutil.KeyError, err)
failed.Store(code, struct{}{})
}
}
}

// saveToFile downloads translation by url and saves it to a file, or returns
// error.
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
data, err := getTranslation(client, uri.String())
func saveToFile(
ctx context.Context,
l *slog.Logger,
client *http.Client,
uri *url.URL,
code string,
) (err error) {
data, err := getTranslation(ctx, l, client, uri.String())
if err != nil {
log.Info("%s", data)

return fmt.Errorf("getting translation: %s", err)
return fmt.Errorf("getting translation %q: %s", code, err)
}

name := filepath.Join(localesDir, code+".json")
Expand All @@ -134,13 +141,18 @@ func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {

// getTranslation returns received translation data and error. If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
func getTranslation(
ctx context.Context,
l *slog.Logger,
client *http.Client,
url string,
) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}

defer log.OnCloserError(resp.Body, log.ERROR)
defer slogutil.CloseAndLog(ctx, l, resp.Body, slog.LevelError)

if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
Expand Down
19 changes: 12 additions & 7 deletions scripts/translations/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import (
"bufio"
"bytes"
"cmp"
"context"
"encoding/json"
"fmt"
"log/slog"
"net/url"
"os"
"os/exec"
Expand All @@ -18,7 +20,7 @@ import (

"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/logutil/slogutil"
"golang.org/x/exp/maps"
)

Expand Down Expand Up @@ -63,6 +65,9 @@ type textLabel string
type locales map[textLabel]string

func main() {
ctx := context.Background()
l := slogutil.New(nil)

if len(os.Args) == 1 {
usage("need a command")
}
Expand All @@ -83,9 +88,9 @@ func main() {
cli, err = conf.toClient()
check(err)

err = cli.download()
err = cli.download(ctx, l)
case "unused":
err = unused(conf.LocalizableFiles[0])
err = unused(ctx, l, conf.LocalizableFiles[0])
case "upload":
cli, err = conf.toClient()
check(err)
Expand Down Expand Up @@ -322,7 +327,7 @@ func summary(langs languages) (err error) {
}

// unused prints unused text labels.
func unused(basePath string) (err error) {
func unused(ctx context.Context, l *slog.Logger, basePath string) (err error) {
defer func() { err = errors.Annotate(err, "unused: %w") }()

baseLoc, err := readLocales(basePath)
Expand All @@ -331,7 +336,7 @@ func unused(basePath string) (err error) {
}

locDir := filepath.Clean(localesDir)
js, err := findJS(locDir)
js, err := findJS(ctx, l, locDir)
if err != nil {
return err
}
Expand All @@ -340,10 +345,10 @@ func unused(basePath string) (err error) {
}

// findJS returns list of JavaScript and JSON files or error.
func findJS(locDir string) (fileNames []string, err error) {
func findJS(ctx context.Context, l *slog.Logger, locDir string) (fileNames []string, err error) {
walkFn := func(name string, _ os.FileInfo, err error) error {
if err != nil {
log.Info("warning: accessing a path %q: %s", name, err)
l.WarnContext(ctx, "accessing a path", slogutil.KeyError, err)

return nil
}
Expand Down

0 comments on commit 6b40366

Please sign in to comment.