Skip to content

Commit

Permalink
Pull request 1778: AG-20200-translation-script-update-auto
Browse files Browse the repository at this point in the history 
Merge in DNS/adguard-home from AG-20200-translation-script-update-auto to master

Squashed commit of the following:

commit 22c68f8
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 18:08:40 2023 +0300

    scripts: imp err more

commit a6ea94b
Merge: 69749b1 2a0d062
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 17:38:05 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 69749b1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 17:37:27 2023 +0300

    scripts: imp err

commit 6d3eb32
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 13:10:08 2023 +0300

    scripts: imp err msg

commit a95e338
Merge: 16caba7 3575aa0
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 12:06:28 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 16caba7
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Apr 3 12:05:49 2023 +0300

    scripts: fix err

commit 3566193
Merge: 55efdeb da9008a
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 30 13:13:54 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 55efdeb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 30 13:13:05 2023 +0300

    scripts: simplify

commit 4a090a6
Merge: 571b2a2 c576d50
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 29 14:10:06 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 571b2a2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 29 14:09:17 2023 +0300

    scripts: fix log msg

commit 6e92a76
Merge: 207c8ba 487675b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 28 18:01:58 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 207c8ba
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 28 18:01:23 2023 +0300

    scripts: add verbose flag

commit e82270f
Merge: 11761bd 132ec55
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 27 15:09:21 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 11761bd
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 27 15:08:39 2023 +0300

    scripts: upd readme

commit cdac6cf
Merge: 5f82435 df61741
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 23 16:28:37 2023 +0300

    Merge branch 'master' into AG-20200-translation-script-update-auto

commit 5f82435
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Mar 23 16:27:01 2023 +0300

    scripts: add locale update auto
  • Loading branch information
@schzhn
schzhn committed Apr 3, 2023
1 parent 2a0d062 commit 195300f
Show file tree
Hide file tree
Showing 2 changed files with 126 additions and 27 deletions.
3 changes: 3 additions & 0 deletions scripts/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,9 @@ manifest file templates, and helper scripts.

* `go run main.go unused`: show the list of unused strings.

* `go run main.go auto-add`: add locales with additions to the git and
restore locales with deletions.

After the download you'll find the output locales in the `client/src/__locales/`
directory.
Expand Down
150 changes: 123 additions & 27 deletions scripts/translations/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package main

import (
"bufio"
"bytes"
"encoding/json"
"flag"
Expand All @@ -13,12 +14,14 @@ import (
"net/textproto"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"

"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/maps"
Expand Down Expand Up @@ -76,19 +79,19 @@ func main() {
switch os.Args[1] {
case "summary":
err = summary(conf.Languages)
check(err)
case "download":
err = download(uri, projectID, conf.Languages)
check(err)
case "unused":
err = unused()
check(err)
err = unused(conf.LocalizableFiles[0])
case "upload":
err = upload(uri, projectID, conf.BaseLangcode)
check(err)
case "auto-add":
err = autoAdd(conf.LocalizableFiles[0])
default:
usage("unknown command")
}

check(err)
}

// check is a simple error-checking helper for scripts.
Expand All @@ -108,11 +111,14 @@ Commands:
summary
Print summary.
download [-n <count>]
Download translations. count is a number of concurrent downloads.
Download translations. count is a number of concurrent downloads.
unused
Print unused strings.
upload
Upload translations.`
Upload translations.
auto-add
Add locales with additions to the git and restore locales with
deletions.`

if addStr != "" {
fmt.Printf("%s\n%s\n", addStr, usageStr)
Expand All @@ -135,6 +141,8 @@ type twoskyConf struct {

// readTwoskyConf returns configuration.
func readTwoskyConf() (t twoskyConf, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()

b, err := os.ReadFile(twoskyConfFile)
if err != nil {
// Don't wrap the error since it's informative enough as is.
Expand Down Expand Up @@ -163,6 +171,10 @@ func readTwoskyConf() (t twoskyConf, err error) {
}
}

if len(conf.LocalizableFiles) == 0 {
return twoskyConf{}, errors.Error("no localizable files specified")
}

return conf, nil
}

Expand Down Expand Up @@ -232,7 +244,7 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {

err = flagSet.Parse(os.Args[2:])
if err != nil {
// Don't wrap the error since there is exit on error.
// Don't wrap the error since it's informative enough as is.
return err
}

Expand All @@ -246,12 +258,12 @@ func download(uri *url.URL, projectID string, langs languages) (err error) {
Timeout: 10 * time.Second,
}

var wg sync.WaitGroup
wg := &sync.WaitGroup{}
uriCh := make(chan *url.URL, len(langs))

for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(&wg, client, uriCh)
go downloadWorker(wg, client, uriCh)
}

for lang := range langs {
Expand Down Expand Up @@ -342,19 +354,18 @@ func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode)
}

// unused prints unused text labels.
func unused() (err error) {
fileNames := []string{}
basePath := filepath.Join(localesDir, defaultBaseFile)
func unused(basePath string) (err error) {
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("unused: %w", err)
}

locDir := filepath.Clean(localesDir)

fileNames := []string{}
err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
if err != nil {
log.Info("accessing a path %q: %s", name, err)
log.Info("warning: accessing a path %q: %s", name, err)

return nil
}
Expand All @@ -379,12 +390,11 @@ func unused() (err error) {
return fmt.Errorf("filepath walking %q: %w", srcDir, err)
}

err = removeUnused(fileNames, baseLoc)

return errors.Annotate(err, "removing unused: %w")
return findUnused(fileNames, baseLoc)
}

func removeUnused(fileNames []string, loc locales) (err error) {
// findUnused prints unused text labels from fileNames.
func findUnused(fileNames []string, loc locales) (err error) {
knownUsed := []textLabel{
"blocking_mode_refused",
"blocking_mode_nxdomain",
Expand All @@ -399,8 +409,7 @@ func removeUnused(fileNames []string, loc locales) (err error) {
var buf []byte
buf, err = os.ReadFile(fn)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
return fmt.Errorf("finding unused: %w", err)
}

for k := range loc {
Expand All @@ -410,19 +419,14 @@ func removeUnused(fileNames []string, loc locales) (err error) {
}
}

printUnused(loc)

return nil
}

// printUnused text labels to stdout.
func printUnused(loc locales) {
keys := maps.Keys(loc)
slices.Sort(keys)

for _, v := range keys {
fmt.Println(v)
}

return nil
}

// upload base translation. uri is the base URL. projectID is the name of the
Expand Down Expand Up @@ -536,3 +540,95 @@ func send(uriStr, cType string, buf *bytes.Buffer) (err error) {

return nil
}

// autoAdd adds locales with additions to the git and restores locales with
// deletions.
func autoAdd(basePath string) (err error) {
defer func() { err = errors.Annotate(err, "auto add: %w") }()

adds, dels, err := changedLocales()
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}

if slices.Contains(dels, basePath) {
return errors.Error("base locale contains deletions")
}

var (
args []string
code int
out []byte
)

if len(adds) > 0 {
args = append([]string{"add"}, adds...)
code, out, err = aghos.RunCommand("git", args...)

if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
}

if len(dels) > 0 {
args = append([]string{"restore"}, dels...)
code, out, err = aghos.RunCommand("git", args...)

if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
}

return nil
}

// changedLocales returns cleaned paths of locales with changes or error. adds
// is the list of locales with only additions. dels is the list of locales
// with only deletions.
func changedLocales() (adds, dels []string, err error) {
defer func() { err = errors.Annotate(err, "getting changes: %w") }()

cmd := exec.Command("git", "diff", "--numstat", localesDir)

stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, fmt.Errorf("piping: %w", err)
}

err = cmd.Start()
if err != nil {
return nil, nil, fmt.Errorf("starting: %w", err)
}

scanner := bufio.NewScanner(stdout)

for scanner.Scan() {
line := scanner.Text()

fields := strings.Fields(line)
if len(fields) < 3 {
return nil, nil, fmt.Errorf("invalid input: %q", line)
}

path := fields[2]

if fields[0] == "0" {
dels = append(dels, path)
} else if fields[1] == "0" {
adds = append(adds, path)
}
}

err = scanner.Err()
if err != nil {
return nil, nil, fmt.Errorf("scanning: %w", err)
}

err = cmd.Wait()
if err != nil {
return nil, nil, fmt.Errorf("waiting: %w", err)
}

return adds, dels, nil
}

0 comments on commit 195300f

Please sign in to comment.