From 55efdeb80b44183767b188109f1e21aac2fa9839 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Thu, 30 Mar 2023 13:13:05 +0300 Subject: [PATCH] scripts: simplify --- scripts/README.md | 6 +- scripts/translations/main.go | 342 ++++++++--------------------------- 2 files changed, 80 insertions(+), 268 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 664bfb2c0cd..cc823a995c3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -189,10 +189,8 @@ manifest file templates, and helper scripts. * `go run main.go unused`: show the list of unused strings. - * `go run main.go auto-add [-n ]`: download translation updates from - Crowdin and add them to the git. `n` is optional flag where count is a - number of concurrent downloads. `v` is optional flag for enabling verbose - output` + * `go run main.go auto-add`: add locales with additions to the git and + restore locales with deletions. After the download you'll find the output locales in the `client/src/__locales/` directory. diff --git a/scripts/translations/main.go b/scripts/translations/main.go index 6f35b7bd7f3..25dd1f441b2 100644 --- a/scripts/translations/main.go +++ b/scripts/translations/main.go @@ -51,9 +51,6 @@ type textLabel string // locales is a map, where key is text label and value is translation. type locales map[textLabel]string -// localePath is a cleaned path to the locale. -type localePath string - func main() { if len(os.Args) == 1 { usage("need a command") @@ -79,64 +76,22 @@ func main() { conf, err := readTwoskyConf() check(err) - err = command(uri, projectID, conf) - check(err) -} - -func command(uri *url.URL, projectID string, conf twoskyConf) (err error) { - var flagSet *flag.FlagSet - var numWorker int - var verbose bool - switch os.Args[1] { case "summary": err = summary(conf.Languages) case "download": - flagSet = flag.NewFlagSet("download", flag.ExitOnError) - flagSet.Usage = func() { - usage("download command error") - } - flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads") - - err = flagSet.Parse(os.Args[2:]) - if err != nil { - // Don't wrap the error since there is exit on error. - return err - } - - if numWorker < 1 { - usage("count must be positive") - } - - download(uri, projectID, conf.Languages, numWorker) + err = download(uri, projectID, conf.Languages) case "unused": err = unused(conf.LocalizableFiles[0]) case "upload": err = upload(uri, projectID, conf.BaseLangcode) case "auto-add": - flagSet = flag.NewFlagSet("auto-add", flag.ExitOnError) - flagSet.Usage = func() { - usage("auto-add command error") - } - flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads") - flagSet.BoolVar(&verbose, "v", false, "enable verbose output") - - err = flagSet.Parse(os.Args[2:]) - if err != nil { - // Don't wrap the error since there is exit on error. - return err - } - - if verbose { - log.SetLevel(log.DEBUG) - } - - err = autoAdd(uri, projectID, conf.Languages, conf.BaseLangcode, numWorker) + err = autoAdd(conf.LocalizableFiles[0]) default: usage("unknown command") } - return err + check(err) } // check is a simple error-checking helper for scripts. @@ -146,18 +101,6 @@ func check(err error) { } } -// ask reads line from STDIN, returns true if line is 'y', otherwise false. -func ask(s string) (a bool) { - fmt.Printf("%s y/[n]: ", s) - - scanner := bufio.NewScanner(os.Stdin) - - scanner.Scan() - line := scanner.Text() - - return strings.ToLower(line) == "y" -} - // usage prints usage. If addStr is not empty print addStr and exit with code // 1, otherwise exit with code 0. func usage(addStr string) { @@ -173,10 +116,9 @@ Commands: Print unused strings. upload Upload translations. - auto-add [-n ] [-v] - Download translation updates from Crowdin and add them to the git. - count is a number of concurrent downloads. - -v Enable verbose output` + auto-add + Add locales with additions to the git and restore locales with + deletions.` if addStr != "" { fmt.Printf("%s\n%s\n", addStr, usageStr) @@ -199,6 +141,8 @@ type twoskyConf struct { // readTwoskyConf returns configuration. func readTwoskyConf() (t twoskyConf, err error) { + defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }() + b, err := os.ReadFile(twoskyConfFile) if err != nil { // Don't wrap the error since it's informative enough as is. @@ -228,7 +172,7 @@ func readTwoskyConf() (t twoskyConf, err error) { } if len(conf.LocalizableFiles) == 0 { - return twoskyConf{}, errors.Error("localizable_files are empty") + return twoskyConf{}, errors.Error("localizable_files field is empty") } return conf, nil @@ -256,25 +200,10 @@ func readLocales(fn string) (loc locales, err error) { // summary prints summary for translations. func summary(langs languages) (err error) { - sum, err := getSummary(langs) - if err != nil { - return fmt.Errorf("summary: %w", err) - } - - for lang, f := range sum { - fmt.Printf("%s\t %6.2f %%\n", lang, f) - } - - return nil -} - -func getSummary(langs languages) (sum map[langCode]float64, err error) { - sum = make(map[langCode]float64) - basePath := filepath.Join(localesDir, defaultBaseFile) baseLoc, err := readLocales(basePath) if err != nil { - return nil, fmt.Errorf("reading locales: %w", err) + return fmt.Errorf("summary: %w", err) } size := float64(len(baseLoc)) @@ -291,18 +220,38 @@ func getSummary(langs languages) (sum map[langCode]float64, err error) { var loc locales loc, err = readLocales(name) if err != nil { - return nil, fmt.Errorf("summary: reading locales: %w", err) + return fmt.Errorf("summary: reading locales: %w", err) } - sum[lang] = float64(len(loc)) * 100 / size + f := float64(len(loc)) * 100 / size + + fmt.Printf("%s\t %6.2f %%\n", lang, f) } - return sum, nil + return nil } // download and save all translations. uri is the base URL. projectID is the // name of the project. -func download(uri *url.URL, projectID string, langs languages, numWorker int) { +func download(uri *url.URL, projectID string, langs languages) (err error) { + var numWorker int + + flagSet := flag.NewFlagSet("download", flag.ExitOnError) + flagSet.Usage = func() { + usage("download command error") + } + flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads") + + err = flagSet.Parse(os.Args[2:]) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return err + } + + if numWorker < 1 { + usage("count must be positive") + } + downloadURI := uri.JoinPath("download") client := &http.Client{ @@ -325,6 +274,8 @@ func download(uri *url.URL, projectID string, langs languages, numWorker int) { close(uriCh) wg.Wait() + + return nil } // downloadWorker downloads translations by received urls and saves them. @@ -439,12 +390,10 @@ func unused(basePath string) (err error) { return fmt.Errorf("filepath walking %q: %w", srcDir, err) } - err = findUnused(fileNames, baseLoc) - - return errors.Annotate(err, "removing unused: %w") + return findUnused(fileNames, baseLoc) } -// findUnused text labels from fileNames. +// findUnused prints unused text labels from fileNames. func findUnused(fileNames []string, loc locales) (err error) { knownUsed := []textLabel{ "blocking_mode_refused", @@ -460,8 +409,7 @@ func findUnused(fileNames []string, loc locales) (err error) { var buf []byte buf, err = os.ReadFile(fn) if err != nil { - // Don't wrap the error since it's informative enough as is. - return err + return fmt.Errorf("finding unused: %w", err) } for k := range loc { @@ -471,19 +419,14 @@ func findUnused(fileNames []string, loc locales) (err error) { } } - printUnused(loc) - - return nil -} - -// printUnused text labels to stdout. -func printUnused(loc locales) { keys := maps.Keys(loc) slices.Sort(keys) for _, v := range keys { fmt.Println(v) } + + return nil } // upload base translation. uri is the base URL. projectID is the name of the @@ -598,193 +541,54 @@ func send(uriStr, cType string, buf *bytes.Buffer) (err error) { return nil } -// autoAdd downloads translation updates from Crowdin and adds them to the git. -func autoAdd( - uri *url.URL, - projectID string, - langs languages, - baseLang langCode, - numWorker int, -) (err error) { +// autoAdd adds locales with additions to the git and restores locales with +// deletions. +func autoAdd(basePath string) (err error) { defer func() { err = errors.Annotate(err, "auto add: %w") }() - log.Debug("downloading the locales") - download(uri, projectID, langs, numWorker) - - log.Debug("checking if the base locale is up to date") - adds, dels, err := getChangedLocales() - if err != nil { - // Don't wrap the error since it's informative enough as is and there - // is an annotation deferred already. - return err - } - - basePath := localePath(filepath.Join(localesDir, defaultBaseFile)) - - if slices.Contains(adds, basePath) { - log.Debug("base locale has additions") - - // If there are additions, then someone probably unuploaded changes - // from unmerged branches. This isn't critical, so just add this - // change. - err = addBaseLocale() - if err != nil { - // Don't wrap the error since it's informative enough as is and - // there is an annotation deferred already. - return err - } - } else if slices.Contains(dels, basePath) { - log.Debug("base locale has deletions") - - return baseLocaleDeletions(uri, projectID, baseLang) - } else { - log.Debug("base locale has no changes") - } - - return autoAddChanges(langs, adds) -} - -// baseLocaleDeletions handles deletion changes to the base locale. -func baseLocaleDeletions(uri *url.URL, projectID string, baseLang langCode) (err error) { - err = restoreLocales() - if err != nil { - return err - } - - if !ask("Upload the English locale?") { - return nil - } - - err = upload(uri, projectID, baseLang) + adds, dels, err := changedLocales() if err != nil { + // Don't wrap the error since it's informative enough as is. return err } - return nil -} - -// autoAddChanges adds translation changes to the git. -func autoAddChanges(langs languages, adds []localePath) (err error) { - log.Debug("checking if the release-blocker locales are fully translated") - err = checkReleaseBlockerLocales(langs) - if err != nil { - // Don't wrap the error since it's informative enough as is and there - // is an annotation deferred already. - return err + if slices.Contains(dels, basePath) { + return errors.Error("base locale contains deletions") } - log.Debug("adding non-deletion changes") - err = addAdditions(adds) + args := append([]string{"add"}, adds...) + _, _, err = aghos.RunCommand("git", args...) if err != nil { - // Don't wrap the error since it's informative enough as is and there - // is an annotation deferred already. - return err + return fmt.Errorf("adding additions: %w", err) } - err = restoreLocales() + args = append([]string{"restore"}, dels...) + _, _, err = aghos.RunCommand("git", args...) if err != nil { - // Don't wrap the error since it's informative enough as is and there - // is an annotation deferred already. - return err + return fmt.Errorf("restoring deletions: %w", err) } return nil } -// checkReleaseBlockerLocales returns nil if translations of the release -// blocker locales are up to date. -func checkReleaseBlockerLocales(langs languages) (err error) { - sum, err := getSummary(langs) - if err != nil { - return fmt.Errorf("checking release blocker locales: %w", err) - } - - blockers := []langCode{ - "de", - "es", - "fr", - "it", - "ja", - "ko", - "pt-br", - "pt-pt", - "ru", - "zh-cn", - "zh-tw", - } - - for _, b := range blockers { - if sum[b] != 100 { - return fmt.Errorf("locale %q is not fully translated", b) - } - } - - return nil -} +// changedLocales returns cleaned paths of locales with changes or error. adds +// is the list of locales with only additions. dels is the list of locales +// with only deletions. +func changedLocales() (adds, dels []string, err error) { + defer func() { err = errors.Annotate(err, "getting changes: %w") }() -// restoreLocales reverts changes to the locales. -func restoreLocales() (err error) { - if !ask("Proceed to restore locales with unstaged changes?") { - return nil - } - - log.Debug("restoring locales with unstaged changes") - - _, _, err = aghos.RunCommand("git", "restore", localesDir) - - return errors.Annotate(err, "restoring locales: %w") -} - -// gitAddPatch is a helper that interactively adds changes to the git. -func gitAddPatch(path string) (err error) { - cmd := exec.Command("git", "add", "-p", path) - - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err = cmd.Run() - - return errors.Annotate(err, "adding patch: %w") -} - -// addBaseLocale adds base locale to the git. -func addBaseLocale() (err error) { - path := filepath.Join(localesDir, defaultBaseFile) - - err = gitAddPatch(path) - - return errors.Annotate(err, "adding base locale: %w") -} - -// addAdditions adds locales, except base localse, with additional changes to -// the git. -func addAdditions(adds []localePath) (err error) { - for _, v := range adds { - err = gitAddPatch(string(v)) - - if err != nil { - return fmt.Errorf("adding locale %q: %w", v, err) - } - } - - return nil -} - -// getChangedLocales returns cleaned paths of locales with changes or error. -// adds is the list of locales with at least single addition. dels is the list -// of locales with deletions. -func getChangedLocales() (adds, dels []localePath, err error) { cmd := exec.Command("git", "diff", "--numstat", localesDir) stdout, err := cmd.StdoutPipe() if err != nil { - return nil, nil, fmt.Errorf("getting command: %w", err) + // Don't wrap the error since it's informative enough as is. + return nil, nil, err } err = cmd.Start() if err != nil { - return nil, nil, fmt.Errorf("getting changes: %w", err) + // Don't wrap the error since it's informative enough as is. + return nil, nil, err } scanner := bufio.NewScanner(stdout) @@ -793,19 +597,29 @@ func getChangedLocales() (adds, dels []localePath, err error) { line := scanner.Text() fields := strings.Fields(line) + if len(fields) < 3 { + return nil, nil, err + } - path := localePath(fields[2]) + path := fields[2] - if fields[0] != "0" { - adds = append(adds, path) - } else { + if fields[0] == "0" { dels = append(dels, path) + } else if fields[1] == "0" { + adds = append(adds, path) } } + err = scanner.Err() + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, nil, err + } + err = cmd.Wait() if err != nil { - return nil, nil, fmt.Errorf("waiting command: %w", err) + // Don't wrap the error since it's informative enough as is. + return nil, nil, err } return adds, dels, nil