feat(CLI): Optional cleanup after push [TSI-2234] (#518)

clients/cli/cmd/internal/push.go

@@ -24,6 +24,7 @@ import (
 type PushCommand struct {
  phrase.Config
  Wait bool
+ Cleanup bool
  Branch string
  UseLocalBranchName bool
  Tag string
@@ -35,6 +36,11 @@ func (cmd *PushCommand) Run() error {
  cmd.Config.Debug = false
  Debug = true
  }
+
+ if cmd.Cleanup && !cmd.Wait {
+ return fmt.Errorf("You can only use the --cleanup option together with --wait")
+ }
+
  Config = &cmd.Config
 
  client := newClient()
@@ -136,7 +142,7 @@ func (cmd *PushCommand) Run() error {
  }
 
  for _, source := range sources {
- err := source.Push(client, cmd.Wait, cmd.Branch, cmd.Tag)
+ err := source.Push(client, cmd.Wait, cmd.Cleanup, cmd.Branch, cmd.Tag)
  if err != nil {
  return err
  }
@@ -145,13 +151,14 @@ func (cmd *PushCommand) Run() error {
  return nil
 }
 
-func (source *Source) Push(client *phrase.APIClient, waitForResults bool, branch string, tag string) error {
+func (source *Source) Push(client *phrase.APIClient, waitForResults bool, cleanup bool, branch string, tag string) error {
  localeFiles, err := source.LocaleFiles()
  if err != nil {
  return err
  }
 
  noErrors := true
+ uploadIds := []string{}
  for _, localeFile := range localeFiles {
  print.NonBatchPrintf("Uploading %s... ", localeFile.RelPath())
 
@@ -174,6 +181,7 @@ func (source *Source) Push(client *phrase.APIClient, waitForResults bool, branch
  }
  return err
  }
+ uploadIds = append(uploadIds, upload.Id)
 
  if waitForResults {
  print.NonBatchPrintf("\n")
@@ -209,7 +217,11 @@ func (source *Source) Push(client *phrase.APIClient, waitForResults bool, branch
  fmt.Fprintln(os.Stderr, strings.Repeat("-", 10))
  }
  }
- if !noErrors {
+ if noErrors {
+ if waitForResults && cleanup {
+ return UploadCleanup(client, true, uploadIds, branch, source.ProjectID)
+ }
+ } else {
  return errors.New("not all files were uploaded successfully")
  }
 

clients/cli/cmd/internal/shared.go

@@ -1,12 +1,21 @@
 package internal
 
 import (
+ "encoding/json"
+ "errors"
+ "fmt"
  "reflect"
+ "sort"
+ "strings"
 
  "github.com/antihax/optional"
  "github.com/mitchellh/mapstructure"
+ "github.com/phrase/phrase-cli/cmd/internal/print"
+ "github.com/phrase/phrase-cli/cmd/internal/prompt"
+ "github.com/phrase/phrase-cli/cmd/internal/shared"
  "github.com/phrase/phrase-go/v2"
  "github.com/spf13/viper"
+ "golang.org/x/exp/maps"
 )
 
 var Debug bool
@@ -195,3 +204,127 @@ func StringToInterface() mapstructure.DecodeHookFunc {
  return optional.NewInterface(params), nil
  }
 }
+
+func findCommonUnmentionedKeys(client *phrase.APIClient, ids []string, branch string, projectId string) ([]phrase.TranslationKey, error) {
+ commonUnmentionedKeys := map[string]phrase.TranslationKey{}
+ alreadyInitialized := false
+ for _, id := range ids {
+ q := "unmentioned_in_upload:" + id
+ keysListLocalVarOptionals := phrase.KeysListOpts{
+ Page: optional.NewInt32(1),
+ PerPage: optional.NewInt32(100),
+ Q: optional.NewString(q),
+ Branch: optional.NewString(branch),
+ }
+ allUnmentionedKeysInUpload := map[string]phrase.TranslationKey{}
+ keys, _, err := client.KeysApi.KeysList(Auth, projectId, &keysListLocalVarOptionals)
+ if err != nil {
+ return nil, err
+ }
+ for len(keys) != 0 {
+ for _, key := range keys {
+ allUnmentionedKeysInUpload[key.Id] = key
+ }
+
+ keysListLocalVarOptionals.Page = optional.NewInt32(keysListLocalVarOptionals.Page.Value() + 1)
+
+ keys, _, err = client.KeysApi.KeysList(Auth, projectId, &keysListLocalVarOptionals)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if alreadyInitialized {
+ newUnmentioned := map[string]phrase.TranslationKey{}
+ for id, key := range allUnmentionedKeysInUpload {
+ if _, ok := commonUnmentionedKeys[id]; ok {
+ newUnmentioned[id] = key
+ }
+ }
+ commonUnmentionedKeys = newUnmentioned
+ } else {
+ commonUnmentionedKeys = allUnmentionedKeysInUpload
+ alreadyInitialized = true
+ }
+ }
+
+ return maps.Values(commonUnmentionedKeys), nil
+}
+
+func deleteKeys(client *phrase.APIClient, confirm bool, branch string, projectId string, keys []phrase.TranslationKey) error {
+ ids := make([]string, len(keys))
+ names := make([]string, len(keys))
+ for i, key := range keys {
+ ids[i] = key.Id
+ names[i] = key.Name
+ }
+
+ if !shared.BatchMode {
+ fmt.Println("Following key(s) are about to be deleted from your project:")
+ sort.Strings(names)
+ fmt.Println(strings.Join(names, "\n"))
+ }
+
+ if !confirm {
+ if shared.BatchMode {
+ return errors.New("Can't ask for confirmation in batch mode. Aborting")
+ }
+ confirmation := ""
+ err := prompt.WithDefault("Are you sure you want to continue? (y/n)", &confirmation, "n")
+ if err != nil {
+ return err
+ }
+
+ if strings.ToLower(confirmation) != "y" {
+ fmt.Println("Clean up aborted")
+ return nil
+ }
+ }
+
+ const ChunkSize = 100
+ totalAffected := int32(0)
+
+ for i := 0; i < len(ids); i += ChunkSize {
+ end := i + ChunkSize
+
+ if end > len(ids) {
+ end = len(ids)
+ }
+
+ q := "ids:" + strings.Join(ids[i:end], ",")
+ keysDeletelocalVarOptionals := phrase.KeysDeleteCollectionOpts{
+ Q: optional.NewString(q),
+ Branch: optional.NewString(branch),
+ }
+ affected, _, err := client.KeysApi.KeysDeleteCollection(Auth, projectId, &keysDeletelocalVarOptionals)
+
+ if err != nil {
+ return err
+ }
+ totalAffected += affected.RecordsAffected
+ if shared.BatchMode {
+ jsonBuf, jsonErr := json.MarshalIndent(affected, "", " ")
+ if jsonErr != nil {
+ print.Error(jsonErr)
+ }
+ fmt.Printf("%s\n", string(jsonBuf))
+ }
+ }
+
+ if !shared.BatchMode {
+ print.Success("%d key(s) successfully deleted.\n", totalAffected)
+ }
+ return nil
+}
+
+func UploadCleanup(client *phrase.APIClient, confirm bool, ids []string, branch string, projectId string) error {
+ keys, error := findCommonUnmentionedKeys(client, ids, branch, projectId)
+ if error != nil {
+ return error
+ }
+ if len(keys) == 0 {
+ print.Success("There were no keys unmentioned in the uploads.")
+ return nil
+ }
+ return deleteKeys(client, confirm, branch, projectId, keys)
+}

clients/cli/cmd/internal/upload_cleanup.go

@@ -1,22 +1,12 @@
 package internal
 
 import (
- "encoding/json"
- "errors"
- "fmt"
- "sort"
- "strings"
-
- "github.com/antihax/optional"
- "github.com/phrase/phrase-cli/cmd/internal/print"
- prompt "github.com/phrase/phrase-cli/cmd/internal/prompt"
- "github.com/phrase/phrase-cli/cmd/internal/shared"
  "github.com/phrase/phrase-go/v2"
 )
 
 type UploadCleanupCommand struct {
  phrase.Config
- ID  string
+ IDs []string
  ProjectID string
  Confirm bool
  Branch string
@@ -26,87 +16,5 @@ func (cmd *UploadCleanupCommand) Run() error {
  Config = &cmd.Config
  client := newClient()
 
- return UploadCleanup(client, cmd)
-}
-
-func UploadCleanup(client *phrase.APIClient, cmd *UploadCleanupCommand) error {
- q := "unmentioned_in_upload:" + cmd.ID
- keysListLocalVarOptionals := phrase.KeysListOpts{
- Page: optional.NewInt32(1),
- PerPage: optional.NewInt32(100),
- Q: optional.NewString(q),
- Branch: optional.NewString(cmd.Branch),
- }
- keys, _, err := client.KeysApi.KeysList(Auth, cmd.ProjectID, &keysListLocalVarOptionals)
- if err != nil {
- return err
- }
-
- if len(keys) == 0 {
- print.Success("There were no keys unmentioned in that upload.")
- return nil
- }
-
- for len(keys) != 0 {
- ids := make([]string, len(keys))
- names := make([]string, len(keys))
- for i, key := range keys {
- ids[i] = key.Id
- names[i] = key.Name
- }
-
- if !cmd.Confirm {
- if shared.BatchMode {
- return errors.New("Can't ask for confirmation in batch mode. Aborting")
- }
- fmt.Println("You are about to delete the following key(s) from your project:")
- sort.Strings(names)
- fmt.Println(strings.Join(names, "\n"))
-
- confirmation := ""
- err := prompt.WithDefault("Are you sure you want to continue? (y/n)", &confirmation, "n")
- if err != nil {
- return err
- }
-
- if strings.ToLower(confirmation) != "y" {
- fmt.Println("Clean up aborted")
- return nil
- }
- }
-
- q := "ids:" + strings.Join(ids, ",")
- keysDeletelocalVarOptionals := phrase.KeysDeleteCollectionOpts{
- Q: optional.NewString(q),
- Branch: optional.NewString(cmd.Branch),
- }
- affected, _, err := client.KeysApi.KeysDeleteCollection(Auth, cmd.ProjectID, &keysDeletelocalVarOptionals)
-
- if err != nil {
- return err
- }
-
- outputAffected(affected)
-
- keysListLocalVarOptionals.Page = optional.NewInt32(keysListLocalVarOptionals.Page.Value() + 1)
-
- keys, _, err = client.KeysApi.KeysList(Auth, cmd.ProjectID, &keysListLocalVarOptionals)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func outputAffected(affected phrase.AffectedResources) {
- if shared.BatchMode {
- jsonBuf, jsonErr := json.MarshalIndent(affected, "", " ")
- if jsonErr != nil {
- print.Error(jsonErr)
- }
- fmt.Printf("%s\n", string(jsonBuf))
- } else {
- print.Success("%d key(s) successfully deleted.\n", affected.RecordsAffected)
- }
+ return UploadCleanup(client, cmd.Confirm, cmd.IDs, cmd.Branch, cmd.ProjectID)
 }

clients/cli/cmd/push.go

@@ -20,6 +20,7 @@ func initPush() {
  cmdPush := push.PushCommand{
  Config: *Config,
  Wait: params.GetBool("wait"),
+ Cleanup: params.GetBool("cleanup"),
  Branch: params.GetString("branch"),
  UseLocalBranchName: params.GetBool("use-local-branch-name"),
  Tag: params.GetString("tag"),
@@ -33,6 +34,7 @@ func initPush() {
  rootCmd.AddCommand(pushCmd)
 
  AddFlag(pushCmd, "bool", "wait", "w", "Wait for files to be processed", false)
+ AddFlag(pushCmd, "bool", "cleanup", "c", "Delete keys not mentioned in any of the uploads", false)
  AddFlag(pushCmd, "string", "branch", "b", "branch", false)
  AddFlag(pushCmd, "bool", "use-local-branch-name", "", "push from the branch with the name of your currently checked out branch (git or mercurial)", false)
  AddFlag(pushCmd, "string", "tag", "", "Tag uploaded keys", false)

clients/cli/cmd/upload_cleanup.go

@@ -30,7 +30,7 @@ func initUpoadCleanup() {
 
  cmduploadCleanup := uploadCleanup.UploadCleanupCommand{
  Config: *Config,
- ID:  params.GetString("id"),
+ IDs: []string{params.GetString("id")},
  ProjectID: projectId,
  Confirm: params.GetBool("confirm"),
  Branch: params.GetString("branch"),

clients/cli/go.mod

@@ -27,6 +27,7 @@ require (
  github.com/spf13/jwalterweatherman v1.1.0 // indirect
  github.com/spf13/pflag v1.0.5 // indirect
  github.com/subosito/gotenv v1.2.0 // indirect
+ golang.org/x/exp v0.0.0-20231226003508-02704c960a9b
  golang.org/x/net v0.17.0 // indirect
  golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
  golang.org/x/sys v0.13.0 // indirect

clients/cli/go.sum

@@ -218,8 +218,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/phrase/phrase-go/v2 v2.15.0 h1:9v9B6p8z7yw5OYpAIOajHcD+tQMJyrBT79Dimxdk10A=
-github.com/phrase/phrase-go/v2 v2.15.0/go.mod h1:N1ou4nnB9ak8aAWOH7x25Xhzw8FmFeBRhSGq5A0CY2o=
+github.com/phrase/phrase-go/v2 v2.22.1 h1:cfDmcpuRoETD/1yY6GKEAcLdlVVb1+xHSkAqG8GiZZc=
+github.com/phrase/phrase-go/v2 v2.22.1/go.mod h1:ARQGM+rzC0tPf3Lf4pHoRKpGtmQnBa7V7LSVs3oJf0U=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
@@ -308,6 +308,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=
+golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=