-
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactoring and start on syncer package
- Loading branch information
Showing
9 changed files
with
217 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| package config | ||
|
|
||
| type SyncerRemotes struct { | ||
| Copy []string | ||
| Sync []string | ||
| MoveServerSide []UploaderRemotesMoveServerSide `mapstructure:"move_server_side"` | ||
| } | ||
|
|
||
| type SyncerRcloneParams struct { | ||
| Copy []string | ||
| Move []string | ||
| MoveServerSide []string `mapstructure:"move_server_side"` | ||
| } | ||
|
|
||
| type SyncerConfig struct { | ||
| Enabled bool | ||
| ServiceAccountFolder string `mapstructure:"sa_folder"` | ||
| SourceRemote string `mapstructure:"source_remote"` | ||
| Remotes SyncerRemotes | ||
| RcloneParams SyncerRcloneParams `mapstructure:"rclone_params"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| package syncer | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/l3uddz/crop/cache" | ||
| "github.com/l3uddz/crop/pathutils" | ||
| "github.com/l3uddz/crop/rclone" | ||
| "github.com/l3uddz/crop/stringutils" | ||
| "github.com/pkg/errors" | ||
| "github.com/sirupsen/logrus" | ||
| "time" | ||
| ) | ||
|
|
||
| func (s *Syncer) Copy(additionalRcloneParams []string) error { | ||
| // set variables | ||
| copyParams := s.Config.RcloneParams.Copy | ||
| if additionalRcloneParams != nil { | ||
| copyParams = append(copyParams, additionalRcloneParams...) | ||
| } | ||
|
|
||
| // iterate all remotes and run copy | ||
| for _, remotePath := range s.Config.Remotes.Copy { | ||
| // set variables | ||
| attempts := 1 | ||
| rLog := s.Log.WithFields(logrus.Fields{ | ||
| "copy_remote": remotePath, | ||
| "source_remote": s.Config.SourceRemote, | ||
| "attempts": attempts, | ||
| }) | ||
|
|
||
| // copy to remote | ||
| for { | ||
| // get service account file | ||
| var serviceAccount *pathutils.Path | ||
| var err error | ||
|
|
||
| if s.ServiceAccountCount > 0 { | ||
| serviceAccount, err = rclone.GetAvailableServiceAccount(s.ServiceAccountFiles) | ||
| if err != nil { | ||
| return errors.WithMessagef(err, | ||
| "aborting further copy attempts of %q due to serviceAccount exhaustion", | ||
| s.Config.SourceRemote) | ||
| } | ||
|
|
||
| // reset log | ||
| rLog = s.Log.WithFields(logrus.Fields{ | ||
| "copy_remote": remotePath, | ||
| "source_remote": s.Config.SourceRemote, | ||
| "attempts": attempts, | ||
| "service_account": serviceAccount.RealPath, | ||
| }) | ||
| } | ||
|
|
||
| // copy | ||
| rLog.Info("Copying...") | ||
| success, exitCode, err := rclone.Copy(s.Config.SourceRemote, remotePath, serviceAccount, copyParams) | ||
|
|
||
| // check result | ||
| if err != nil { | ||
| rLog.WithError(err).Errorf("Failed unexpectedly...") | ||
| return errors.WithMessagef(err, "copy failed unexpectedly with exit code: %v", exitCode) | ||
| } else if success { | ||
| // successful exit code | ||
| break | ||
| } | ||
|
|
||
| // is this an exit code we can retry? | ||
| switch exitCode { | ||
| case rclone.EXIT_FATAL_ERROR: | ||
| // are we using service accounts? | ||
| if s.ServiceAccountCount == 0 { | ||
| // we are not using service accounts, so mark this remote as banned | ||
| if err := cache.Set(stringutils.FromLeftUntil(remotePath, ":"), | ||
| time.Now().UTC().Add(25*time.Hour)); err != nil { | ||
| rLog.WithError(err).Errorf("Failed banning remote") | ||
| } | ||
|
|
||
| return fmt.Errorf("copy failed with exit code: %v", exitCode) | ||
| } | ||
|
|
||
| // ban this service account | ||
| if err := cache.Set(serviceAccount.RealPath, time.Now().UTC().Add(25*time.Hour)); err != nil { | ||
| rLog.WithError(err).Error("Failed banning service account, cannot try again...") | ||
| return fmt.Errorf("failed banning service account: %v", serviceAccount.RealPath) | ||
| } | ||
|
|
||
| // attempt copy again | ||
| rLog.Warnf("Copy failed with retryable exit code %v, trying again...", exitCode) | ||
| attempts++ | ||
| continue | ||
| default: | ||
| return fmt.Errorf("failed and cannot proceed with exit code: %v", exitCode) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package syncer | ||
|
|
||
| import ( | ||
| "github.com/l3uddz/crop/config" | ||
| "github.com/l3uddz/crop/logger" | ||
| "github.com/l3uddz/crop/pathutils" | ||
| "github.com/l3uddz/crop/stringutils" | ||
| "github.com/sirupsen/logrus" | ||
| "regexp" | ||
| "sort" | ||
| "strconv" | ||
| "strings" | ||
| ) | ||
|
|
||
| type Syncer struct { | ||
| // Public | ||
| Log *logrus.Entry | ||
| GlobalConfig *config.Configuration | ||
| Config *config.SyncerConfig | ||
| Name string | ||
|
|
||
| ServiceAccountFiles []pathutils.Path | ||
| ServiceAccountCount int | ||
| } | ||
|
|
||
| func New(config *config.Configuration, syncerConfig *config.SyncerConfig, syncerName string) (*Syncer, error) { | ||
| // init syncer dependencies | ||
| // - service account files | ||
| var serviceAccountFiles []pathutils.Path | ||
| if syncerConfig.ServiceAccountFolder != "" { | ||
| serviceAccountFiles, _ = pathutils.GetPathsInFolder(syncerConfig.ServiceAccountFolder, true, | ||
| false, func(path string) *string { | ||
| lowerPath := strings.ToLower(path) | ||
|
|
||
| // ignore non json files | ||
| if !strings.HasSuffix(lowerPath, ".json") { | ||
| return nil | ||
| } | ||
|
|
||
| return &path | ||
| }) | ||
|
|
||
| // sort service files | ||
| if len(serviceAccountFiles) > 0 { | ||
| re := regexp.MustCompile("[0-9]+") | ||
| sort.SliceStable(serviceAccountFiles, func(i, j int) bool { | ||
| is := stringutils.NewOrExisting(re.FindString(serviceAccountFiles[i].RealPath), "0") | ||
| js := stringutils.NewOrExisting(re.FindString(serviceAccountFiles[j].RealPath), "0") | ||
|
|
||
| in, err := strconv.Atoi(is) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| jn, err := strconv.Atoi(js) | ||
| if err != nil { | ||
| return false | ||
| } | ||
|
|
||
| return in < jn | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // init uploader | ||
| syncer := &Syncer{ | ||
| Log: logger.GetLogger(syncerName), | ||
| GlobalConfig: config, | ||
| Config: syncerConfig, | ||
| Name: syncerName, | ||
| ServiceAccountFiles: serviceAccountFiles, | ||
| ServiceAccountCount: len(serviceAccountFiles), | ||
| } | ||
|
|
||
| return syncer, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters