diff --git a/config/config.go b/config/config.go index 8167758..980d18e 100644 --- a/config/config.go +++ b/config/config.go @@ -25,6 +25,7 @@ type BuildVars struct { type Configuration struct { Rclone RcloneConfig Uploader map[string]UploaderConfig + Syncer map[string]SyncerConfig } /* Vars */ diff --git a/config/syncer.go b/config/syncer.go new file mode 100644 index 0000000..319af61 --- /dev/null +++ b/config/syncer.go @@ -0,0 +1,21 @@ +package config + +type SyncerRemotes struct { + Copy []string + Sync []string + MoveServerSide []UploaderRemotesMoveServerSide `mapstructure:"move_server_side"` +} + +type SyncerRcloneParams struct { + Copy []string + Move []string + MoveServerSide []string `mapstructure:"move_server_side"` +} + +type SyncerConfig struct { + Enabled bool + ServiceAccountFolder string `mapstructure:"sa_folder"` + SourceRemote string `mapstructure:"source_remote"` + Remotes SyncerRemotes + RcloneParams SyncerRcloneParams `mapstructure:"rclone_params"` +} diff --git a/config/uploader.go b/config/uploader.go index 9f2f16b..7c20878 100644 --- a/config/uploader.go +++ b/config/uploader.go @@ -25,7 +25,6 @@ type UploaderRemotes struct { Copy []string Move string MoveServerSide []UploaderRemotesMoveServerSide `mapstructure:"move_server_side"` - Dedupe []string } type UploaderRcloneParams struct { diff --git a/rclone/copy.go b/rclone/copy.go index 6901643..d55f236 100644 --- a/rclone/copy.go +++ b/rclone/copy.go @@ -2,7 +2,6 @@ package rclone import ( "github.com/go-cmd/cmd" - "github.com/l3uddz/crop/config" "github.com/l3uddz/crop/pathutils" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -10,7 +9,7 @@ import ( /* Public */ -func Copy(u *config.UploaderConfig, from string, to string, serviceAccountFile *pathutils.Path, +func Copy(from string, to string, serviceAccountFile *pathutils.Path, additionalRcloneParams []string) (bool, int, error) { // set variables rLog := log.WithFields(logrus.Fields{ @@ -34,10 +33,7 @@ func Copy(u *config.UploaderConfig, from string, to string, serviceAccountFile * params = append(params, baseParams...) } - extraParams := u.RcloneParams.Copy - if additionalRcloneParams != nil { - extraParams = append(extraParams, additionalRcloneParams...) - } + extraParams := additionalRcloneParams if additionalParams, err := getAdditionalParams(CMD_COPY, extraParams); err != nil { return false, 1, errors.WithMessagef(err, "failed generating additionalParams to %q: %q -> %q", diff --git a/rclone/move.go b/rclone/move.go index 5373854..4af76a5 100644 --- a/rclone/move.go +++ b/rclone/move.go @@ -2,7 +2,6 @@ package rclone import ( "github.com/go-cmd/cmd" - "github.com/l3uddz/crop/config" "github.com/l3uddz/crop/pathutils" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -10,7 +9,7 @@ import ( /* Public */ -func Move(u *config.UploaderConfig, from string, to string, serviceAccountFile *pathutils.Path, serverSide bool, +func Move(from string, to string, serviceAccountFile *pathutils.Path, serverSide bool, additionalRcloneParams []string) (bool, int, error) { // set variables rLog := log.WithFields(logrus.Fields{ @@ -34,15 +33,10 @@ func Move(u *config.UploaderConfig, from string, to string, serviceAccountFile * params = append(params, baseParams...) } - extraParams := u.RcloneParams.Move + extraParams := additionalRcloneParams if serverSide { - // this is a server side move, so add any additional configured params - extraParams = append(extraParams, u.RcloneParams.MoveServerSide...) // add server side parameter extraParams = append(extraParams, "--drive-server-side-across-configs") - } else if additionalRcloneParams != nil { - // add additional params from parameters - extraParams = append(extraParams, additionalRcloneParams...) } if additionalParams, err := getAdditionalParams(CMD_MOVE, extraParams); err != nil { diff --git a/syncer/copy.go b/syncer/copy.go new file mode 100644 index 0000000..c5eaf82 --- /dev/null +++ b/syncer/copy.go @@ -0,0 +1,98 @@ +package syncer + +import ( + "fmt" + "github.com/l3uddz/crop/cache" + "github.com/l3uddz/crop/pathutils" + "github.com/l3uddz/crop/rclone" + "github.com/l3uddz/crop/stringutils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "time" +) + +func (s *Syncer) Copy(additionalRcloneParams []string) error { + // set variables + copyParams := s.Config.RcloneParams.Copy + if additionalRcloneParams != nil { + copyParams = append(copyParams, additionalRcloneParams...) + } + + // iterate all remotes and run copy + for _, remotePath := range s.Config.Remotes.Copy { + // set variables + attempts := 1 + rLog := s.Log.WithFields(logrus.Fields{ + "copy_remote": remotePath, + "source_remote": s.Config.SourceRemote, + "attempts": attempts, + }) + + // copy to remote + for { + // get service account file + var serviceAccount *pathutils.Path + var err error + + if s.ServiceAccountCount > 0 { + serviceAccount, err = rclone.GetAvailableServiceAccount(s.ServiceAccountFiles) + if err != nil { + return errors.WithMessagef(err, + "aborting further copy attempts of %q due to serviceAccount exhaustion", + s.Config.SourceRemote) + } + + // reset log + rLog = s.Log.WithFields(logrus.Fields{ + "copy_remote": remotePath, + "source_remote": s.Config.SourceRemote, + "attempts": attempts, + "service_account": serviceAccount.RealPath, + }) + } + + // copy + rLog.Info("Copying...") + success, exitCode, err := rclone.Copy(s.Config.SourceRemote, remotePath, serviceAccount, copyParams) + + // check result + if err != nil { + rLog.WithError(err).Errorf("Failed unexpectedly...") + return errors.WithMessagef(err, "copy failed unexpectedly with exit code: %v", exitCode) + } else if success { + // successful exit code + break + } + + // is this an exit code we can retry? + switch exitCode { + case rclone.EXIT_FATAL_ERROR: + // are we using service accounts? + if s.ServiceAccountCount == 0 { + // we are not using service accounts, so mark this remote as banned + if err := cache.Set(stringutils.FromLeftUntil(remotePath, ":"), + time.Now().UTC().Add(25*time.Hour)); err != nil { + rLog.WithError(err).Errorf("Failed banning remote") + } + + return fmt.Errorf("copy failed with exit code: %v", exitCode) + } + + // ban this service account + if err := cache.Set(serviceAccount.RealPath, time.Now().UTC().Add(25*time.Hour)); err != nil { + rLog.WithError(err).Error("Failed banning service account, cannot try again...") + return fmt.Errorf("failed banning service account: %v", serviceAccount.RealPath) + } + + // attempt copy again + rLog.Warnf("Copy failed with retryable exit code %v, trying again...", exitCode) + attempts++ + continue + default: + return fmt.Errorf("failed and cannot proceed with exit code: %v", exitCode) + } + } + } + + return nil +} diff --git a/syncer/syncer.go b/syncer/syncer.go new file mode 100644 index 0000000..c484262 --- /dev/null +++ b/syncer/syncer.go @@ -0,0 +1,75 @@ +package syncer + +import ( + "github.com/l3uddz/crop/config" + "github.com/l3uddz/crop/logger" + "github.com/l3uddz/crop/pathutils" + "github.com/l3uddz/crop/stringutils" + "github.com/sirupsen/logrus" + "regexp" + "sort" + "strconv" + "strings" +) + +type Syncer struct { + // Public + Log *logrus.Entry + GlobalConfig *config.Configuration + Config *config.SyncerConfig + Name string + + ServiceAccountFiles []pathutils.Path + ServiceAccountCount int +} + +func New(config *config.Configuration, syncerConfig *config.SyncerConfig, syncerName string) (*Syncer, error) { + // init syncer dependencies + // - service account files + var serviceAccountFiles []pathutils.Path + if syncerConfig.ServiceAccountFolder != "" { + serviceAccountFiles, _ = pathutils.GetPathsInFolder(syncerConfig.ServiceAccountFolder, true, + false, func(path string) *string { + lowerPath := strings.ToLower(path) + + // ignore non json files + if !strings.HasSuffix(lowerPath, ".json") { + return nil + } + + return &path + }) + + // sort service files + if len(serviceAccountFiles) > 0 { + re := regexp.MustCompile("[0-9]+") + sort.SliceStable(serviceAccountFiles, func(i, j int) bool { + is := stringutils.NewOrExisting(re.FindString(serviceAccountFiles[i].RealPath), "0") + js := stringutils.NewOrExisting(re.FindString(serviceAccountFiles[j].RealPath), "0") + + in, err := strconv.Atoi(is) + if err != nil { + return false + } + jn, err := strconv.Atoi(js) + if err != nil { + return false + } + + return in < jn + }) + } + } + + // init uploader + syncer := &Syncer{ + Log: logger.GetLogger(syncerName), + GlobalConfig: config, + Config: syncerConfig, + Name: syncerName, + ServiceAccountFiles: serviceAccountFiles, + ServiceAccountCount: len(serviceAccountFiles), + } + + return syncer, nil +} diff --git a/uploader/copy.go b/uploader/copy.go index 9e5c6d9..1ccaa8b 100644 --- a/uploader/copy.go +++ b/uploader/copy.go @@ -12,6 +12,12 @@ import ( ) func (u *Uploader) Copy(additionalRcloneParams []string) error { + // set variables + copyParams := u.Config.RcloneParams.Copy + if additionalRcloneParams != nil { + copyParams = append(copyParams, additionalRcloneParams...) + } + // iterate all remotes and run copy for _, remotePath := range u.Config.Remotes.Copy { // set variables @@ -47,8 +53,7 @@ func (u *Uploader) Copy(additionalRcloneParams []string) error { // copy rLog.Info("Copying...") - success, exitCode, err := rclone.Copy(u.Config, u.Config.LocalFolder, remotePath, serviceAccount, - additionalRcloneParams) + success, exitCode, err := rclone.Copy(u.Config.LocalFolder, remotePath, serviceAccount, copyParams) // check result if err != nil { diff --git a/uploader/move.go b/uploader/move.go index 660a251..fa7eccc 100644 --- a/uploader/move.go +++ b/uploader/move.go @@ -19,6 +19,7 @@ type MoveInstruction struct { func (u *Uploader) Move(serverSide bool, additionalRcloneParams []string) error { var moveRemotes []MoveInstruction + var extraParams []string // create move instructions if serverSide { @@ -30,6 +31,8 @@ func (u *Uploader) Move(serverSide bool, additionalRcloneParams []string) error ServerSide: true, }) } + + extraParams = u.Config.RcloneParams.MoveServerSide } else { // this is a normal move (to only one location) moveRemotes = append(moveRemotes, MoveInstruction{ @@ -37,6 +40,13 @@ func (u *Uploader) Move(serverSide bool, additionalRcloneParams []string) error To: u.Config.Remotes.Move, ServerSide: false, }) + + extraParams = u.Config.RcloneParams.Move + } + + // set variables + if additionalRcloneParams != nil { + extraParams = append(extraParams, additionalRcloneParams...) } // iterate all remotes and run copy @@ -75,8 +85,7 @@ func (u *Uploader) Move(serverSide bool, additionalRcloneParams []string) error // move rLog.Info("Moving...") - success, exitCode, err := rclone.Move(u.Config, move.From, move.To, serviceAccount, serverSide, - additionalRcloneParams) + success, exitCode, err := rclone.Move(move.From, move.To, serviceAccount, serverSide, extraParams) // check result if err != nil {