From 324980795d96bfb44572c06e81bcca206d9ecf38 Mon Sep 17 00:00:00 2001 From: l3uddz Date: Sat, 18 Apr 2020 16:41:24 +0100 Subject: [PATCH] add support for banning standalone remotes and dont proceed with upload if any of them are banned --- cache/get.go | 6 ++++-- cmd/upload.go | 27 ++++++++++++++++++++++++++- stringutils/until.go | 12 ++++++++++++ uploader/{sa.go => cache.go} | 31 ++++++++++++++++++++++++++++++- uploader/copy.go | 17 ++++++++++++----- uploader/move.go | 19 ++++++++++++++----- 6 files changed, 98 insertions(+), 14 deletions(-) create mode 100644 stringutils/until.go rename uploader/{sa.go => cache.go} (53%) diff --git a/cache/get.go b/cache/get.go index 1582c2a..d40dac0 100644 --- a/cache/get.go +++ b/cache/get.go @@ -4,7 +4,8 @@ import ( "time" ) -func Get(key string) bool { +func Get(key string) (bool, time.Time) { + var expires time.Time changes := false exists := false @@ -24,6 +25,7 @@ func Get(key string) bool { } else { // the item has not expired exists = true + expires = expiry } } @@ -35,5 +37,5 @@ func Get(key string) bool { } } - return exists + return exists, expires } diff --git a/cmd/upload.go b/cmd/upload.go index 2168e07..6f31b90 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -1,9 +1,11 @@ package cmd import ( + "github.com/dustin/go-humanize" "github.com/l3uddz/crop/config" "github.com/l3uddz/crop/uploader" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/yale8848/gorpool" ) @@ -27,7 +29,6 @@ var uploadCmd = &cobra.Command{ } log := log.WithField("uploader", uploaderName) - log.Info("Uploader commencing...") // create uploader upload, err := uploader.New(config.Config, &uploaderConfig, uploaderName) @@ -40,8 +41,32 @@ var uploadCmd = &cobra.Command{ if upload.ServiceAccountCount > 0 { upload.Log.WithField("found_files", upload.ServiceAccountCount). Info("Loaded service accounts") + } else { + // no service accounts were loaded + // check to see if any of the copy or move remote(s) are banned + banned, expiry := upload.RemotesBanned(upload.Config.Remotes.Copy) + if banned && !expiry.IsZero() { + // one of the copy remotes is banned, abort + upload.Log.WithFields(logrus.Fields{ + "expires_time": expiry, + "expires_in": humanize.Time(expiry), + }).Warn("Cannot proceed with upload as a copy remote is banned") + continue + } + + banned, expiry = upload.RemotesBanned([]string{upload.Config.Remotes.Move}) + if banned && !expiry.IsZero() { + // the move remote is banned, abort + upload.Log.WithFields(logrus.Fields{ + "expires_time": expiry, + "expires_in": humanize.Time(expiry), + }).Warn("Cannot proceed with upload as the move remote is banned") + continue + } } + log.Info("Uploader commencing...") + // refresh details about files to upload if err := upload.RefreshLocalFiles(); err != nil { upload.Log.WithError(err).Error("Failed refreshing details of files to upload") diff --git a/stringutils/until.go b/stringutils/until.go new file mode 100644 index 0000000..b180052 --- /dev/null +++ b/stringutils/until.go @@ -0,0 +1,12 @@ +package stringutils + +import "strings" + +func FromLeftUntil(from string, sub string) string { + pos := strings.Index(from, sub) + if pos < 0 { + return from + } + + return from[:pos] +} diff --git a/uploader/sa.go b/uploader/cache.go similarity index 53% rename from uploader/sa.go rename to uploader/cache.go index 16400b9..b690603 100644 --- a/uploader/sa.go +++ b/uploader/cache.go @@ -3,7 +3,9 @@ package uploader import ( "github.com/l3uddz/crop/cache" "github.com/l3uddz/crop/pathutils" + "github.com/l3uddz/crop/stringutils" "github.com/pkg/errors" + "time" ) /* Private */ @@ -13,7 +15,7 @@ func (u *Uploader) getAvailableServiceAccount() (*pathutils.Path, error) { // find an unbanned service account for _, sa := range u.ServiceAccountFiles { // does the cache already contain this service account? - if cache.Get(sa.RealPath) { + if exists, _ := cache.Get(sa.RealPath); exists { // service account is currently banned continue } @@ -30,3 +32,30 @@ func (u *Uploader) getAvailableServiceAccount() (*pathutils.Path, error) { return serviceAccountFile, nil } + +/* Public */ +func (u *Uploader) RemotesBanned(remotes []string) (bool, time.Time) { + var banned bool + var expires time.Time + + // ignore empty remotes slice + if remotes == nil { + return banned, expires + } + + // format remotes into remote names if possible + var checkRemotes []string + for _, remote := range remotes { + checkRemotes = append(checkRemotes, stringutils.FromLeftUntil(remote, ":")) + } + + // iterate remotes + for _, remote := range checkRemotes { + banned, expires = cache.Get(remote) + if banned { + break + } + } + + return banned, expires +} diff --git a/uploader/copy.go b/uploader/copy.go index bd57b84..24d7b8a 100644 --- a/uploader/copy.go +++ b/uploader/copy.go @@ -5,6 +5,7 @@ import ( "github.com/l3uddz/crop/cache" "github.com/l3uddz/crop/pathutils" "github.com/l3uddz/crop/rclone" + "github.com/l3uddz/crop/stringutils" "github.com/pkg/errors" "github.com/sirupsen/logrus" "time" @@ -58,14 +59,20 @@ func (u *Uploader) Copy(additionalRcloneParams []string) error { break } - // are we using service accounts? - if u.ServiceAccountCount == 0 { - return fmt.Errorf("copy failed with exit code: %v", exitCode) - } - // is this an exit code we can retry? switch exitCode { case rclone.EXIT_FATAL_ERROR: + // are we using service accounts? + if u.ServiceAccountCount == 0 { + // we are not using service accounts, so mark this remote as banned + if err := cache.Set(stringutils.FromLeftUntil(remotePath, ":"), + time.Now().UTC().Add(25*time.Hour)); err != nil { + rLog.WithError(err).Errorf("Failed banning remote") + } + + return fmt.Errorf("copy failed with exit code: %v", exitCode) + } + // ban this service account if err := cache.Set(serviceAccount.RealPath, time.Now().UTC().Add(25*time.Hour)); err != nil { rLog.WithError(err).Error("Failed banning service account, cannot try again...") diff --git a/uploader/move.go b/uploader/move.go index c5ca9a1..21c8677 100644 --- a/uploader/move.go +++ b/uploader/move.go @@ -5,6 +5,7 @@ import ( "github.com/l3uddz/crop/cache" "github.com/l3uddz/crop/pathutils" "github.com/l3uddz/crop/rclone" + "github.com/l3uddz/crop/stringutils" "github.com/pkg/errors" "github.com/sirupsen/logrus" "time" @@ -89,14 +90,22 @@ func (u *Uploader) Move(serverSide bool, additionalRcloneParams []string) error return fmt.Errorf("failed and cannot proceed with exit code: %v", exitCode) } - // are we using service accounts? - if u.ServiceAccountCount == 0 { - return fmt.Errorf("move failed with exit code: %v", exitCode) - } - // is this an exit code we can retry? switch exitCode { case rclone.EXIT_FATAL_ERROR: + // are we using service accounts? + if u.ServiceAccountCount == 0 { + // we are not using service accounts, so mark this remote as banned (if non server side move) + if !serverSide { + if err := cache.Set(stringutils.FromLeftUntil(move.To, ":"), + time.Now().UTC().Add(25*time.Hour)); err != nil { + rLog.WithError(err).Errorf("Failed banning remote") + } + } + + return fmt.Errorf("move failed with exit code: %v", exitCode) + } + // ban this service account if err := cache.Set(serviceAccount.RealPath, time.Now().UTC().Add(25*time.Hour)); err != nil { rLog.WithError(err).Error("Failed banning service account, cannot try again...")