diff --git a/cmd/clean.go b/cmd/clean.go new file mode 100644 index 0000000..ee8d8d1 --- /dev/null +++ b/cmd/clean.go @@ -0,0 +1,79 @@ +package cmd + +import ( + "github.com/l3uddz/crop/config" + "github.com/l3uddz/crop/uploader" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/yale8848/gorpool" +) + +var cleanCmd = &cobra.Command{ + Use: "clean [UPLOADER]", + Short: "Perform uploader clean", + Long: `This command can be used to trigger an uploader clean.`, + + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + // init core + initCore(true) + + // iterate uploader's + for uploaderName, uploaderConfig := range config.Config.Uploader { + // skip disabled uploader(s) + if !uploaderConfig.Enabled { + log.WithField("uploader", uploaderName).Trace("Skipping disabled uploader") + continue + } + + log := log.WithField("uploader", uploaderName) + + // create uploader + upload, err := uploader.New(config.Config, &uploaderConfig, uploaderName) + if err != nil { + log.WithField("uploader", uploaderName).WithError(err). + Error("Failed initializing uploader, skipping...") + continue + } + + log.Info("Clean commencing...") + + // perform upload + if err := performClean(upload); err != nil { + upload.Log.WithError(err).Error("Error occurred while running clean, skipping...") + continue + } + } + }, +} + +func init() { + rootCmd.AddCommand(cleanCmd) +} + +func performClean(u *uploader.Uploader) error { + u.Log.Info("Running cleans...") + + /* Cleans */ + if u.Config.Hidden.Enabled { + // set worker count + workers := u.Config.Hidden.Workers + if workers == 0 { + workers = 8 + } + + // create worker pool + gp := gorpool.NewPool(workers, 0). + Start(). + EnableWaitForAll(true) + + // queue clean tasks + err := u.PerformCleans(gp) + if err != nil { + return errors.Wrap(err, "failed clearing remotes") + } + } + + u.Log.Info("Finished cleans!") + return nil +} diff --git a/cmd/root.go b/cmd/root.go index d445cd9..5bdeb98 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -78,6 +78,8 @@ func initCore(showAppInfo bool) { log.WithError(err).Fatal("Failed to initialize config") } + setConfigOverrides() + // Init Cache if err := cache.Init(flagCacheFile); err != nil { log.WithError(err).Fatal("Failed to initialize cache") @@ -94,6 +96,13 @@ func initCore(showAppInfo bool) { } } +func setConfigOverrides() { + // set dry-run if enabled by flag + if flagDryRun { + config.Config.Rclone.DryRun = true + } +} + func showUsing() { // show app info log.Infof("Using %s = %s (%s@%s)", stringutils.LeftJust("VERSION", " ", 10), diff --git a/cmd/upload.go b/cmd/upload.go index 6f31b90..db41de6 100644 --- a/cmd/upload.go +++ b/cmd/upload.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/yale8848/gorpool" ) var uploadCmd = &cobra.Command{ @@ -104,11 +103,7 @@ func performUpload(u *uploader.Uploader) error { /* Cleans */ if u.Config.Hidden.Enabled { - gp := gorpool.NewPool(config.Config.Core.Workers, 0). - Start(). - EnableWaitForAll(true) - - err := u.PerformCleans(gp) + err := performClean(u) if err != nil { return errors.Wrap(err, "failed clearing remotes") } @@ -119,23 +114,35 @@ func performUpload(u *uploader.Uploader) error { /* Copies */ if len(u.Config.Remotes.Copy) > 0 { + u.Log.Info("Running copies...") + if err := u.Copy(additionalRcloneParams); err != nil { return errors.WithMessage(err, "failed performing all copies") } + + u.Log.Info("Finished copies!") } /* Move */ if len(u.Config.Remotes.Move) > 0 { + u.Log.Info("Running move...") + if err := u.Move(false, additionalRcloneParams); err != nil { return errors.WithMessage(err, "failed performing move") } + + u.Log.Info("Finished move!") } /* Move Server Side */ if len(u.Config.Remotes.MoveServerSide) > 0 { + u.Log.Info("Running move server-sides...") + if err := u.Move(true, nil); err != nil { return errors.WithMessage(err, "failed performing server-side moves") } + + u.Log.Info("Finished move server-sides!") } u.Log.Info("Finished!") diff --git a/config/config.go b/config/config.go index 0d5d8c8..8167758 100644 --- a/config/config.go +++ b/config/config.go @@ -23,7 +23,6 @@ type BuildVars struct { } type Configuration struct { - Core CoreConfig Rclone RcloneConfig Uploader map[string]UploaderConfig } @@ -129,13 +128,9 @@ func setConfigDefault(key string, value interface{}, check bool) int { func setConfigDefaults(check bool) error { added := 0 - // core settings - added += setConfigDefault("core.workers", 8, check) - // rclone settings added += setConfigDefault("rclone.path", "/usr/bin/rclone", check) added += setConfigDefault("rclone.config", "/Users/l3uddz/.config/rclone/rclone.conf", check) - added += setConfigDefault("rclone.dryrun", true, check) // were new settings added? if check && added > 0 { diff --git a/config/core.go b/config/core.go deleted file mode 100644 index c235f23..0000000 --- a/config/core.go +++ /dev/null @@ -1,5 +0,0 @@ -package config - -type CoreConfig struct { - Workers int -} diff --git a/config/rclone.go b/config/rclone.go index 53841c4..7739c81 100644 --- a/config/rclone.go +++ b/config/rclone.go @@ -3,5 +3,5 @@ package config type RcloneConfig struct { Path string Config string - DryRun bool + DryRun bool `mapstructure:"dry_run"` } diff --git a/config/uploader.go b/config/uploader.go index 091e49c..9f2f16b 100644 --- a/config/uploader.go +++ b/config/uploader.go @@ -12,6 +12,7 @@ type UploaderHidden struct { Type string Folder string Cleanup bool + Workers int } type UploaderRemotesMoveServerSide struct { diff --git a/uploader/cleans.go b/uploader/cleans.go index 15756a1..b715569 100644 --- a/uploader/cleans.go +++ b/uploader/cleans.go @@ -16,7 +16,6 @@ func (u *Uploader) PerformCleans(gp *gorpool.Pool) error { // perform clean files if len(u.HiddenFiles) > 0 { u.Log.Info("Performing clean of hidden files...") - for _, path := range u.HiddenFiles { p := path