move workers to hidden, add clean command and make dry-run via cli work

cmd/clean.go

@@ -0,0 +1,79 @@
+package cmd
+
+import (
+	"github.com/l3uddz/crop/config"
+	"github.com/l3uddz/crop/uploader"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/yale8848/gorpool"
+)
+
+var cleanCmd = &cobra.Command{
+	Use:   "clean [UPLOADER]",
+	Short: "Perform uploader clean",
+	Long:  `This command can be used to trigger an uploader clean.`,
+
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		// init core
+		initCore(true)
+
+		// iterate uploader's
+		for uploaderName, uploaderConfig := range config.Config.Uploader {
+			// skip disabled uploader(s)
+			if !uploaderConfig.Enabled {
+				log.WithField("uploader", uploaderName).Trace("Skipping disabled uploader")
+				continue
+			}
+
+			log := log.WithField("uploader", uploaderName)
+
+			// create uploader
+			upload, err := uploader.New(config.Config, &uploaderConfig, uploaderName)
+			if err != nil {
+				log.WithField("uploader", uploaderName).WithError(err).
+					Error("Failed initializing uploader, skipping...")
+				continue
+			}
+
+			log.Info("Clean commencing...")
+
+			// perform upload
+			if err := performClean(upload); err != nil {
+				upload.Log.WithError(err).Error("Error occurred while running clean, skipping...")
+				continue
+			}
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(cleanCmd)
+}
+
+func performClean(u *uploader.Uploader) error {
+	u.Log.Info("Running cleans...")
+
+	/* Cleans */
+	if u.Config.Hidden.Enabled {
+		// set worker count
+		workers := u.Config.Hidden.Workers
+		if workers == 0 {
+			workers = 8
+		}
+
+		// create worker pool
+		gp := gorpool.NewPool(workers, 0).
+			Start().
+			EnableWaitForAll(true)
+
+		// queue clean tasks
+		err := u.PerformCleans(gp)
+		if err != nil {
+			return errors.Wrap(err, "failed clearing remotes")
+		}
+	}
+
+	u.Log.Info("Finished cleans!")
+	return nil
+}

cmd/root.go

@@ -78,6 +78,8 @@ func initCore(showAppInfo bool) {
 		log.WithError(err).Fatal("Failed to initialize config")
 	}
 
+	setConfigOverrides()
+
 	// Init Cache
 	if err := cache.Init(flagCacheFile); err != nil {
 		log.WithError(err).Fatal("Failed to initialize cache")
@@ -94,6 +96,13 @@ func initCore(showAppInfo bool) {
 	}
 }
 
+func setConfigOverrides() {
+	// set dry-run if enabled by flag
+	if flagDryRun {
+		config.Config.Rclone.DryRun = true
+	}
+}
+
 func showUsing() {
 	// show app info
 	log.Infof("Using %s = %s (%s@%s)", stringutils.LeftJust("VERSION", " ", 10),

cmd/upload.go

@@ -7,7 +7,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/yale8848/gorpool"
 )
 
 var uploadCmd = &cobra.Command{
@@ -104,11 +103,7 @@ func performUpload(u *uploader.Uploader) error {
 
 	/* Cleans */
 	if u.Config.Hidden.Enabled {
-		gp := gorpool.NewPool(config.Config.Core.Workers, 0).
-			Start().
-			EnableWaitForAll(true)
-
-		err := u.PerformCleans(gp)
+		err := performClean(u)
 		if err != nil {
 			return errors.Wrap(err, "failed clearing remotes")
 		}
@@ -119,23 +114,35 @@ func performUpload(u *uploader.Uploader) error {
 
 	/* Copies */
 	if len(u.Config.Remotes.Copy) > 0 {
+		u.Log.Info("Running copies...")
+
 		if err := u.Copy(additionalRcloneParams); err != nil {
 			return errors.WithMessage(err, "failed performing all copies")
 		}
+
+		u.Log.Info("Finished copies!")
 	}
 
 	/* Move */
 	if len(u.Config.Remotes.Move) > 0 {
+		u.Log.Info("Running move...")
+
 		if err := u.Move(false, additionalRcloneParams); err != nil {
 			return errors.WithMessage(err, "failed performing move")
 		}
+
+		u.Log.Info("Finished move!")
 	}
 
 	/* Move Server Side */
 	if len(u.Config.Remotes.MoveServerSide) > 0 {
+		u.Log.Info("Running move server-sides...")
+
 		if err := u.Move(true, nil); err != nil {
 			return errors.WithMessage(err, "failed performing server-side moves")
 		}
+
+		u.Log.Info("Finished move server-sides!")
 	}
 
 	u.Log.Info("Finished!")

config/config.go

@@ -23,7 +23,6 @@ type BuildVars struct {
 }
 
 type Configuration struct {
-	Core     CoreConfig
 	Rclone   RcloneConfig
 	Uploader map[string]UploaderConfig
 }
@@ -129,13 +128,9 @@ func setConfigDefault(key string, value interface{}, check bool) int {
 func setConfigDefaults(check bool) error {
 	added := 0
 
-	// core settings
-	added += setConfigDefault("core.workers", 8, check)
-
 	// rclone settings
 	added += setConfigDefault("rclone.path", "/usr/bin/rclone", check)
 	added += setConfigDefault("rclone.config", "/Users/l3uddz/.config/rclone/rclone.conf", check)
-	added += setConfigDefault("rclone.dryrun", true, check)
 
 	// were new settings added?
 	if check && added > 0 {

config/rclone.go

@@ -3,5 +3,5 @@ package config
 type RcloneConfig struct {
 	Path   string
 	Config string
-	DryRun bool
+	DryRun bool `mapstructure:"dry_run"`
 }

config/uploader.go

@@ -12,6 +12,7 @@ type UploaderHidden struct {
 	Type    string
 	Folder  string
 	Cleanup bool
+	Workers int
 }
 
 type UploaderRemotesMoveServerSide struct {

uploader/cleans.go

@@ -16,7 +16,6 @@ func (u *Uploader) PerformCleans(gp *gorpool.Pool) error {
 	// perform clean files
 	if len(u.HiddenFiles) > 0 {
 		u.Log.Info("Performing clean of hidden files...")
-
 		for _, path := range u.HiddenFiles {
 			p := path