command/{cp, mv, rm}: add dry-run option

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## not released yet
 
+#### Features
+- Added `--dry-run` option for run, copy, move and remove operations. It displays which commands will be executed without actually having a side effect. ([#90](https://github.com/peak/s5cmd/issues/90))
+
 #### Improvements
 
 - For some operations errors were printed at the end of the program execution. Now, errors are displayed immediately after being detected. ([#136](https://github.com/peak/s5cmd/issues/136))

README.md

@@ -37,6 +37,7 @@ storage services and local filesystems.
 - Wildcard support for all operations
 - Multiple arguments support for delete operation
 - Command file support to run commands in batches at very high execution speeds
+- Dry run support for operations that have a side effect.
 - [S3 Transfer Acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html) support
 - Google Cloud Storage (and any other S3 API compatible service) support
 - Structured logging for querying command outputs
@@ -179,6 +180,11 @@ they'll be deleted in a single request.
 Will copy all the matching objects to the given S3 prefix, respecting the source
 folder hierarchy.
 
+`--dry-run` flag can be set to observe what operations will be performed without
+actually having `s5cmd` carry out those operations.
+
+    s5cmd cp --dry-run cp s3://bucket/pre/* s3://another-bucket/
+
 ⚠️ Copying objects (from S3 to S3) larger than 5GB is not supported yet. We have
 an [open ticket](https://github.com/peak/s5cmd/issues/29) to track the issue.
 

command/cp.go

@@ -70,6 +70,9 @@ Examples:
 
 	12. Perform KMS-SSE of the object(s) at the destination using customer managed Customer Master Key (CMK) key id
 		> s5cmd {{.HelpName}} -sse aws:kms -sse-kms-key-id <your-kms-key-id> s3://bucket/object s3://target-bucket/prefix/object
+
+	13. Check what s5cmd will do, without actually doing so
+		> s5cmd {{.HelpName}} --dry-run dir/ s3://bucket/ 
 `
 
 var copyCommandFlags = []cli.Flag{
@@ -125,6 +128,10 @@ var copyCommandFlags = []cli.Flag{
 		Name:  "acl",
 		Usage: "set acl for target: defines granted accesses and their types on different accounts/groups",
 	},
+	&cli.BoolFlag{
+		Name:  "dry-run",
+		Usage: "show what commands will be executed without actually executing them",
+	},
 }
 
 var copyCommand = &cli.Command{
@@ -153,6 +160,7 @@ var copyCommand = &cli.Command{
 			ifSourceNewer:    c.Bool("if-source-newer"),
 			flatten:          c.Bool("flatten"),
 			followSymlinks:   !c.Bool("no-follow-symlinks"),
+			dryRun:           c.Bool("dry-run") || c.Bool("dry-run-all"),
 			storageClass:     storage.StorageClass(c.String("storage-class")),
 			concurrency:      c.Int("concurrency"),
 			partSize:         c.Int64("part-size") * megabytes,
@@ -178,6 +186,7 @@ type Copy struct {
 	ifSourceNewer    bool
 	flatten          bool
 	followSymlinks   bool
+	dryRun           bool
 	storageClass     storage.StorageClass
 	encryptionMethod string
 	encryptionKeyID  string
@@ -310,7 +319,7 @@ func (c Copy) prepareDownloadTask(
 	isBatch bool,
 ) func() error {
 	return func() error {
-		dsturl, err := prepareLocalDestination(ctx, srcurl, dsturl, c.flatten, isBatch)
+		dsturl, err := prepareLocalDestination(ctx, srcurl, dsturl, c.flatten, isBatch, c.dryRun)
 		if err != nil {
 			return err
 		}
@@ -364,20 +373,23 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
 		return err
 	}
 
-	f, err := os.Create(dsturl.Absolute())
-	if err != nil {
-		return err
-	}
-	defer f.Close()
+	var size int64
+	if !c.dryRun {
+		f, err := os.Create(dsturl.Absolute())
+		if err != nil {
+			return err
+		}
+		defer f.Close()
 
-	size, err := srcClient.Get(ctx, srcurl, f, c.concurrency, c.partSize)
-	if err != nil {
-		_ = dstClient.Delete(ctx, dsturl)
-		return err
-	}
+		size, err = srcClient.Get(ctx, srcurl, f, c.concurrency, c.partSize)
+		if err != nil {
+			_ = dstClient.Delete(ctx, dsturl)
+			return err
+		}
 
-	if c.deleteSource {
-		_ = srcClient.Delete(ctx, srcurl)
+		if c.deleteSource {
+			_ = srcClient.Delete(ctx, srcurl)
+		}
 	}
 
 	msg := log.InfoMessage{
@@ -394,46 +406,50 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
 }
 
 func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) error {
-	// TODO(ig): use storage abstraction
-	f, err := os.Open(srcurl.Absolute())
-	if err != nil {
-		return err
-	}
-	defer f.Close()
 
-	err = c.shouldOverride(ctx, srcurl, dsturl)
-	if err != nil {
-		if errorpkg.IsWarning(err) {
-			printDebug(c.op, srcurl, dsturl, err)
-			return nil
+	var size int64
+	if !c.dryRun {
+		// TODO(ig): use storage abstraction
+		f, err := os.Open(srcurl.Absolute())
+		if err != nil {
+			return err
 		}
-		return err
-	}
+		defer f.Close()
 
-	dstClient := storage.NewClient(dsturl)
+		err = c.shouldOverride(ctx, srcurl, dsturl)
+		if err != nil {
+			if errorpkg.IsWarning(err) {
+				printDebug(c.op, srcurl, dsturl, err)
+				return nil
+			}
+			return err
+		}
 
-	metadata := storage.NewMetadata().
-		SetContentType(guessContentType(f)).
-		SetStorageClass(string(c.storageClass)).
-		SetSSE(c.encryptionMethod).
-		SetSSEKeyID(c.encryptionKeyID).
-		SetACL(c.acl)
+		dstClient := storage.NewClient(dsturl)
 
-	err = dstClient.Put(ctx, f, dsturl, metadata, c.concurrency, c.partSize)
-	if err != nil {
-		return err
-	}
+		metadata := storage.NewMetadata().
+			SetContentType(guessContentType(f)).
+			SetStorageClass(string(c.storageClass)).
+			SetSSE(c.encryptionMethod).
+			SetSSEKeyID(c.encryptionKeyID).
+			SetACL(c.acl)
 
-	srcClient := storage.NewClient(srcurl)
+		err = dstClient.Put(ctx, f, dsturl, metadata, c.concurrency, c.partSize)
+		if err != nil {
+			return err
+		}
 
-	obj, _ := srcClient.Stat(ctx, srcurl)
-	size := obj.Size
+		srcClient := storage.NewClient(srcurl)
 
-	if c.deleteSource {
-		// close the file before deleting
-		f.Close()
-		if err := srcClient.Delete(ctx, srcurl); err != nil {
-			return err
+		obj, _ := srcClient.Stat(ctx, srcurl)
+		size = obj.Size
+
+		if c.deleteSource {
+			// close the file before deleting
+			f.Close()
+			if err := srcClient.Delete(ctx, srcurl); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -469,15 +485,17 @@ func (c Copy) doCopy(ctx context.Context, srcurl *url.URL, dsturl *url.URL) erro
 		return err
 	}
 
-	err = srcClient.Copy(ctx, srcurl, dsturl, metadata)
-	if err != nil {
-		return err
-	}
-
-	if c.deleteSource {
-		if err := srcClient.Delete(ctx, srcurl); err != nil {
+	if !c.dryRun {
+		err = srcClient.Copy(ctx, srcurl, dsturl, metadata)
+		if err != nil {
 			return err
 		}
+
+		if c.deleteSource {
+			if err := srcClient.Delete(ctx, srcurl); err != nil {
+				return err
+			}
+		}
 	}
 
 	msg := log.InfoMessage{
@@ -573,13 +591,14 @@ func prepareLocalDestination(
 	dsturl *url.URL,
 	flatten bool,
 	isBatch bool,
+	dryRun bool,
 ) (*url.URL, error) {
 	objname := srcurl.Base()
 	if isBatch && !flatten {
 		objname = srcurl.Relative()
 	}
 
-	if isBatch {
+	if isBatch && !dryRun {
 		if err := os.MkdirAll(dsturl.Absolute(), os.ModePerm); err != nil {
 			return nil, err
 		}
@@ -594,15 +613,20 @@ func prepareLocalDestination(
 
 	if isBatch && !flatten {
 		dsturl = dsturl.Join(objname)
-		if err := os.MkdirAll(dsturl.Dir(), os.ModePerm); err != nil {
+		if dryRun {
+			// dryRun => no side effects.
+		} else if err := os.MkdirAll(dsturl.Dir(), os.ModePerm); err != nil {
 			return nil, err
 		}
 	}
 
 	if err == storage.ErrGivenObjectNotFound {
-		if err := os.MkdirAll(dsturl.Dir(), os.ModePerm); err != nil {
+		if dryRun {
+			// dryRun => no side effects.
+		} else if err := os.MkdirAll(dsturl.Dir(), os.ModePerm); err != nil {
 			return nil, err
 		}
+
 		if strings.HasSuffix(dsturl.Absolute(), "/") {
 			dsturl = dsturl.Join(objname)
 		}

command/mv.go

@@ -56,6 +56,7 @@ var moveCommand = &cli.Command{
 			ifSizeDiffer:     c.Bool("if-size-differ"),
 			ifSourceNewer:    c.Bool("if-source-newer"),
 			flatten:          c.Bool("flatten"),
+			dryRun:           c.Bool("dry-run") || c.Bool("dry-run-all"),
 			storageClass:     storage.StorageClass(c.String("storage-class")),
 			encryptionMethod: c.String("sse"),
 			encryptionKeyID:  c.String("sse-kms-key-id"),

command/rm.go

@@ -34,13 +34,22 @@ Examples:
 
 	4. Delete all matching objects and a specific object
 		 > s5cmd {{.HelpName}} s3://bucketname/prefix/* s3://bucketname/object1.gz
+
+	5. Check what s5cmd will do, without actually doing so
+		> s5cmd {{.HelpName}} --dry-run s3://bucket/prefix/*
 `
 
 var deleteCommand = &cli.Command{
 	Name:               "rm",
 	HelpName:           "rm",
 	Usage:              "remove objects",
 	CustomHelpTemplate: deleteHelpTemplate,
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "dry-run",
+			Usage: "show what commands will be executed without actually executing them",
+		},
+	},
 	Before: func(c *cli.Context) error {
 		err := validateRMCommand(c)
 		if err != nil {
@@ -49,25 +58,30 @@ var deleteCommand = &cli.Command{
 		return err
 	},
 	Action: func(c *cli.Context) error {
-		return Delete(
-			c.Context,
-			c.Command.Name,
-			givenCommand(c),
-			c.Args().Slice()...,
-		)
+		return Delete{
+			src:         c.Args().Slice(),
+			op:          c.Command.Name,
+			fullCommand: givenCommand(c),
+			dryRun:      c.Bool("dry-run") || c.Bool("dry-run-all"),
+		}.Run(c.Context)
 	},
 }
 
-// Delete remove given sources.
-func Delete(
-	ctx context.Context,
-	op string,
-	fullCommand string,
-	src ...string,
-) error {
-	srcurls, err := newURLs(src...)
+// Delete holds remove operation flags and states.
+type Delete struct {
+	src         []string
+	op          string
+	fullCommand string
+
+	// flags
+	dryRun bool
+}
+
+// Run removes given sources.
+func (d Delete) Run(ctx context.Context) error {
+	srcurls, err := newURLs(d.src...)
 	if err != nil {
-		printError(fullCommand, op, err)
+		printError(d.fullCommand, d.op, err)
 		return err
 	}
 	srcurl := srcurls[0]
@@ -87,12 +101,22 @@ func Delete(
 			}
 
 			if err := object.Err; err != nil {
-				printError(fullCommand, op, err)
+				printError(d.fullCommand, d.op, err)
 				continue
 			}
 			urlch <- object.URL
 		}
 	}()
+	if d.dryRun {
+		for u := range urlch {
+			msg := log.InfoMessage{
+				Operation: d.op,
+				Source:    u,
+			}
+			log.Info(msg)
+		}
+		return nil
+	}
 
 	resultch := client.MultiDelete(ctx, urlch)
 
@@ -104,12 +128,12 @@ func Delete(
 			}
 
 			merror = multierror.Append(merror, obj.Err)
-			printError(fullCommand, op, obj.Err)
+			printError(d.fullCommand, d.op, obj.Err)
 			continue
 		}
 
 		msg := log.InfoMessage{
-			Operation: op,
+			Operation: d.op,
 			Source:    obj.URL,
 		}
 		log.Info(msg)

command/run.go

@@ -32,10 +32,19 @@ Examples:
 		 > cat commands.txt | s5cmd {{.HelpName}}
 `
 
+var runCommandFlags = []cli.Flag{
+	&cli.BoolFlag{
+		Name:    "dry-run-all",
+		Aliases: []string{"dry-run"},
+		Usage:   "check what s5cmd will do, without making it perform any operations with side effects",
+	},
+}
+
 var runCommand = &cli.Command{
 	Name:               "run",
 	HelpName:           "run",
 	Usage:              "run commands in batch",
+	Flags:              runCommandFlags,
 	CustomHelpTemplate: runHelpTemplate,
 	Before: func(c *cli.Context) error {
 		err := validateRunCommand(c)