Skip to content

Commit

Permalink
rm: add timebased delete and object list from stdin (#1837)
Browse files Browse the repository at this point in the history 
* Add multi-delete support by taking object list from stdin
* Add support to `rm` to remove objects older than given time
* Add --prefix flag to differenciate prefix and target names.

Fixes #1827 #1828
  • Loading branch information
@balamurugana @harshavardhana
balamurugana authored and harshavardhana committed Sep 28, 2016
1 parent add7dcc commit 3094af8
Show file tree
Hide file tree
Showing 2 changed files with 156 additions and 27 deletions.
3 changes: 2 additions & 1 deletion cmd/mirror-main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"strings"
"sync"
"syscall"
"time"

"github.com/cheggaaa/pb"
"github.com/fatih/color"
Expand Down Expand Up @@ -193,7 +194,7 @@ func (ms *mirrorSession) doRemove(sURLs URLs) URLs {
isIncomplete := false

// Remove extraneous file on target.
err := rm(targetAlias, targetURL.String(), isIncomplete, isFake)
err := rm(targetAlias, targetURL.String(), isIncomplete, isFake, time.Duration(0))
if err != nil {
return sURLs.WithError(err.Trace(targetAlias, targetURL.String()))
}
Expand Down
180 changes: 154 additions & 26 deletions cmd/rm-main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,31 @@
package cmd

import (
"bufio"
"encoding/json"
"fmt"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"time"

"github.com/fatih/color"
"github.com/minio/cli"
"github.com/minio/mc/pkg/console"
"github.com/minio/minio/pkg/probe"
)

// default older time.
const defaultOlderTime = time.Hour

// rm specific flags.
var (
rmFlags = []cli.Flag{
cli.BoolFlag{
Name: "help, h",
Usage: "Help of rm.",
Usage: "Show this help.",
},
cli.BoolFlag{
Name: "recursive, r",
Expand All @@ -43,6 +51,10 @@ var (
Name: "force",
Usage: "Force a dangerous remove operation.",
},
cli.BoolFlag{
Name: "prefix",
Usage: "Remove objects matching this prefix.",
},
cli.BoolFlag{
Name: "incomplete, I",
Usage: "Remove an incomplete upload(s).",
Expand All @@ -51,6 +63,14 @@ var (
Name: "fake",
Usage: "Perform a fake remove operation.",
},
cli.BoolFlag{
Name: "stdin",
Usage: "Read object list from STDIN.",
},
cli.StringFlag{
Name: "older",
Usage: "Remove object only if its created older than given time.",
},
}
)

Expand All @@ -73,20 +93,23 @@ EXAMPLES:
1. Remove a file.
$ mc {{.Name}} 1999/old-backup.tgz
2. Remove contents of a folder, excluding its sub-folders.
$ mc {{.Name}} --force s3/jazz-songs/louis/
2. Remove all objects from music/ogg folder matching this prefix, excluding its sub-folders.
$ mc {{.Name}} --prefix s3/music/ogg/gunmetal
3. Remove contents of a folder recursively.
$ mc {{.Name}} --force --recursive s3/jazz-songs/louis/
3. Remove contents of a folder, excluding its sub-folders.
$ mc {{.Name}} --prefix s3/jazz-songs/louis/
4. Remove all matching objects with this prefix.
$ mc {{.Name}} --force s3/ogg/gunmetal
4. Remove contents of a folder recursively.
$ mc {{.Name}} --recursive s3/jazz-songs/louis/
5. Drop an incomplete upload of an object.
$ mc {{.Name}} --incomplete s3/jazz-songs/louis/file01.mp3
$ mc {{.Name}} --incomplete s3/jazz-songs/louis/file01.mp4
6. Drop all incomplete uploads recursively matching this prefix.
$ mc {{.Name}} --incomplete --force --recursive s3/jazz-songs/louis/
6. Remove all matching objects whose names are read from STDIN.
$ mc {{.Name}} --force --stdin
7. Remove object only if its created older than one day.
$ mc {{.Name}} --force --older=24h s3/jazz-songs/louis/
`,
}

Expand All @@ -112,41 +135,83 @@ func (r rmMessage) JSON() string {
func checkRmSyntax(ctx *cli.Context) {
// Set command flags from context.
isForce := ctx.Bool("force")
isPrefix := ctx.Bool("prefix")
isRecursive := ctx.Bool("recursive")
isFake := ctx.Bool("fake")
isStdin := ctx.Bool("stdin")
olderString := ctx.String("older")

if olderString != "" {
if older, err := time.ParseDuration(olderString); err != nil {
fatalIf(errDummy().Trace(), "Invalid older time format.")
} else if older < defaultOlderTime {
fatalIf(errDummy().Trace(), "older time should not be less than one hour.")
}

if !isStdin && !ctx.Args().Present() {
exitCode := 1
cli.ShowCommandHelpAndExit(ctx, "rm", exitCode)
}
}

if !ctx.Args().Present() {
if !ctx.Args().Present() && !isStdin {
exitCode := 1
cli.ShowCommandHelpAndExit(ctx, "rm", exitCode)
}

// For all recursive operations make sure to check for 'force' flag.
if isRecursive && !isForce && !isFake {
if (isPrefix || isRecursive || isStdin) && !isForce {
fatalIf(errDummy().Trace(),
"Recursive removal requires --force option. Please review carefully before performing this *DANGEROUS* operation.")
"Removal requires --force option. This operational is irreversible. Please review carefully before performing this *DANGEROUS* operation.")
}
}

// Remove a single object.
func rm(targetAlias, targetURL string, isIncomplete, isFake bool) *probe.Error {
func rmObject(targetAlias, targetURL string, isIncomplete bool) *probe.Error {
clnt, err := newClientFromAlias(targetAlias, targetURL)
if err != nil {
return err.Trace(targetURL)
}

if isFake { // It is a fake remove. Return success.
return nil
if err = clnt.Remove(isIncomplete); err != nil {
return err.Trace(targetURL)
}

if err = clnt.Remove(isIncomplete); err != nil {
return nil
}

// Remove a single object.
func rm(targetAlias, targetURL string, isIncomplete, isFake bool, older time.Duration) *probe.Error {
clnt, err := newClientFromAlias(targetAlias, targetURL)
if err != nil {
return err.Trace(targetURL)
}

// Check whether object is created older than given time only if older is >= one hour.
if older >= defaultOlderTime {
info, err := clnt.Stat()
if err != nil {
return err.Trace(targetURL)
}

now := time.Now().UTC()
timeDiff := now.Sub(info.Time)
if timeDiff < older {
// time difference of info.Time with current time is less than older duration.
return nil
}
}

if !isFake {
if err := rmObject(targetAlias, targetURL, isIncomplete); err != nil {
return err.Trace(targetURL)
}
}

return nil
}

// Remove all objects recursively.
func rmAll(targetAlias, targetURL string, isRecursive, isIncomplete, isFake bool) {
func rmAll(targetAlias, targetURL, prefix string, isRecursive, isIncomplete, isFake bool, older time.Duration) {
// Initialize new client.
clnt, err := newClientFromAlias(targetAlias, targetURL)
if err != nil {
Expand All @@ -163,20 +228,38 @@ func rmAll(targetAlias, targetURL string, isRecursive, isIncomplete, isFake bool
return // End of journey.
}

if !strings.HasPrefix(path.Base(entry.URL.Path), prefix) {
// Skip the entry if it doesn't starts with the prefix.
continue
}

if entry.Type.IsDir() && isRecursive {
// Add separator at the end to remove all its contents.
url := entry.URL
url.Path = strings.TrimSuffix(entry.URL.Path, string(entry.URL.Separator)) + string(entry.URL.Separator)

// Recursively remove contents of this directory.
rmAll(targetAlias, url.String(), isRecursive, isIncomplete, isFake)
rmAll(targetAlias, url.String(), prefix, isRecursive, isIncomplete, isFake, older)
}

// Check whether object is created older than given time only if older is >= one hour.
if older >= defaultOlderTime {
now := time.Now().UTC()
timeDiff := now.Sub(entry.Time)
if timeDiff < older {
// time difference of info.Time with current time is less than older duration.
continue
}
}

// Regular type.
if err = rm(targetAlias, entry.URL.String(), isIncomplete, isFake); err != nil {
errorIf(err.Trace(entry.URL.String()), "Unable to remove ‘"+entry.URL.String()+"’.")
continue
if !isFake {
if err = rmObject(targetAlias, entry.URL.String(), isIncomplete); err != nil {
errorIf(err.Trace(entry.URL.String()), "Unable to remove ‘"+entry.URL.String()+"’.")
continue
}
}

// Construct user facing message and path.
entryPath := filepath.ToSlash(filepath.Join(targetAlias, entry.URL.Path))
printMsg(rmMessage{Status: "success", URL: entryPath})
Expand All @@ -193,20 +276,65 @@ func mainRm(ctx *cli.Context) {

// rm specific flags.
isForce := ctx.Bool("force")
isPrefix := ctx.Bool("prefix")
isIncomplete := ctx.Bool("incomplete")
isRecursive := ctx.Bool("recursive")
isFake := ctx.Bool("fake")
isStdin := ctx.Bool("stdin")
olderString := ctx.String("older")
older, _ := time.ParseDuration(olderString)

// Set color.
console.SetColor("Remove", color.New(color.FgGreen, color.Bold))

// Support multiple targets.
for _, url := range ctx.Args() {
prefix := ""
if isPrefix {
if !strings.HasSuffix(url, "/") {
prefix = path.Base(url)
url = path.Dir(url) + "/"
} else if runtime.GOOS == "windows" && !strings.HasSuffix(url, `\`) {
prefix = path.Base(url)
url = path.Dir(url) + `\`
}
}

targetAlias, targetURL, _ := mustExpandAlias(url)
if (isPrefix || isRecursive) && isForce {
rmAll(targetAlias, targetURL, prefix, isRecursive, isIncomplete, isFake, older)
} else {
if err := rm(targetAlias, targetURL, isIncomplete, isFake, older); err != nil {
errorIf(err.Trace(url), "Unable to remove ‘"+url+"’.")
continue
}
printMsg(rmMessage{Status: "success", URL: url})
}
}

if !isStdin {
return
}

scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
url := scanner.Text()
prefix := ""
if isPrefix {
if strings.HasSuffix(url, "/") {
prefix = path.Base(url)
url = path.Dir(url)
} else if runtime.GOOS == "windows" && strings.HasSuffix(url, `\`) {
prefix = path.Base(url)
url = path.Dir(url)
}
}

targetAlias, targetURL, _ := mustExpandAlias(url)
if isRecursive && isForce || isFake {
rmAll(targetAlias, targetURL, isRecursive, isIncomplete, isFake)
if (isPrefix || isRecursive) && isForce {
rmAll(targetAlias, targetURL, prefix, isRecursive, isIncomplete, isFake, older)
} else {
if err := rm(targetAlias, targetURL, isIncomplete, isFake); err != nil {
if err := rm(targetAlias, targetURL, isIncomplete, isFake, older); err != nil {
errorIf(err.Trace(url), "Unable to remove ‘"+url+"’.")
continue
}
Expand Down

0 comments on commit 3094af8

Please sign in to comment.