From 801ed2e24ce64ace6caf243ef860bcf48a6fe04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Solbj=C3=B8rg?= Date: Tue, 26 Feb 2019 17:58:16 +0100 Subject: [PATCH 1/6] feat: Make it possible to supply a regex for matching files to remove in S3. --- main.go | 6 +++++ plugin.go | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/main.go b/main.go index 06b0756..39e1158 100644 --- a/main.go +++ b/main.go @@ -98,6 +98,11 @@ func main() { Name: "env-file", Usage: "source env file", }, + cli.StringFlag{ + Name: "target-remove", + Usage: "regex for which files to remove from target", + EnvVar: "PLUGIN_TARGET_REMOVE", + }, } if err := app.Run(os.Args); err != nil { @@ -125,6 +130,7 @@ func run(c *cli.Context) error { CacheControl: c.String("cache-control"), PathStyle: c.Bool("path-style"), DryRun: c.Bool("dry-run"), + TargetRemove: c.String("target-remove"), } return plugin.Exec() diff --git a/plugin.go b/plugin.go index 50b919f..8bc529d 100644 --- a/plugin.go +++ b/plugin.go @@ -4,6 +4,7 @@ import ( "mime" "os" "path/filepath" + "regexp" "strings" "github.com/aws/aws-sdk-go/aws" @@ -74,6 +75,8 @@ type Plugin struct { PathStyle bool // Dry run without uploading/ DryRun bool + // Regex for which files to remove from target + TargetRemove string } // Exec runs the plugin @@ -114,6 +117,81 @@ func (p *Plugin) Exec() error { return err } + if len(p.TargetRemove) != 0 { + reg, regexperr := regexp.Compile(p.TargetRemove) + + if regexperr != nil { + log.WithFields(log.Fields{ + "error": regexperr, + "regexp": p.TargetRemove, + }).Error("Regular expression failed to compile") + return regexperr + } + + log.WithFields(log.Fields{ + "regexp": p.TargetRemove, + }).Info("Deleting files according to regexp") + + log.Info("Listing files in bucket") // @ToDo: Log.Debug + list_input := &s3.ListObjectsInput{ + Bucket: &p.Bucket, + } + + s3_objects, list_err := client.ListObjects(list_input) + if list_err != nil { + log.WithFields(log.Fields{ + "error": list_err, + }).Error("Error listing objects from bucket") + return list_err + } + + var to_remove []string + for _, object := range s3_objects.Contents { + filename := object.Key + if reg.MatchString(*filename) { + to_remove = append(to_remove, *filename) + } + } + + if len(to_remove) > 0 { + log.WithFields(log.Fields{ + "num_files": len(to_remove), + }).Info("Deleting files from bucket") + + var remove_identifiers []*s3.ObjectIdentifier + for _, key := range to_remove { + id := s3.ObjectIdentifier{ + Key: aws.String(key), + } + remove_identifiers = append(remove_identifiers, &id) + } + + delete_input := &s3.DeleteObjectsInput{ + Bucket: &p.Bucket, + Delete: &s3.Delete{ + Objects: remove_identifiers, + Quiet: aws.Bool(false), + }, + } + + // when executing a dry-run we skip this step because we don't actually + // want to remove files from S3. + if !p.DryRun { + log.WithFields(log.Fields{ + "num_files": len(remove_identifiers), + }).Info("Attempting to delete files") + _, delete_err := client.DeleteObjects(delete_input) + + if delete_err != nil { + log.WithFields(log.Fields{ + "error": delete_err, + }).Error("Error deleting objects from S3") + return delete_err + } + } + } + } + for _, match := range matches { stat, err := os.Stat(match) From 8d206b7a75ed88b83d25debd7379703eec711c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Solbj=C3=B8rg?= Date: Wed, 27 Feb 2019 18:42:05 +0100 Subject: [PATCH 2/6] feat: Use globbing instead of regexp for path matching files to remove. --- go.mod | 1 + go.sum | 2 ++ main.go | 2 +- plugin.go | 32 +++++++++++++++++--------------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 10938b4..a1119b9 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/drone-plugins/drone-s3 require ( github.com/aws/aws-sdk-go v1.16.17 + github.com/bmatcuk/doublestar v1.1.1 github.com/joho/godotenv v1.3.0 github.com/mattn/go-zglob v0.0.1 github.com/sirupsen/logrus v1.3.0 diff --git a/go.sum b/go.sum index 0745407..2aa1ac9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/aws/aws-sdk-go v1.16.17 h1:hHRKZhoB4qEY17aGNp71UxQFyYpx6WZXGMUzx9y/A4w= github.com/aws/aws-sdk-go v1.16.17/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= diff --git a/main.go b/main.go index 39e1158..1013825 100644 --- a/main.go +++ b/main.go @@ -100,7 +100,7 @@ func main() { }, cli.StringFlag{ Name: "target-remove", - Usage: "regex for which files to remove from target", + Usage: "glob for which files to remove from target", EnvVar: "PLUGIN_TARGET_REMOVE", }, } diff --git a/plugin.go b/plugin.go index 8bc529d..b768c64 100644 --- a/plugin.go +++ b/plugin.go @@ -4,13 +4,13 @@ import ( "mime" "os" "path/filepath" - "regexp" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/bmatcuk/doublestar" "github.com/mattn/go-zglob" log "github.com/sirupsen/logrus" ) @@ -75,7 +75,7 @@ type Plugin struct { PathStyle bool // Dry run without uploading/ DryRun bool - // Regex for which files to remove from target + // Glob for which files to remove from target TargetRemove string } @@ -118,21 +118,12 @@ func (p *Plugin) Exec() error { } if len(p.TargetRemove) != 0 { - reg, regexperr := regexp.Compile(p.TargetRemove) - - if regexperr != nil { - log.WithFields(log.Fields{ - "error": regexperr, - "regexp": p.TargetRemove, - }).Error("Regular expression failed to compile") - return regexperr - } log.WithFields(log.Fields{ - "regexp": p.TargetRemove, - }).Info("Deleting files according to regexp") + "glob": p.TargetRemove, + }).Info("Deleting files according to glob") - log.Info("Listing files in bucket") // @ToDo: Log.Debug + log.Info("Listing files in bucket") list_input := &s3.ListObjectsInput{ Bucket: &p.Bucket, } @@ -148,7 +139,18 @@ func (p *Plugin) Exec() error { var to_remove []string for _, object := range s3_objects.Contents { filename := object.Key - if reg.MatchString(*filename) { + + globmatch, globerr := doublestar.PathMatch(p.TargetRemove, *filename) + + if globerr != nil { + log.WithFields(log.Fields{ + "error": globerr, + "glob": p.TargetRemove, + }).Error("Error with provided glob") + return globerr + } + + if globmatch { to_remove = append(to_remove, *filename) } } From d3ede087a8c12be5910139b86ffeeebc2a272343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Solbj=C3=B8rg?= Date: Thu, 7 Mar 2019 13:42:46 +0100 Subject: [PATCH 3/6] chore: Lint some variable names. --- plugin.go | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/plugin.go b/plugin.go index b768c64..e5901a0 100644 --- a/plugin.go +++ b/plugin.go @@ -124,54 +124,54 @@ func (p *Plugin) Exec() error { }).Info("Deleting files according to glob") log.Info("Listing files in bucket") - list_input := &s3.ListObjectsInput{ + listInput := &s3.ListObjectsInput{ Bucket: &p.Bucket, } - s3_objects, list_err := client.ListObjects(list_input) - if list_err != nil { + s3Objects, err := client.ListObjects(listInput) + if err != nil { log.WithFields(log.Fields{ - "error": list_err, + "error": err, }).Error("Error listing objects from bucket") - return list_err + return err } - var to_remove []string - for _, object := range s3_objects.Contents { + var toRemove []string + for _, object := range s3Objects.Contents { filename := object.Key - globmatch, globerr := doublestar.PathMatch(p.TargetRemove, *filename) + globmatch, err := doublestar.PathMatch(p.TargetRemove, *filename) - if globerr != nil { + if err != nil { log.WithFields(log.Fields{ - "error": globerr, + "error": err, "glob": p.TargetRemove, }).Error("Error with provided glob") - return globerr + return err } if globmatch { - to_remove = append(to_remove, *filename) + toRemove = append(toRemove, *filename) } } - if len(to_remove) > 0 { + if len(toRemove) > 0 { log.WithFields(log.Fields{ - "num_files": len(to_remove), + "num_files": len(toRemove), }).Info("Deleting files from bucket") - var remove_identifiers []*s3.ObjectIdentifier - for _, key := range to_remove { + var removeIdentifiers []*s3.ObjectIdentifier + for _, key := range toRemove { id := s3.ObjectIdentifier{ Key: aws.String(key), } - remove_identifiers = append(remove_identifiers, &id) + removeIdentifiers = append(removeIdentifiers, &id) } - delete_input := &s3.DeleteObjectsInput{ + deleteInput := &s3.DeleteObjectsInput{ Bucket: &p.Bucket, Delete: &s3.Delete{ - Objects: remove_identifiers, + Objects: removeIdentifiers, Quiet: aws.Bool(false), }, } @@ -180,15 +180,15 @@ func (p *Plugin) Exec() error { // want to remove files from S3. if !p.DryRun { log.WithFields(log.Fields{ - "num_files": len(remove_identifiers), + "num_files": len(removeIdentifiers), }).Info("Attempting to delete files") - _, delete_err := client.DeleteObjects(delete_input) + _, err := client.DeleteObjects(deleteInput) - if delete_err != nil { + if err != nil { log.WithFields(log.Fields{ - "error": delete_err, + "error": err, }).Error("Error deleting objects from S3") - return delete_err + return err } } } From ea8d4130a374b58cff02271a7e9b5b41b8a24b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Solbj=C3=B8rg?= Date: Sun, 10 Mar 2019 03:00:55 +0100 Subject: [PATCH 4/6] chore: Rename num_files to files. --- plugin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin.go b/plugin.go index e5901a0..f76193e 100644 --- a/plugin.go +++ b/plugin.go @@ -157,7 +157,7 @@ func (p *Plugin) Exec() error { if len(toRemove) > 0 { log.WithFields(log.Fields{ - "num_files": len(toRemove), + "files": len(toRemove), }).Info("Deleting files from bucket") var removeIdentifiers []*s3.ObjectIdentifier @@ -180,7 +180,7 @@ func (p *Plugin) Exec() error { // want to remove files from S3. if !p.DryRun { log.WithFields(log.Fields{ - "num_files": len(removeIdentifiers), + "files": len(removeIdentifiers), }).Info("Attempting to delete files") _, err := client.DeleteObjects(deleteInput) From caada99ceb279cb1fc2617d4f5d0a6ee11752dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Solbj=C3=B8rg?= Date: Sun, 10 Mar 2019 03:02:07 +0100 Subject: [PATCH 5/6] fix: Combine _, err assignment since we don't care about the value further on in the program. --- plugin.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugin.go b/plugin.go index f76193e..00ff5eb 100644 --- a/plugin.go +++ b/plugin.go @@ -182,9 +182,7 @@ func (p *Plugin) Exec() error { log.WithFields(log.Fields{ "files": len(removeIdentifiers), }).Info("Attempting to delete files") - _, err := client.DeleteObjects(deleteInput) - - if err != nil { + if _, err := client.DeleteObjects(deleteInput); err != nil { log.WithFields(log.Fields{ "error": err, }).Error("Error deleting objects from S3") From 648ca2c1f0429224cad98f56f661cc35f7a2805b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Solbj=C3=B8rg?= Date: Sun, 10 Mar 2019 03:03:00 +0100 Subject: [PATCH 6/6] fix: Propagate errors up to main.go to remove some logs. --- plugin.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/plugin.go b/plugin.go index 00ff5eb..7b61dec 100644 --- a/plugin.go +++ b/plugin.go @@ -130,9 +130,6 @@ func (p *Plugin) Exec() error { s3Objects, err := client.ListObjects(listInput) if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Error listing objects from bucket") return err } @@ -143,10 +140,6 @@ func (p *Plugin) Exec() error { globmatch, err := doublestar.PathMatch(p.TargetRemove, *filename) if err != nil { - log.WithFields(log.Fields{ - "error": err, - "glob": p.TargetRemove, - }).Error("Error with provided glob") return err } @@ -182,10 +175,8 @@ func (p *Plugin) Exec() error { log.WithFields(log.Fields{ "files": len(removeIdentifiers), }).Info("Attempting to delete files") + if _, err := client.DeleteObjects(deleteInput); err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Error deleting objects from S3") return err } }