From 761ccaf8fa061bbbdf6f87fb72057e560c505c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 16 Oct 2024 12:01:25 +0200 Subject: [PATCH] Add flags/config -skip-local-files, -skip-local-dirs Fixes #53 Closes #58 --- go.mod | 1 + go.sum | 2 + lib/config.go | 133 +++++++++++++++++++------------ lib/config_test.go | 30 +++++-- lib/deployer.go | 14 ++-- lib/files.go | 12 +++ testscripts/skipdirs_custom.txt | 23 ++++++ testscripts/skipdirs_default.txt | 20 +++++ 8 files changed, 171 insertions(+), 64 deletions(-) create mode 100644 testscripts/skipdirs_custom.txt create mode 100644 testscripts/skipdirs_default.txt diff --git a/go.mod b/go.mod index afb468a..d1e42dd 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 github.com/bep/helpers v0.5.0 + github.com/bep/predicate v0.2.0 github.com/dsnet/golib/memfile v1.0.0 github.com/frankban/quicktest v1.14.6 github.com/oklog/ulid/v2 v2.1.0 diff --git a/go.sum b/go.sum index b654138..b5753a4 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bep/helpers v0.5.0 h1:rneezhnG7GzLFlsEWO/EnleaBRuluBDGFimalO6Y50o= github.com/bep/helpers v0.5.0/go.mod h1:dSqCzIvHbzsk5YOesp1M7sKAq5xUcvANsRoKdawxH4Q= +github.com/bep/predicate v0.2.0 h1:+jHhIbj1UOZn1POqZNKDryuJoi/9wPYg83siaRPb2b0= +github.com/bep/predicate v0.2.0/go.mod h1:MQHXILk/U5Dg7eazQsAB69BrQrYSsl5jLlEejgBQyzg= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/lib/config.go b/lib/config.go index bfb5dc1..eb45cc1 100644 --- a/lib/config.go +++ b/lib/config.go @@ -20,6 +20,7 @@ import ( "sync" "github.com/bep/helpers/envhelpers" + "github.com/bep/predicate" "github.com/peterbourgon/ff/v3" "gopkg.in/yaml.v2" ) @@ -44,7 +45,6 @@ func ConfigFromArgs(args []string) (*Config, error) { } return cfg, nil - } // Config configures a deployment. @@ -78,8 +78,17 @@ type Config struct { Silent bool Force bool Try bool - Ignore string - IgnoreRE *regexp.Regexp // compiled version of Ignore + Ignore Strings + + // One or more regular expressions of files to ignore when walking the local directory. + // If not set, defaults to ".DS_Store". + // Note that the path given will have Unix separators, regardless of the OS. + SkipLocalFiles Strings + + // A list of regular expressions of directories to ignore when walking the local directory. + // If not set, defaults to ignoring hidden directories. + // Note that the path given will have Unix separators, regardless of the OS. + SkipLocalDirs Strings // CLI state PrintVersion bool @@ -93,6 +102,11 @@ type Config struct { fs *flag.FlagSet initOnce sync.Once + + // Compiled values. + skipLocalFiles predicate.P[string] + skipLocalDirs predicate.P[string] + ignore predicate.P[string] } func (cfg *Config) Usage() { @@ -108,51 +122,30 @@ func (cfg *Config) Init() error { } func (cfg *Config) loadFileConfig() error { - configFile := cfg.ConfigFile - - if configFile == "" { - return nil - } - - data, err := os.ReadFile(configFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - - s := envhelpers.Expand(string(data), func(k string) string { - return os.Getenv(k) - }) - data = []byte(s) - - conf := fileConfig{} - - err = yaml.Unmarshal(data, &conf) - if err != nil { - return err - } - - for _, r := range conf.Routes { - r.routerRE, err = regexp.Compile(r.Route) - + if cfg.ConfigFile != "" { + data, err := os.ReadFile(cfg.ConfigFile) if err != nil { - return err + if !os.IsNotExist(err) { + return err + } + } else { + s := envhelpers.Expand(string(data), func(k string) string { + return os.Getenv(k) + }) + data = []byte(s) + + err = yaml.Unmarshal(data, &cfg.fileConf) + if err != nil { + return err + } } } - cfg.fileConf = conf - - return nil + return cfg.fileConf.init() } func (cfg *Config) shouldIgnoreLocal(key string) bool { - if cfg.Ignore == "" { - return false - } - - return cfg.IgnoreRE.MatchString(key) + return cfg.ignore(key) } func (cfg *Config) shouldIgnoreRemote(key string) bool { @@ -165,13 +158,14 @@ func (cfg *Config) shouldIgnoreRemote(key string) bool { } } - if cfg.Ignore == "" { - return false - } - - return cfg.IgnoreRE.MatchString(sub) + return cfg.ignore(sub) } +const ( + defaultSkipLocalFiles = `^(.*/)?/?.DS_Store$` + defaultSkipLocalDirs = `^\/?(?:\w+\/)*(\.\w+)` +) + func (cfg *Config) init() error { if cfg.BucketName == "" { return errors.New("AWS bucket is required") @@ -209,12 +203,46 @@ func (cfg *Config) init() error { return errors.New("you passed a value for the flags public-access and acl, which is not supported. the public-access flag is deprecated. please use the acl flag moving forward") } - if cfg.Ignore != "" { - re, err := regexp.Compile(cfg.Ignore) + if cfg.Ignore != nil { + for _, pattern := range cfg.Ignore { + re, err := regexp.Compile(pattern) + if err != nil { + return errors.New("cannot compile 'ignore' flag pattern " + err.Error()) + } + fn := func(s string) bool { + return re.MatchString(s) + } + cfg.ignore = cfg.ignore.Or(fn) + } + } + + if cfg.SkipLocalFiles == nil { + cfg.SkipLocalFiles = Strings{defaultSkipLocalFiles} + } + if cfg.SkipLocalDirs == nil { + cfg.SkipLocalDirs = Strings{defaultSkipLocalDirs} + } + + for _, pattern := range cfg.SkipLocalFiles { + re, err := regexp.Compile(pattern) if err != nil { - return errors.New("cannot compile 'ignore' flag pattern " + err.Error()) + return err + } + fn := func(s string) bool { + return re.MatchString(s) } - cfg.IgnoreRE = re + cfg.skipLocalFiles = cfg.skipLocalFiles.Or(fn) + } + + for _, pattern := range cfg.SkipLocalDirs { + re, err := regexp.Compile(pattern) + if err != nil { + return err + } + fn := func(s string) bool { + return re.MatchString(s) + } + cfg.skipLocalDirs = cfg.skipLocalDirs.Or(fn) } // load additional config (routes) from file if it exists. @@ -253,7 +281,9 @@ func flagsToConfig(f *flag.FlagSet) *Config { f.BoolVar(&cfg.PublicReadACL, "public-access", false, "DEPRECATED: please set -acl='public-read'") f.StringVar(&cfg.ACL, "acl", "", "provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default \"private\")") f.BoolVar(&cfg.Force, "force", false, "upload even if the etags match") - f.StringVar(&cfg.Ignore, "ignore", "", "regexp pattern for ignoring files") + f.Var(&cfg.Ignore, "ignore", "regexp pattern for ignoring files, repeat flag for multiple patterns,") + f.Var(&cfg.SkipLocalFiles, "skip-local-files", "regexp pattern of files to ignore when walking the local directory, repeat flag for multiple patterns, default "+defaultSkipLocalFiles) + f.Var(&cfg.SkipLocalDirs, "skip-local-dirs", "regexp pattern of files of directories to ignore when walking the local directory, repeat flag for multiple patterns, default "+defaultSkipLocalDirs) f.BoolVar(&cfg.Try, "try", false, "trial run, no remote updates") f.BoolVar(&cfg.Verbose, "v", false, "enable verbose logging") f.BoolVar(&cfg.Silent, "quiet", false, "enable silent mode") @@ -343,5 +373,4 @@ func valsToStrs(val interface{}) ([]string, error) { return nil, err } return []string{s}, nil - } diff --git a/lib/config_test.go b/lib/config_test.go index 2730eb0..535a0b8 100644 --- a/lib/config_test.go +++ b/lib/config_test.go @@ -78,7 +78,7 @@ routes: gzip: false - route: "^.+\\.(c)$" gzip: "${S3TEST_GZIP@U}" -`), 0644), qt.IsNil) +`), 0o644), qt.IsNil) args := []string{ "-config=" + cfgFile, @@ -96,7 +96,6 @@ routes: c.Assert(routes[0].Headers["Cache-Control"], qt.Equals, "max-age=1234") c.Assert(routes[0].Gzip, qt.IsTrue) c.Assert(routes[2].Gzip, qt.IsTrue) - } func TestConfigFromFileErrors(t *testing.T) { @@ -105,7 +104,7 @@ func TestConfigFromFileErrors(t *testing.T) { cfgFileInvalidYaml := filepath.Join(dir, "config_invalid_yaml.yml") c.Assert(os.WriteFile(cfgFileInvalidYaml, []byte(` bucket=foo -`), 0644), qt.IsNil) +`), 0o644), qt.IsNil) args := []string{ "-config=" + cfgFileInvalidYaml, @@ -119,7 +118,7 @@ bucket=foo bucket: foo routes: - route: "*" # invalid regexp. -`), 0644), qt.IsNil) +`), 0o644), qt.IsNil) args = []string{ "-config=" + cfgFileInvalidRoute, @@ -129,7 +128,6 @@ routes: c.Assert(err, qt.IsNil) err = cfg.Init() c.Assert(err, qt.IsNotNil) - } func TestSetAclAndPublicAccessFlag(t *testing.T) { @@ -196,3 +194,25 @@ func TestShouldIgnore(t *testing.T) { c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/any"), qt.IsFalse) c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/ignored-prefix/file.txt"), qt.IsTrue) } + +func TestSkipLocalDefault(t *testing.T) { + c := qt.New(t) + + args := []string{ + "-bucket=mybucket", + } + + cfg, err := ConfigFromArgs(args) + c.Assert(err, qt.IsNil) + c.Assert(cfg.Init(), qt.IsNil) + + c.Assert(cfg.skipLocalFiles("foo"), qt.IsFalse) + c.Assert(cfg.skipLocalDirs("foo"), qt.IsFalse) + c.Assert(cfg.skipLocalFiles(".DS_Store"), qt.IsTrue) + c.Assert(cfg.skipLocalFiles("a.DS_Store"), qt.IsFalse) + c.Assert(cfg.skipLocalFiles("foo/bar/.DS_Store"), qt.IsTrue) + + c.Assert(cfg.skipLocalDirs("foo/bar/.git"), qt.IsTrue) + c.Assert(cfg.skipLocalDirs(".git"), qt.IsTrue) + c.Assert(cfg.skipLocalDirs("a.b"), qt.IsFalse) +} diff --git a/lib/deployer.go b/lib/deployer.go index 501d691..1d6abe6 100644 --- a/lib/deployer.go +++ b/lib/deployer.go @@ -13,7 +13,6 @@ import ( "path" "path/filepath" "runtime" - "strings" "sync/atomic" "time" @@ -243,17 +242,18 @@ func (d *Deployer) walk(ctx context.Context, basePath string, files chan<- *osFi return err } + pathUnix := filepath.ToSlash(path) + if info.IsDir() { // skip hidden directories like .git - if path != basePath && strings.HasPrefix(info.Name(), ".") { + if path != basePath && d.cfg.skipLocalDirs(pathUnix) { return filepath.SkipDir } - - return nil - } - - if info.Name() == ".DS_Store" { return nil + } else { + if d.cfg.skipLocalFiles(pathUnix) { + return nil + } } if runtime.GOOS == "darwin" { diff --git a/lib/files.go b/lib/files.go index 4e64b9a..0cb3e9f 100644 --- a/lib/files.go +++ b/lib/files.go @@ -238,6 +238,18 @@ type fileConfig struct { Routes routes `yaml:"routes"` } +func (c *fileConfig) init() error { + for _, r := range c.Routes { + var err error + r.routerRE, err = regexp.Compile(r.Route) + if err != nil { + return err + } + } + + return nil +} + type route struct { Route string `yaml:"route"` Headers map[string]string `yaml:"headers"` diff --git a/testscripts/skipdirs_custom.txt b/testscripts/skipdirs_custom.txt new file mode 100644 index 0000000..7b4aea4 --- /dev/null +++ b/testscripts/skipdirs_custom.txt @@ -0,0 +1,23 @@ +env AWS_ACCESS_KEY_ID=$S3DEPLOY_TEST_KEY +env AWS_SECRET_ACCESS_KEY=$S3DEPLOY_TEST_SECRET + +s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=public/ -skip-local-files 'foo' -skip-local-files bar -skip-local-dirs baz + +stdout 'Deleted 0 of 0, uploaded 2, skipped 0.*100% changed' +stdout 'baz.txt \(not found\) ↑ index.html \(not found\) ↑ $' + +head /$S3DEPLOY_TEST_ID/ +stdout 'Status: 200' + +# By default we skip all . directories and the .DS_Store file. +-- public/index.html -- +Test

Test

+-- public/foo.txt -- +foo content. +-- public/bar.txt -- +bar content. +-- public/baz.txt -- +baz content. +-- public/baz/moo.txt -- +moo content. + diff --git a/testscripts/skipdirs_default.txt b/testscripts/skipdirs_default.txt new file mode 100644 index 0000000..ac17a77 --- /dev/null +++ b/testscripts/skipdirs_default.txt @@ -0,0 +1,20 @@ +env AWS_ACCESS_KEY_ID=$S3DEPLOY_TEST_KEY +env AWS_SECRET_ACCESS_KEY=$S3DEPLOY_TEST_SECRET + +s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=public/ + +stdout 'Deleted 0 of 0, uploaded 1, skipped 0.*100% changed' + +head /$S3DEPLOY_TEST_ID/ +stdout 'Status: 200' + +# By default we skip all . directories and the .DS_Store file. +-- public/index.html -- +Test

Test

+-- public/.hidden/foo.txt -- +foo content. +-- public/.DS_Store -- +binary +-- public/foo/.DS_Store -- +binary +