Add flags/config -skip-local-files, -skip-local-dirs

Fixes #53
Closes #58

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.7
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0
 	github.com/bep/helpers v0.5.0
+	github.com/bep/predicate v0.2.0
 	github.com/dsnet/golib/memfile v1.0.0
 	github.com/frankban/quicktest v1.14.6
 	github.com/oklog/ulid/v2 v2.1.0

go.sum

@@ -34,6 +34,8 @@ github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/bep/helpers v0.5.0 h1:rneezhnG7GzLFlsEWO/EnleaBRuluBDGFimalO6Y50o=
 github.com/bep/helpers v0.5.0/go.mod h1:dSqCzIvHbzsk5YOesp1M7sKAq5xUcvANsRoKdawxH4Q=
+github.com/bep/predicate v0.2.0 h1:+jHhIbj1UOZn1POqZNKDryuJoi/9wPYg83siaRPb2b0=
+github.com/bep/predicate v0.2.0/go.mod h1:MQHXILk/U5Dg7eazQsAB69BrQrYSsl5jLlEejgBQyzg=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

lib/config.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 
 	"github.com/bep/helpers/envhelpers"
+	"github.com/bep/predicate"
 	"github.com/peterbourgon/ff/v3"
 	"gopkg.in/yaml.v2"
 )
@@ -44,7 +45,6 @@ func ConfigFromArgs(args []string) (*Config, error) {
 	}
 
 	return cfg, nil
-
 }
 
 // Config configures a deployment.
@@ -78,8 +78,17 @@ type Config struct {
 	Silent          bool
 	Force           bool
 	Try             bool
-	Ignore          string
-	IgnoreRE        *regexp.Regexp // compiled version of Ignore
+	Ignore          Strings
+
+	// One or more regular expressions of files to ignore when walking the local directory.
+	// If not set, defaults to ".DS_Store".
+	// Note that the path given will have Unix separators, regardless of the OS.
+	SkipLocalFiles Strings
+
+	// A list of regular expressions of directories to ignore when walking the local directory.
+	// If not set, defaults to ignoring hidden directories.
+	// Note that the path given will have Unix separators, regardless of the OS.
+	SkipLocalDirs Strings
 
 	// CLI state
 	PrintVersion bool
@@ -93,6 +102,11 @@ type Config struct {
 	fs *flag.FlagSet
 
 	initOnce sync.Once
+
+	// Compiled values.
+	skipLocalFiles predicate.P[string]
+	skipLocalDirs  predicate.P[string]
+	ignore         predicate.P[string]
 }
 
 func (cfg *Config) Usage() {
@@ -108,51 +122,30 @@ func (cfg *Config) Init() error {
 }
 
 func (cfg *Config) loadFileConfig() error {
-	configFile := cfg.ConfigFile
-
-	if configFile == "" {
-		return nil
-	}
-
-	data, err := os.ReadFile(configFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-
-	s := envhelpers.Expand(string(data), func(k string) string {
-		return os.Getenv(k)
-	})
-	data = []byte(s)
-
-	conf := fileConfig{}
-
-	err = yaml.Unmarshal(data, &conf)
-	if err != nil {
-		return err
-	}
-
-	for _, r := range conf.Routes {
-		r.routerRE, err = regexp.Compile(r.Route)
-
+	if cfg.ConfigFile != "" {
+		data, err := os.ReadFile(cfg.ConfigFile)
 		if err != nil {
-			return err
+			if !os.IsNotExist(err) {
+				return err
+			}
+		} else {
+			s := envhelpers.Expand(string(data), func(k string) string {
+				return os.Getenv(k)
+			})
+			data = []byte(s)
+
+			err = yaml.Unmarshal(data, &cfg.fileConf)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
-	cfg.fileConf = conf
-
-	return nil
+	return cfg.fileConf.init()
 }
 
 func (cfg *Config) shouldIgnoreLocal(key string) bool {
-	if cfg.Ignore == "" {
-		return false
-	}
-
-	return cfg.IgnoreRE.MatchString(key)
+	return cfg.ignore(key)
 }
 
 func (cfg *Config) shouldIgnoreRemote(key string) bool {
@@ -165,13 +158,14 @@ func (cfg *Config) shouldIgnoreRemote(key string) bool {
 		}
 	}
 
-	if cfg.Ignore == "" {
-		return false
-	}
-
-	return cfg.IgnoreRE.MatchString(sub)
+	return cfg.ignore(sub)
 }
 
+const (
+	defaultSkipLocalFiles = `^(.*/)?/?.DS_Store$`
+	defaultSkipLocalDirs  = `^\/?(?:\w+\/)*(\.\w+)`
+)
+
 func (cfg *Config) init() error {
 	if cfg.BucketName == "" {
 		return errors.New("AWS bucket is required")
@@ -209,12 +203,46 @@ func (cfg *Config) init() error {
 		return errors.New("you passed a value for the flags public-access and acl, which is not supported. the public-access flag is deprecated. please use the acl flag moving forward")
 	}
 
-	if cfg.Ignore != "" {
-		re, err := regexp.Compile(cfg.Ignore)
+	if cfg.Ignore != nil {
+		for _, pattern := range cfg.Ignore {
+			re, err := regexp.Compile(pattern)
+			if err != nil {
+				return errors.New("cannot compile 'ignore' flag pattern " + err.Error())
+			}
+			fn := func(s string) bool {
+				return re.MatchString(s)
+			}
+			cfg.ignore = cfg.ignore.Or(fn)
+		}
+	}
+
+	if cfg.SkipLocalFiles == nil {
+		cfg.SkipLocalFiles = Strings{defaultSkipLocalFiles}
+	}
+	if cfg.SkipLocalDirs == nil {
+		cfg.SkipLocalDirs = Strings{defaultSkipLocalDirs}
+	}
+
+	for _, pattern := range cfg.SkipLocalFiles {
+		re, err := regexp.Compile(pattern)
 		if err != nil {
-			return errors.New("cannot compile 'ignore' flag pattern " + err.Error())
+			return err
+		}
+		fn := func(s string) bool {
+			return re.MatchString(s)
 		}
-		cfg.IgnoreRE = re
+		cfg.skipLocalFiles = cfg.skipLocalFiles.Or(fn)
+	}
+
+	for _, pattern := range cfg.SkipLocalDirs {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return err
+		}
+		fn := func(s string) bool {
+			return re.MatchString(s)
+		}
+		cfg.skipLocalDirs = cfg.skipLocalDirs.Or(fn)
 	}
 
 	// load additional config (routes) from file if it exists.
@@ -253,7 +281,9 @@ func flagsToConfig(f *flag.FlagSet) *Config {
 	f.BoolVar(&cfg.PublicReadACL, "public-access", false, "DEPRECATED: please set -acl='public-read'")
 	f.StringVar(&cfg.ACL, "acl", "", "provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default \"private\")")
 	f.BoolVar(&cfg.Force, "force", false, "upload even if the etags match")
-	f.StringVar(&cfg.Ignore, "ignore", "", "regexp pattern for ignoring files")
+	f.Var(&cfg.Ignore, "ignore", "regexp pattern for ignoring files, repeat flag for multiple patterns,")
+	f.Var(&cfg.SkipLocalFiles, "skip-local-files", "regexp pattern of files to ignore when walking the local directory, repeat flag for multiple patterns, default "+defaultSkipLocalFiles)
+	f.Var(&cfg.SkipLocalDirs, "skip-local-dirs", "regexp pattern of files of directories to ignore when walking the local directory, repeat flag for multiple patterns, default "+defaultSkipLocalDirs)
 	f.BoolVar(&cfg.Try, "try", false, "trial run, no remote updates")
 	f.BoolVar(&cfg.Verbose, "v", false, "enable verbose logging")
 	f.BoolVar(&cfg.Silent, "quiet", false, "enable silent mode")
@@ -343,5 +373,4 @@ func valsToStrs(val interface{}) ([]string, error) {
 		return nil, err
 	}
 	return []string{s}, nil
-
 }

lib/config_test.go

@@ -78,7 +78,7 @@ routes:
       gzip: false
     - route: "^.+\\.(c)$"
       gzip: "${S3TEST_GZIP@U}"
-`), 0644), qt.IsNil)
+`), 0o644), qt.IsNil)
 
 	args := []string{
 		"-config=" + cfgFile,
@@ -96,7 +96,6 @@ routes:
 	c.Assert(routes[0].Headers["Cache-Control"], qt.Equals, "max-age=1234")
 	c.Assert(routes[0].Gzip, qt.IsTrue)
 	c.Assert(routes[2].Gzip, qt.IsTrue)
-
 }
 
 func TestConfigFromFileErrors(t *testing.T) {
@@ -105,7 +104,7 @@ func TestConfigFromFileErrors(t *testing.T) {
 	cfgFileInvalidYaml := filepath.Join(dir, "config_invalid_yaml.yml")
 	c.Assert(os.WriteFile(cfgFileInvalidYaml, []byte(`
 bucket=foo
-`), 0644), qt.IsNil)
+`), 0o644), qt.IsNil)
 
 	args := []string{
 		"-config=" + cfgFileInvalidYaml,
@@ -119,7 +118,7 @@ bucket=foo
 bucket: foo
 routes:
    - route: "*" # invalid regexp.
-`), 0644), qt.IsNil)
+`), 0o644), qt.IsNil)
 
 	args = []string{
 		"-config=" + cfgFileInvalidRoute,
@@ -129,7 +128,6 @@ routes:
 	c.Assert(err, qt.IsNil)
 	err = cfg.Init()
 	c.Assert(err, qt.IsNotNil)
-
 }
 
 func TestSetAclAndPublicAccessFlag(t *testing.T) {
@@ -196,3 +194,25 @@ func TestShouldIgnore(t *testing.T) {
 	c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/any"), qt.IsFalse)
 	c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/ignored-prefix/file.txt"), qt.IsTrue)
 }
+
+func TestSkipLocalDefault(t *testing.T) {
+	c := qt.New(t)
+
+	args := []string{
+		"-bucket=mybucket",
+	}
+
+	cfg, err := ConfigFromArgs(args)
+	c.Assert(err, qt.IsNil)
+	c.Assert(cfg.Init(), qt.IsNil)
+
+	c.Assert(cfg.skipLocalFiles("foo"), qt.IsFalse)
+	c.Assert(cfg.skipLocalDirs("foo"), qt.IsFalse)
+	c.Assert(cfg.skipLocalFiles(".DS_Store"), qt.IsTrue)
+	c.Assert(cfg.skipLocalFiles("a.DS_Store"), qt.IsFalse)
+	c.Assert(cfg.skipLocalFiles("foo/bar/.DS_Store"), qt.IsTrue)
+
+	c.Assert(cfg.skipLocalDirs("foo/bar/.git"), qt.IsTrue)
+	c.Assert(cfg.skipLocalDirs(".git"), qt.IsTrue)
+	c.Assert(cfg.skipLocalDirs("a.b"), qt.IsFalse)
+}

lib/deployer.go

@@ -13,7 +13,6 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"sync/atomic"
 	"time"
 
@@ -243,17 +242,18 @@ func (d *Deployer) walk(ctx context.Context, basePath string, files chan<- *osFi
 			return err
 		}
 
+		pathUnix := filepath.ToSlash(path)
+
 		if info.IsDir() {
 			// skip hidden directories like .git
-			if path != basePath && strings.HasPrefix(info.Name(), ".") {
+			if path != basePath && d.cfg.skipLocalDirs(pathUnix) {
 				return filepath.SkipDir
 			}
-
-			return nil
-		}
-
-		if info.Name() == ".DS_Store" {
 			return nil
+		} else {
+			if d.cfg.skipLocalFiles(pathUnix) {
+				return nil
+			}
 		}
 
 		if runtime.GOOS == "darwin" {

lib/files.go

@@ -238,6 +238,18 @@ type fileConfig struct {
 	Routes routes `yaml:"routes"`
 }
 
+func (c *fileConfig) init() error {
+	for _, r := range c.Routes {
+		var err error
+		r.routerRE, err = regexp.Compile(r.Route)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 type route struct {
 	Route   string            `yaml:"route"`
 	Headers map[string]string `yaml:"headers"`

testscripts/skipdirs_custom.txt

@@ -0,0 +1,23 @@
+env AWS_ACCESS_KEY_ID=$S3DEPLOY_TEST_KEY
+env AWS_SECRET_ACCESS_KEY=$S3DEPLOY_TEST_SECRET
+
+s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=public/ -skip-local-files 'foo' -skip-local-files bar -skip-local-dirs baz
+
+stdout 'Deleted 0 of 0, uploaded 2, skipped 0.*100% changed'
+stdout 'baz.txt \(not found\) ↑ index.html \(not found\) ↑ $'
+
+head /$S3DEPLOY_TEST_ID/
+stdout 'Status: 200'
+
+# By default we skip all . directories and the .DS_Store file.
+-- public/index.html --
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Test</title></head><body><h1>Test</h1></body></html>
+-- public/foo.txt --
+foo content.
+-- public/bar.txt --
+bar content.
+-- public/baz.txt --
+baz content.
+-- public/baz/moo.txt --
+moo content.
+

testscripts/skipdirs_default.txt

@@ -0,0 +1,20 @@
+env AWS_ACCESS_KEY_ID=$S3DEPLOY_TEST_KEY
+env AWS_SECRET_ACCESS_KEY=$S3DEPLOY_TEST_SECRET
+
+s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=public/
+
+stdout 'Deleted 0 of 0, uploaded 1, skipped 0.*100% changed'
+
+head /$S3DEPLOY_TEST_ID/
+stdout 'Status: 200'
+
+# By default we skip all . directories and the .DS_Store file.
+-- public/index.html --
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Test</title></head><body><h1>Test</h1></body></html>
+-- public/.hidden/foo.txt --
+foo content.
+-- public/.DS_Store --
+binary
+-- public/foo/.DS_Store --
+binary
+