Skip to content

Commit

Permalink
Simplify walker to use PatternMatcher instead of custom matching
Browse files Browse the repository at this point in the history 
This changes the pattern matching implementation in walker to use
PatternMatcher, making it consistent with the "copy" code.

This makes it support ** patterns as well, which formerly didn't work.

The added benchmark shows the new implementation is slightly faster than
the old one.
  • Loading branch information
@aaronlehmann
aaronlehmann committed Aug 18, 2021
1 parent 4442383 commit e929e02
Show file tree
Hide file tree
Showing 4 changed files with 497 additions and 190 deletions.
45 changes: 0 additions & 45 deletions prefix/match.go
View file

This file was deleted.

57 changes: 0 additions & 57 deletions prefix/match_test.go
View file

This file was deleted.

224 changes: 138 additions & 86 deletions walker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

"github.com/docker/docker/pkg/fileutils"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/prefix"
"github.com/tonistiigi/fsutil/types"
)

Expand All @@ -36,20 +35,15 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR})
}

var pm *fileutils.PatternMatcher
if opt != nil && opt.ExcludePatterns != nil {
pm, err = fileutils.NewPatternMatcher(opt.ExcludePatterns)
if err != nil {
return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
}
}
var (
includePatterns []string
includeMatcher *fileutils.PatternMatcher
excludeMatcher *fileutils.PatternMatcher
)

var includePatterns []string
if opt != nil && opt.IncludePatterns != nil {
includePatterns = make([]string, len(opt.IncludePatterns))
for k := range opt.IncludePatterns {
includePatterns[k] = filepath.Clean(opt.IncludePatterns[k])
}
copy(includePatterns, opt.IncludePatterns)
}
if opt != nil && opt.FollowPaths != nil {
targets, err := FollowLinks(p, opt.FollowPaths)
Expand All @@ -61,13 +55,32 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
includePatterns = dedupePaths(includePatterns)
}
}
if len(includePatterns) != 0 {
includeMatcher, err = fileutils.NewPatternMatcher(includePatterns)
if err != nil {
return errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns)
}
}

var (
lastIncludedDir string
if opt != nil && opt.ExcludePatterns != nil {
excludeMatcher, err = fileutils.NewPatternMatcher(opt.ExcludePatterns)
if err != nil {
return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
}
}

parentDirs []string // used only for exclude handling
parentMatchedExclude []bool
)
type visitedDir struct {
fi os.FileInfo
path string
origpath string
pathWithSep string
matchedInclude bool
matchedExclude bool
calledFn bool
}

// used only for include/exclude handling
var parentDirs []visitedDir

seenFiles := make(map[uint64]string)
return filepath.Walk(root, func(path string, fi os.FileInfo, err error) (retErr error) {
Expand All @@ -90,87 +103,84 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
return nil
}

if opt != nil {
if includePatterns != nil {
skip := false
if lastIncludedDir != "" {
if strings.HasPrefix(path, lastIncludedDir+string(filepath.Separator)) {
skip = true
}
}
var dir visitedDir

if !skip {
matched := false
partial := true
for _, pattern := range includePatterns {
if ok, p := prefix.Match(pattern, path, false); ok {
matched = true
if !p {
partial = false
break
}
}
}
if !matched {
if fi.IsDir() {
return filepath.SkipDir
}
return nil
}
if !partial && fi.IsDir() {
lastIncludedDir = path
}
if includeMatcher != nil || excludeMatcher != nil {
for len(parentDirs) != 0 {
lastParentDir := parentDirs[len(parentDirs)-1].pathWithSep
if strings.HasPrefix(path, lastParentDir) {
break
}
parentDirs = parentDirs[:len(parentDirs)-1]
}
if pm != nil {
for len(parentMatchedExclude) != 0 {
lastParentDir := parentDirs[len(parentDirs)-1]
if strings.HasPrefix(path, lastParentDir) {
break
}
parentDirs = parentDirs[:len(parentDirs)-1]
parentMatchedExclude = parentMatchedExclude[:len(parentMatchedExclude)-1]
}

var m bool
if len(parentMatchedExclude) != 0 {
m, err = pm.MatchesUsingParentResult(path, parentMatchedExclude[len(parentMatchedExclude)-1])
} else {
m, err = pm.MatchesOrParentMatches(path)
if fi.IsDir() {
dir = visitedDir{
fi: fi,
path: path,
origpath: origpath,
pathWithSep: path + string(filepath.Separator),
}
if err != nil {
return errors.Wrap(err, "failed to match excludepatterns")
}
}

skip := false

if includeMatcher != nil {
var parentMatchedInclude *bool
if len(parentDirs) != 0 {
parentMatchedInclude = &parentDirs[len(parentDirs)-1].matchedInclude
}
m, err := matchesPatterns(includeMatcher, path, parentMatchedInclude)
if err != nil {
return errors.Wrap(err, "failed to match includepatterns")
}

if fi.IsDir() {
dir.matchedInclude = m
}

if !m {
skip = true
}
}

if excludeMatcher != nil {
var parentMatchedExclude *bool
if len(parentDirs) != 0 {
parentMatchedExclude = &parentDirs[len(parentDirs)-1].matchedExclude
}
m, err := matchesPatterns(excludeMatcher, path, parentMatchedExclude)
if err != nil {
return errors.Wrap(err, "failed to match excludepatterns")
}

if fi.IsDir() {
dir.matchedExclude = m
}

if m {
if fi.IsDir() && !excludeMatcher.Exclusions() {
return filepath.SkipDir
}
skip = true
}
}

var dirSlash string
if includeMatcher != nil || excludeMatcher != nil {
defer func() {
if fi.IsDir() {
dirSlash = path + string(filepath.Separator)
parentDirs = append(parentDirs, dirSlash)
parentMatchedExclude = append(parentMatchedExclude, m)
parentDirs = append(parentDirs, dir)
}
}()
}

if m {
if fi.IsDir() {
if !pm.Exclusions() {
return filepath.SkipDir
}
for _, pat := range pm.Patterns() {
if !pat.Exclusion() {
continue
}
patStr := pat.String() + string(filepath.Separator)
if strings.HasPrefix(patStr, dirSlash) {
goto passedFilter
}
}
return filepath.SkipDir
}
return nil
}
}
if skip {
return nil
}

passedFilter:
dir.calledFn = true

stat, err := mkstat(origpath, path, fi, seenFiles)
if err != nil {
return err
Expand All @@ -185,6 +195,31 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
return nil
}
}
for i, parentDir := range parentDirs {
if parentDir.calledFn {
continue
}
parentStat, err := mkstat(parentDir.origpath, parentDir.path, parentDir.fi, seenFiles)
if err != nil {
return err
}

select {
case <-ctx.Done():
return ctx.Err()
default:
}
if opt != nil && opt.Map != nil {
if allowed := opt.Map(parentStat.Path, parentStat); !allowed {
continue
}
}

if err := fn(parentStat.Path, &StatInfo{parentStat}, nil); err != nil {
return err
}
parentDirs[i].calledFn = true
}
if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil {
return err
}
Expand All @@ -193,6 +228,23 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
})
}

func matchesPatterns(pm *fileutils.PatternMatcher, path string, parentMatched *bool) (bool, error) {
var (
m bool
err error
)
if parentMatched != nil {
m, err = pm.MatchesUsingParentResult(path, *parentMatched)
} else {
m, err = pm.MatchesOrParentMatches(path)
}
if err != nil {
return false, err
}

return m, nil
}

type StatInfo struct {
*types.Stat
}
Expand Down
Loading

0 comments on commit e929e02

Please sign in to comment.