From ad445f3167db590fa487a353d50ef79e9a9f2e3d Mon Sep 17 00:00:00 2001 From: Inhere Date: Sat, 3 Jun 2023 19:15:07 +0800 Subject: [PATCH] :fire: up: file/finder - refactoring the find elem filter logic --- fsutil/finder/config.go | 472 +++++++++++++++++++++++++++++++--- fsutil/finder/elem.go | 7 + fsutil/finder/filter.go | 55 ++-- fsutil/finder/filter_test.go | 16 -- fsutil/finder/filters.go | 356 +++++++++++++++++-------- fsutil/finder/filters_test.go | 107 +++++++- fsutil/finder/finder.go | 399 ++++++++++++---------------- fsutil/finder/finder_test.go | 188 ++++++++------ 8 files changed, 1088 insertions(+), 512 deletions(-) diff --git a/fsutil/finder/config.go b/fsutil/finder/config.go index a5665607b..02244f61c 100644 --- a/fsutil/finder/config.go +++ b/fsutil/finder/config.go @@ -1,9 +1,11 @@ package finder +import "strings" + // commonly dot file and dirs var ( - CommonlyDotDir = []string{".git", ".idea", ".vscode", ".svn", ".hg"} - CommonlyDotFile = []string{".gitignore", ".dockerignore", ".npmignore", ".DS_Store"} + CommonlyDotDirs = []string{".git", ".idea", ".vscode", ".svn", ".hg"} + CommonlyDotFiles = []string{".gitignore", ".dockerignore", ".npmignore", ".DS_Store", ".env"} ) // FindFlag type for find result. @@ -15,61 +17,473 @@ const ( FlagDir ) +// ToFlag convert string to FindFlag +func ToFlag(s string) FindFlag { + switch strings.ToLower(s) { + case "dir", "d": + return FlagDir + case "both", "b": + return FlagFile | FlagDir + default: + return FlagFile + } +} + // Config for finder type Config struct { - curDepth int + init bool + depth int - // DirPaths src paths for find file. - DirPaths []string + // ScanDirs scan dir paths for find. + ScanDirs []string // FindFlags for find result. default is FlagFile FindFlags FindFlag - // MaxDepth for find result. default is 0 + // MaxDepth for find result. default is 0 - not limit MaxDepth int + // UseAbsPath use abs path for find result. default is false + UseAbsPath bool // CacheResult cache result for find result. default is false CacheResult bool + // ExcludeDotDir exclude dot dir. default is true + ExcludeDotDir bool + // ExcludeDotFile exclude dot dir. default is false + ExcludeDotFile bool - // IncludeDirs name list. eg: {"model"} + // Filters generic include filters for filter file/dir elems + Filters []Filter + // ExFilters generic exclude filters for filter file/dir elems + ExFilters []Filter + // DirFilters include filters for dir elems + DirFilters []Filter + // DirFilters exclude filters for dir elems + DirExFilters []Filter + // FileFilters include filters for file elems + FileFilters []Filter + // FileExFilters exclude filters for file elems + FileExFilters []Filter + + // commonly settings for build filters + + // IncludeDirs include dir name list. eg: {"model"} IncludeDirs []string - // IncludeExts name list. eg: {".go", ".md"} + // IncludeExts include file ext name list. eg: {".go", ".md"} IncludeExts []string - // IncludeFiles name list. eg: {"go.mod"} + // IncludeFiles include file name list. eg: {"go.mod"} IncludeFiles []string - // IncludePaths list. eg: {"path/to"} + // IncludePaths include file/dir path list. eg: {"path/to"} IncludePaths []string + // IncludeNames include file/dir name list. eg: {"test", "some.go"} + IncludeNames []string - // ExcludeDirs name list. eg: {"test"} + // ExcludeDirs exclude dir name list. eg: {"test"} ExcludeDirs []string - // ExcludeExts name list. eg: {".go", ".md"} + // ExcludeExts exclude file ext name list. eg: {".go", ".md"} ExcludeExts []string - // ExcludeFiles name list. eg: {"go.mod"} + // ExcludeFiles exclude file name list. eg: {"go.mod"} ExcludeFiles []string - // ExcludePaths list. eg: {"path/to"} + // ExcludePaths exclude file/dir path list. eg: {"path/to"} ExcludePaths []string - // ExcludeNames file/dir name list. eg: {"test", "some.go"} + // ExcludeNames exclude file/dir name list. eg: {"test", "some.go"} ExcludeNames []string - - ExcludeDotDir bool - ExcludeDotFile bool - - // Filters generic filters for filter file/dir paths - Filters []Filter - - DirFilters []Filter // filters for filter dir paths - FileFilters []Filter // filters for filter file paths - BodyFilters []BodyFilter // filters for filter file body } // NewConfig create a new Config func NewConfig(dirs ...string) *Config { return &Config{ - DirPaths: dirs, + ScanDirs: dirs, FindFlags: FlagFile, // with default setting. ExcludeDotDir: true, } } -// NewFinder create a new FileFinder by config -func (c *Config) NewFinder() *FileFinder { - return NewWithConfig(c) +// NewFinder create a new Finder by config +func (c *Config) NewFinder() *Finder { + return NewWithConfig(c.Init()) +} + +// Init build filters by config and append to Filters. +func (c *Config) Init() *Config { + if c.init { + return c + } + + // generic filters + if len(c.IncludeNames) > 0 { + c.Filters = append(c.Filters, WithNames(c.IncludeNames)) + } + + if len(c.IncludePaths) > 0 { + c.Filters = append(c.Filters, WithPaths(c.IncludePaths)) + } + + if len(c.ExcludePaths) > 0 { + c.ExFilters = append(c.ExFilters, ExcludePaths(c.ExcludePaths)) + } + + if len(c.ExcludeNames) > 0 { + c.ExFilters = append(c.ExFilters, ExcludeNames(c.ExcludeNames)) + } + + // dir filters + if len(c.IncludeDirs) > 0 { + c.DirFilters = append(c.DirFilters, IncludeNames(c.IncludeDirs)) + } + + if len(c.ExcludeDirs) > 0 { + c.DirExFilters = append(c.DirExFilters, ExcludeNames(c.ExcludeDirs)) + } + + // file filters + if len(c.IncludeExts) > 0 { + c.FileFilters = append(c.FileFilters, IncludeExts(c.IncludeExts)) + } + + if len(c.IncludeFiles) > 0 { + c.FileFilters = append(c.FileFilters, IncludeNames(c.IncludeFiles)) + } + + if len(c.ExcludeExts) > 0 { + c.FileExFilters = append(c.FileExFilters, ExcludeExts(c.ExcludeExts)) + } + + if len(c.ExcludeFiles) > 0 { + c.FileExFilters = append(c.FileExFilters, ExcludeNames(c.ExcludeFiles)) + } + + return c +} + +// +// --------- config for finder --------- +// + +// WithConfig on the finder +func (f *Finder) WithConfig(c *Config) *Finder { + f.c = c + return f +} + +// ConfigFn the finder. alias of WithConfigFn() +func (f *Finder) ConfigFn(fns ...func(c *Config)) *Finder { + return f.WithConfigFn(fns...) +} + +// WithConfigFn the finder +func (f *Finder) WithConfigFn(fns ...func(c *Config)) *Finder { + if f.c == nil { + f.c = &Config{} + } + + for _, fn := range fns { + fn(f.c) + } + return f +} + +// AddScanDirs add source dir for find +func (f *Finder) AddScanDirs(dirPaths []string) *Finder { + f.c.ScanDirs = append(f.c.ScanDirs, dirPaths...) + return f +} + +// AddScanDir add source dir for find. alias of AddScanDirs() +func (f *Finder) AddScanDir(dirPaths ...string) *Finder { return f.AddScanDirs(dirPaths) } + +// ScanDir add source dir for find. alias of AddScanDirs() +func (f *Finder) ScanDir(dirPaths ...string) *Finder { return f.AddScanDirs(dirPaths) } + +// CacheResult cache result for find result. +func (f *Finder) CacheResult(enable ...bool) *Finder { + if len(enable) > 0 { + f.c.CacheResult = enable[0] + } else { + f.c.CacheResult = true + } + return f +} + +// WithCacheResult cache result for find result. alias of CacheResult() +func (f *Finder) WithCacheResult(enable ...bool) *Finder { + return f.CacheResult(enable...) +} + +// WithFlags set find flags. +func (f *Finder) WithFlags(flags FindFlag) *Finder { + f.c.FindFlags = flags + return f +} + +// WithStrFlag set find flags by string. +func (f *Finder) WithStrFlag(s string) *Finder { + f.c.FindFlags = ToFlag(s) + return f +} + +// OnlyFindDir only find dir. +func (f *Finder) OnlyFindDir() *Finder { return f.WithFlags(FlagDir) } + +// FileAndDir both find file and dir. +func (f *Finder) FileAndDir() *Finder { return f.WithFlags(FlagDir | FlagFile) } + +// UseAbsPath use absolute path for find result. alias of WithUseAbsPath() +func (f *Finder) UseAbsPath(enable ...bool) *Finder { return f.WithUseAbsPath(enable...) } + +// WithUseAbsPath use absolute path for find result. +func (f *Finder) WithUseAbsPath(enable ...bool) *Finder { + if len(enable) > 0 { + f.c.UseAbsPath = enable[0] + } else { + f.c.UseAbsPath = true + } + return f +} + +// WithMaxDepth set max depth for find. +func (f *Finder) WithMaxDepth(i int) *Finder { + f.c.MaxDepth = i + return f +} + +// IncludeDir include dir names. +func (f *Finder) IncludeDir(dirs ...string) *Finder { + f.c.IncludeDirs = append(f.c.IncludeDirs, dirs...) + return f +} + +// WithDirName include dir names. alias of IncludeDir() +func (f *Finder) WithDirName(dirs ...string) *Finder { return f.IncludeDir(dirs...) } + +// IncludeFile include file names. +func (f *Finder) IncludeFile(files ...string) *Finder { + f.c.IncludeFiles = append(f.c.IncludeFiles, files...) + return f +} + +// WithFileName include file names. alias of IncludeFile() +func (f *Finder) WithFileName(files ...string) *Finder { return f.IncludeFile(files...) } + +// IncludeName include file or dir names. +func (f *Finder) IncludeName(names ...string) *Finder { + f.c.IncludeNames = append(f.c.IncludeNames, names...) + return f +} + +// WithNames include file or dir names. alias of IncludeName() +func (f *Finder) WithNames(names []string) *Finder { return f.IncludeName(names...) } + +// IncludeExt include file exts. +func (f *Finder) IncludeExt(exts ...string) *Finder { + f.c.IncludeExts = append(f.c.IncludeExts, exts...) + return f +} + +// WithExts include file exts. alias of IncludeExt() +func (f *Finder) WithExts(exts []string) *Finder { return f.IncludeExt(exts...) } + +// WithFileExt include file exts. alias of IncludeExt() +func (f *Finder) WithFileExt(exts ...string) *Finder { return f.IncludeExt(exts...) } + +// IncludePath include file or dir paths. +func (f *Finder) IncludePath(paths ...string) *Finder { + f.c.IncludePaths = append(f.c.IncludePaths, paths...) + return f +} + +// WithPaths include file or dir paths. alias of IncludePath() +func (f *Finder) WithPaths(paths []string) *Finder { return f.IncludePath(paths...) } + +// WithSubPath include file or dir paths. alias of IncludePath() +func (f *Finder) WithSubPath(paths ...string) *Finder { return f.IncludePath(paths...) } + +// ExcludeDir exclude dir names. +func (f *Finder) ExcludeDir(dirs ...string) *Finder { + f.c.ExcludeDirs = append(f.c.ExcludeDirs, dirs...) + return f +} + +// WithoutDir exclude dir names. alias of ExcludeDir() +func (f *Finder) WithoutDir(dirs ...string) *Finder { return f.ExcludeDir(dirs...) } + +// WithoutNames exclude file or dir names. +func (f *Finder) WithoutNames(names []string) *Finder { + f.c.ExcludeNames = append(f.c.ExcludeNames, names...) + return f +} + +// ExcludeName exclude file names. alias of WithoutNames() +func (f *Finder) ExcludeName(names ...string) *Finder { return f.WithoutNames(names) } + +// ExcludeFile exclude file names. +func (f *Finder) ExcludeFile(files ...string) *Finder { + f.c.ExcludeFiles = append(f.c.ExcludeFiles, files...) + return f +} + +// WithoutFile exclude file names. alias of ExcludeFile() +func (f *Finder) WithoutFile(files ...string) *Finder { return f.ExcludeFile(files...) } + +// ExcludeExt exclude file exts. +// +// eg: ExcludeExt(".go", ".java") +func (f *Finder) ExcludeExt(exts ...string) *Finder { + f.c.ExcludeExts = append(f.c.ExcludeExts, exts...) + return f +} + +// WithoutExt exclude file exts. alias of ExcludeExt() +func (f *Finder) WithoutExt(exts ...string) *Finder { return f.ExcludeExt(exts...) } + +// WithoutExts exclude file exts. alias of ExcludeExt() +func (f *Finder) WithoutExts(exts []string) *Finder { return f.ExcludeExt(exts...) } + +// ExcludePath exclude file paths. +func (f *Finder) ExcludePath(paths ...string) *Finder { + f.c.ExcludePaths = append(f.c.ExcludePaths, paths...) + return f +} + +// WithoutPath exclude file paths. alias of ExcludePath() +func (f *Finder) WithoutPath(paths ...string) *Finder { return f.ExcludePath(paths...) } + +// WithoutPaths exclude file paths. alias of ExcludePath() +func (f *Finder) WithoutPaths(paths []string) *Finder { return f.ExcludePath(paths...) } + +// ExcludeDotDir exclude dot dir names. eg: ".idea" +func (f *Finder) ExcludeDotDir(exclude ...bool) *Finder { + if len(exclude) > 0 { + f.c.ExcludeDotDir = exclude[0] + } else { + f.c.ExcludeDotDir = true + } + return f +} + +// WithoutDotDir exclude dot dir names. alias of ExcludeDotDir(). +func (f *Finder) WithoutDotDir(exclude ...bool) *Finder { + return f.ExcludeDotDir(exclude...) +} + +// NoDotDir exclude dot dir names. alias of ExcludeDotDir(). +func (f *Finder) NoDotDir(exclude ...bool) *Finder { + return f.ExcludeDotDir(exclude...) +} + +// ExcludeDotFile exclude dot dir names. eg: ".gitignore" +func (f *Finder) ExcludeDotFile(exclude ...bool) *Finder { + if len(exclude) > 0 { + f.c.ExcludeDotFile = exclude[0] + } else { + f.c.ExcludeDotFile = true + } + return f +} + +// WithoutDotFile exclude dot dir names. alias of ExcludeDotFile(). +func (f *Finder) WithoutDotFile(exclude ...bool) *Finder { + return f.ExcludeDotFile(exclude...) +} + +// NoDotFile exclude dot dir names. alias of ExcludeDotFile(). +func (f *Finder) NoDotFile(exclude ...bool) *Finder { + return f.ExcludeDotFile(exclude...) +} + +// +// --------- add filters to finder --------- +// + +// Includes add include match filters +func (f *Finder) Includes(fls []Filter) *Finder { + f.c.Filters = append(f.c.Filters, fls...) + return f +} + +// Include add include match filters. alias of Includes() +func (f *Finder) Include(fls ...Filter) *Finder { return f.Includes(fls) } + +// With add include match filters. alias of Includes() +func (f *Finder) With(fls ...Filter) *Finder { return f.Includes(fls) } + +// Adds include match filters. alias of Includes() +func (f *Finder) Adds(fls []Filter) *Finder { return f.Includes(fls) } + +// Add include match filters. alias of Includes() +func (f *Finder) Add(fls ...Filter) *Finder { return f.Includes(fls) } + +// Excludes add exclude match filters +func (f *Finder) Excludes(fls []Filter) *Finder { + f.c.ExFilters = append(f.c.ExFilters, fls...) + return f +} + +// Exclude add exclude match filters. alias of Excludes() +func (f *Finder) Exclude(fls ...Filter) *Finder { return f.Excludes(fls) } + +// Without add exclude match filters. alias of Excludes() +func (f *Finder) Without(fls ...Filter) *Finder { return f.Excludes(fls) } + +// Nots add exclude match filters. alias of Excludes() +func (f *Finder) Nots(fls []Filter) *Finder { return f.Excludes(fls) } + +// Not add exclude match filters. alias of Excludes() +func (f *Finder) Not(fls ...Filter) *Finder { return f.Excludes(fls) } + +// WithFilters add include filters +func (f *Finder) WithFilters(fls []Filter) *Finder { + f.c.Filters = append(f.c.Filters, fls...) + return f +} + +// WithFilter add include filters +func (f *Finder) WithFilter(fls ...Filter) *Finder { return f.WithFilters(fls) } + +// MatchFiles add include file filters +func (f *Finder) MatchFiles(fls []Filter) *Finder { + f.c.FileFilters = append(f.c.FileFilters, fls...) + return f +} + +// MatchFile add include file filters +func (f *Finder) MatchFile(fls ...Filter) *Finder { return f.MatchFiles(fls) } + +// AddFiles add include file filters +func (f *Finder) AddFiles(fls []Filter) *Finder { return f.MatchFiles(fls) } + +// AddFile add include file filters +func (f *Finder) AddFile(fls ...Filter) *Finder { return f.MatchFiles(fls) } + +// NotFiles add exclude file filters +func (f *Finder) NotFiles(fls []Filter) *Finder { + f.c.FileExFilters = append(f.c.FileExFilters, fls...) + return f +} + +// NotFile add exclude file filters +func (f *Finder) NotFile(fls ...Filter) *Finder { return f.NotFiles(fls) } + +// MatchDirs add exclude dir filters +func (f *Finder) MatchDirs(fls []Filter) *Finder { + f.c.DirFilters = append(f.c.DirFilters, fls...) + return f +} + +// MatchDir add exclude dir filters +func (f *Finder) MatchDir(fls ...Filter) *Finder { return f.MatchDirs(fls) } + +// WithDirs add exclude dir filters +func (f *Finder) WithDirs(fls []Filter) *Finder { return f.MatchDirs(fls) } + +// WithDir add exclude dir filters +func (f *Finder) WithDir(fls ...Filter) *Finder { return f.MatchDirs(fls) } + +// NotDirs add exclude dir filters +func (f *Finder) NotDirs(fls []Filter) *Finder { + f.c.DirExFilters = append(f.c.DirExFilters, fls...) + return f +} + +// NotDir add exclude dir filters +func (f *Finder) NotDir(fls ...Filter) *Finder { + return f.NotDirs(fls) } diff --git a/fsutil/finder/elem.go b/fsutil/finder/elem.go index f07006a14..7ffa82c46 100644 --- a/fsutil/finder/elem.go +++ b/fsutil/finder/elem.go @@ -2,6 +2,8 @@ package finder import ( "io/fs" + + "github.com/gookit/goutil/strutil" ) // Elem of find file/dir result @@ -40,3 +42,8 @@ func (e *elem) Info() (fs.FileInfo, error) { } return e.stat, e.sErr } + +// String get string representation +func (e *elem) String() string { + return strutil.OrCond(e.IsDir(), "dir: ", "file: ") + e.Path() +} diff --git a/fsutil/finder/filter.go b/fsutil/finder/filter.go index b254245ec..25c17706c 100644 --- a/fsutil/finder/filter.go +++ b/fsutil/finder/filter.go @@ -2,7 +2,6 @@ package finder import ( "bytes" - "io/fs" "github.com/gookit/goutil/fsutil" ) @@ -21,38 +20,6 @@ func (fn FilterFunc) Apply(elem Elem) bool { return fn(elem) } -// ------------------ raw filter wrapper ------------------ - -// RawFilter for filter file path. -type RawFilter interface { - // Apply check file path. return False will filter this file. - Apply(fPath string, ent fs.DirEntry) bool -} - -// RawFilterFunc for filter file info, return False will filter this file -type RawFilterFunc func(fPath string, ent fs.DirEntry) bool - -// Apply check file path. return False will filter this file. -func (fn RawFilterFunc) Apply(fPath string, ent fs.DirEntry) bool { - return fn(fPath, ent) -} - -// WrapRawFilter wrap a RawFilter to Filter -func WrapRawFilter(rf RawFilter) Filter { - return FilterFunc(func(elem Elem) bool { - return rf.Apply(elem.Path(), elem) - }) -} - -// WrapRawFilters wrap RawFilter list to Filter list -func WrapRawFilters(rfs ...RawFilter) []Filter { - fls := make([]Filter, len(rfs)) - for i, rf := range rfs { - fls[i] = WrapRawFilter(rf) - } - return fls -} - // ------------------ Multi filter wrapper ------------------ // MultiFilter wrapper for multi filters @@ -117,6 +84,20 @@ type BodyFilters struct { } // NewBodyFilters create a new body filters +// +// Usage: +// +// bf := finder.NewBodyFilters( +// finder.BodyFilterFunc(func(filePath string, buf *bytes.Buffer) bool { +// // filter file contents +// return true +// }), +// ) +// +// es := finder.NewFinder('path/to/dir').WithFileFilter(bf).Elems() +// for el := range es { +// fmt.Println(el.Path()) +// } func NewBodyFilters(fls ...BodyFilter) *BodyFilters { return &BodyFilters{ Filters: fls, @@ -129,14 +110,14 @@ func (mf *BodyFilters) AddFilter(fls ...BodyFilter) { } // Apply check file path. return False will filter this file. -func (mf *BodyFilters) Apply(fPath string, ent fs.DirEntry) bool { - if ent.IsDir() { +func (mf *BodyFilters) Apply(el Elem) bool { + if el.IsDir() { return false } // read file contents buf := bytes.NewBuffer(nil) - file, err := fsutil.OpenReadFile(fPath) + file, err := fsutil.OpenReadFile(el.Path()) if err != nil { return false } @@ -150,7 +131,7 @@ func (mf *BodyFilters) Apply(fPath string, ent fs.DirEntry) bool { // apply filters for _, fl := range mf.Filters { - if !fl.Apply(fPath, buf) { + if !fl.Apply(el.Path(), buf) { return false } } diff --git a/fsutil/finder/filter_test.go b/fsutil/finder/filter_test.go index 7082d20d6..1ebe31c9b 100644 --- a/fsutil/finder/filter_test.go +++ b/fsutil/finder/filter_test.go @@ -1,17 +1 @@ package finder_test - -import ( - "testing" - - "github.com/gookit/goutil/fsutil/finder" - "github.com/gookit/goutil/testutil" - "github.com/gookit/goutil/testutil/assert" -) - -func TestFilterFunc(t *testing.T) { - fn := finder.FilterFunc(func(el finder.Elem) bool { - return false - }) - - assert.False(t, fn(finder.NewElem("path/some.txt", &testutil.DirEnt{}))) -} diff --git a/fsutil/finder/filters.go b/fsutil/finder/filters.go index 06a7001ee..d6cacf078 100644 --- a/fsutil/finder/filters.go +++ b/fsutil/finder/filters.go @@ -1,11 +1,13 @@ package finder import ( - "io/fs" "path" "regexp" "strings" + "time" + "github.com/gookit/goutil/fsutil" + "github.com/gookit/goutil/mathutil" "github.com/gookit/goutil/strutil" "github.com/gookit/goutil/timex" ) @@ -22,27 +24,64 @@ var OnlyDirFilter = FilterFunc(func(el Elem) bool { return el.IsDir() }) -// DotDirFilter filter dot dirname. eg: ".idea" -func DotDirFilter(include bool) FilterFunc { +// WithDotFile include dot filename. +func WithDotFile() FilterFunc { return dotFileFilter(true) } + +// WithoutDotFile exclude dot filename. +func WithoutDotFile() FilterFunc { return dotFileFilter(false) } + +// dotFileFilter filter dot filename. eg: ".gitignore" +func dotFileFilter(include bool) FilterFunc { return func(el Elem) bool { - if el.IsDir() && el.Path()[0] == '.' { + name := el.Name() + if len(name) > 0 && name[0] == '.' { return include } return !include } } -// OnlyFileFilter2 filter only file path. -func OnlyFileFilter2(exts ...string) FilterFunc { +// DotDirFilter filter dot dirname. eg: ".idea" +func DotDirFilter(include bool) FilterFunc { return func(el Elem) bool { - if el.IsDir() { - return false + if el.IsDir() && el.Name()[0] == '.' { + return include } + return !include + } +} + +// WithExt filter filepath by given file ext. +// +// Usage: +// +// f := NewFinder('path/to/dir') +// f.AddFilter(WithExt(".go")) +// f.AddExFilter(WithoutExt(".md")) +func WithExt(exts ...string) FilterFunc { return fileExtFilter(true, exts) } + +// WithExts filter filepath by given file ext. +func WithExts(exts []string) FilterFunc { return fileExtFilter(true, exts) } + +// IncludeExts filter filepath by given file ext. +func IncludeExts(exts []string) FilterFunc { return fileExtFilter(true, exts) } +// WithoutExt filter filepath by given file ext. +func WithoutExt(exts ...string) FilterFunc { return fileExtFilter(false, exts) } + +// WithoutExts filter filepath by given file ext. +func WithoutExts(exts []string) FilterFunc { return fileExtFilter(false, exts) } + +// ExcludeExts filter filepath by given file ext. +func ExcludeExts(exts []string) FilterFunc { return fileExtFilter(false, exts) } + +// fileExtFilter filter filepath by given file ext. +func fileExtFilter(include bool, exts []string) FilterFunc { + return func(el Elem) bool { if len(exts) == 0 { return true } - return isContains(path.Ext(el.Name()), exts, true) + return isContains(path.Ext(el.Name()), exts, include) } } @@ -55,37 +94,77 @@ func isContains(sub string, list []string, include bool) bool { return !include } -// ExtFilter filter filepath by given file ext. -// -// Usage: -// -// f := NewEmpty() -// f.AddFilter(ExtFilter(".go")) -// f.AddFilter(ExtFilter(".go", ".php")) -func ExtFilter(include bool, exts ...string) FilterFunc { +// WithName filter filepath by given names. +func WithName(names ...string) FilterFunc { return MatchNames(true, names) } + +// WithNames filter filepath by given names. +func WithNames(names []string) FilterFunc { return MatchNames(true, names) } + +// IncludeNames filter filepath by given names. +func IncludeNames(names []string) FilterFunc { return MatchNames(true, names) } + +// WithoutName filter filepath by given names. +func WithoutName(names ...string) FilterFunc { return MatchNames(false, names) } + +// WithoutNames filter filepath by given names. +func WithoutNames(names []string) FilterFunc { return MatchNames(false, names) } + +// ExcludeNames filter filepath by given names. +func ExcludeNames(names []string) FilterFunc { return MatchNames(false, names) } + +// MatchName filter filepath by given names. +func MatchName(names ...string) FilterFunc { return MatchNames(names) } + +// MatchNames filter filepath by given names. +func MatchNames(names []string) FilterFunc { return func(el Elem) bool { - if len(exts) == 0 { - return true + elName := el.Name() + for _, name := range names { + if name == elName || fsutil.PathMatch(name, elName) { + return true + } } - return isContains(path.Ext(el.Path()), exts, include) + return false } } -// NameFilter filter filepath by given names. -func NameFilter(include bool, names ...string) FilterFunc { +// WithPrefix include filepath by check given prefixes. +// +// Usage: +// +// f := NewFinder('path/to/dir') +// f.AddFilter(finder.WithPrefix("app_", "README")) +func WithPrefix(prefixes ...string) FilterFunc { return prefixFilter(true, prefixes...) } + +// WithoutPrefix exclude filepath by check given prefixes. +func WithoutPrefix(prefixes ...string) FilterFunc { return prefixFilter(false, prefixes...) } + +// prefixFilter filter filepath by check name has prefixes. +func prefixFilter(include bool, prefixes ...string) FilterFunc { return func(el Elem) bool { - return isContains(el.Name(), names, include) + for _, pfx := range prefixes { + if strings.HasPrefix(el.Name(), pfx) { + return include + } + } + return !include } } -// SuffixFilter filter filepath by check given suffixes. +// WithSuffix include filepath by check given suffixes. // // Usage: // -// f := EmptyFinder() -// f.AddFilter(finder.SuffixFilter(true, "util.go", "en.md")) -// f.AddFilter(finder.SuffixFilter(false, "_test.go", ".log")) -func SuffixFilter(include bool, suffixes ...string) FilterFunc { +// f := NewFinder('path/to/dir') +// f.AddFilter(finder.WithSuffix("util.go", "en.md")) +// f.AddFilter(finder.WithoutSuffix("_test.go", ".log")) +func WithSuffix(suffixes ...string) FilterFunc { return suffixFilter(true, suffixes...) } + +// WithoutSuffix exclude filepath by check given suffixes. +func WithoutSuffix(suffixes ...string) FilterFunc { return suffixFilter(false, suffixes...) } + +// suffixFilter filter filepath by check path has suffixes. +func suffixFilter(include bool, suffixes ...string) FilterFunc { return func(el Elem) bool { for _, sfx := range suffixes { if strings.HasSuffix(el.Path(), sfx) { @@ -96,12 +175,31 @@ func SuffixFilter(include bool, suffixes ...string) FilterFunc { } } -// PathFilter filter filepath by given sub paths. +// WithPath include file/dir by given sub paths. // // Usage: // -// f.AddFilter(PathFilter(true, "need/path")) -func PathFilter(include bool, subPaths ...string) FilterFunc { +// f := NewFinder('path/to/dir') +// f.AddFilter(WithPath("need/path")) +func WithPath(subPaths ...string) FilterFunc { return pathFilter(true, subPaths) } + +// WithPaths include file/dir by given sub paths. +func WithPaths(subPaths []string) FilterFunc { return pathFilter(true, subPaths) } + +// IncludePaths include file/dir by given sub paths. +func IncludePaths(subPaths []string) FilterFunc { return pathFilter(true, subPaths) } + +// WithoutPath exclude file/dir by given sub paths. +func WithoutPath(subPaths ...string) FilterFunc { return pathFilter(false, subPaths) } + +// WithoutPaths exclude file/dir by given sub paths. +func WithoutPaths(subPaths []string) FilterFunc { return pathFilter(false, subPaths) } + +// ExcludePaths exclude file/dir by given sub paths. +func ExcludePaths(subPaths []string) FilterFunc { return pathFilter(false, subPaths) } + +// pathFilter filter file/dir by given sub paths. +func pathFilter(include bool, subPaths []string) FilterFunc { return func(el Elem) bool { for _, subPath := range subPaths { if strings.Contains(el.Path(), subPath) { @@ -112,27 +210,27 @@ func PathFilter(include bool, subPaths ...string) FilterFunc { } } -// DotFileFilter filter dot filename. eg: ".gitignore" -func DotFileFilter(include bool) FilterFunc { - return func(el Elem) bool { - name := el.Name() - if len(name) > 0 && name[0] == '.' { - return include - } - return !include - } -} - -// GlobFilterFunc filter filepath by given patterns. +// WithGlobMatch include filepath by given patterns. // // Usage: // -// f := EmptyFiler() -// f.AddFilter(GlobFilterFunc(true, "*_test.go")) -func GlobFilterFunc(include bool, patterns ...string) FilterFunc { +// f := NewFinder('path/to/dir') +// f.AddFilter(WithGlobMatch("*_test.go")) +func WithGlobMatch(patterns ...string) FilterFunc { return globFilter(true, patterns) } + +func WithGlobMatches(patterns []string) FilterFunc { return globFilter(true, patterns) } + +// WithoutGlobMatch exclude filepath by given patterns. +func WithoutGlobMatch(patterns ...string) FilterFunc { return globFilter(false, patterns) } + +// WithoutGlobMatches exclude filepath by given patterns. +func WithoutGlobMatches(patterns []string) FilterFunc { return globFilter(false, patterns) } + +// GlobFilter filter filepath by given patterns. +func globFilter(include bool, patterns []string) FilterFunc { return func(el Elem) bool { for _, pattern := range patterns { - if ok, _ := path.Match(pattern, el.Path()); ok { + if ok, _ := path.Match(pattern, el.Name()); ok { return include } } @@ -140,13 +238,19 @@ func GlobFilterFunc(include bool, patterns ...string) FilterFunc { } } -// RegexFilterFunc filter filepath by given regex pattern +// WithRegexMatch include filepath by given regex pattern // // Usage: // -// f := EmptyFiler() -// f.AddFilter(RegexFilterFunc(`[A-Z]\w+`, true)) -func RegexFilterFunc(pattern string, include bool) FilterFunc { +// f := NewFinder('path/to/dir') +// f.AddFilter(WithRegexMatch(`[A-Z]\w+`)) +func WithRegexMatch(pattern string) FilterFunc { return regexFilter(pattern, true) } + +// WithoutRegexMatch exclude filepath by given regex pattern +func WithoutRegexMatch(pattern string) FilterFunc { return regexFilter(pattern, false) } + +// regexFilter filter filepath by given regex pattern +func regexFilter(pattern string, include bool) FilterFunc { reg := regexp.MustCompile(pattern) return func(el Elem) bool { @@ -157,76 +261,121 @@ func RegexFilterFunc(pattern string, include bool) FilterFunc { } } +// WithNameLike include filepath by given name match. +func WithNameLike(patterns ...string) FilterFunc { return nameLikeFilter(true, patterns) } + +// WithNameLikes include filepath by given name match. +func WithNameLikes(patterns []string) FilterFunc { return nameLikeFilter(true, patterns) } + +// WithoutNameLike exclude filepath by given name match. +func WithoutNameLike(patterns ...string) FilterFunc { return nameLikeFilter(false, patterns) } + +// WithoutNameLikes exclude filepath by given name match. +func WithoutNameLikes(patterns []string) FilterFunc { return nameLikeFilter(false, patterns) } + +// nameLikeFilter filter filepath by given name match. +func nameLikeFilter(include bool, patterns []string) FilterFunc { + return func(el Elem) bool { + for _, pattern := range patterns { + if strutil.LikeMatch(pattern, el.Name()) { + return include + } + } + return !include + } +} + // // ----------------- built in file info filters ----------------- // -// ModTimeFilter filter file by modify time. +// WithModTime filter file by modify time. +// +// Note: if time is zero, it will be ignored. // // Usage: // -// f := EmptyFinder() -// f.AddFilter(ModTimeFilter(600, '>', true)) // 600 seconds to Now(last 10 minutes -// f.AddFilter(ModTimeFilter(600, '<', false)) // before 600 seconds(before 10 minutes) -func ModTimeFilter(limitSec int, op rune, include bool) FilterFunc { +// f := NewFinder('path/to/dir') +// // -600 seconds to now(last 10 minutes) +// f.AddFilter(WithModTime(timex.NowAddSec(-600), timex.ZeroTime)) +// // before 600 seconds(before 10 minutes) +// f.AddFilter(WithModTime(timex.ZeroTime, timex.NowAddSec(-600))) +func WithModTime(start, end time.Time) FilterFunc { + return modTimeFilter(start, end, true) +} + +// WithoutModTime filter file by modify time. +func WithoutModTime(start, end time.Time) FilterFunc { + return modTimeFilter(start, end, false) +} + +// modTimeFilter filter file by modify time. +func modTimeFilter(start, end time.Time, include bool) FilterFunc { return func(el Elem) bool { fi, err := el.Info() if err != nil { return !include } - lt := timex.Now().AddSeconds(-limitSec) - if op == '>' { - if lt.After(fi.ModTime()) { - return include - } - return !include - } - - // '<' - if lt.Before(fi.ModTime()) { + if timex.InRange(fi.ModTime(), start, end) { return include } return !include } } -// HumanModTimeFilter filter file by modify time string. +// WithHumanModTime filter file by modify time string. // // Usage: // // f := EmptyFinder() -// f.AddFilter(HumanModTimeFilter("10m", '>', true)) // 10 minutes to Now -// f.AddFilter(HumanModTimeFilter("10m", '<', false)) // before 10 minutes -func HumanModTimeFilter(limit string, op rune, include bool) FilterFunc { - return func(elem Elem) bool { - fi, err := elem.Info() - if err != nil { - return !include +// f.AddFilter(WithHumanModTime(">10m")) // before 10 minutes +// f.AddFilter(WithHumanModTime("<10m")) // latest 10 minutes, to Now +func WithHumanModTime(expr string) FilterFunc { return humanModTimeFilter(expr, true) } + +// WithoutHumanModTime filter file by modify time string. +func WithoutHumanModTime(expr string) FilterFunc { return humanModTimeFilter(expr, false) } + +var timeNumReg = regexp.MustCompile(`(-?\d+)`) + +// humanModTimeFilter filter file by modify time string. +func humanModTimeFilter(expr string, include bool) FilterFunc { + opt := &timex.ParseRangeOpt{AutoSort: true} + // convert > to <, < to > + expr = strutil.Replaces(expr, map[string]string{">": "<", "<": ">"}) + expr = timeNumReg.ReplaceAllStringFunc(expr, func(s string) string { + if s[0] == '-' { + return s } + return "-" + s + }) - lt, err := strutil.ToDuration(limit) - if err != nil { - return !include - } + start, end, err := timex.ParseRange(expr, opt) + if err != nil { + panic(err) + } - if op == '>' { - if timex.Now().Add(-lt).After(fi.ModTime()) { - return include - } + return func(elem Elem) bool { + fi, err := elem.Info() + if err != nil { return !include } - // '<' - if timex.Now().Add(-lt).Before(fi.ModTime()) { + if timex.InRange(fi.ModTime(), start, end) { return include } return !include } } -// FileSizeFilter filter file by file size. -func FileSizeFilter(min, max int64, include bool) FilterFunc { +// WithFileSize filter file by file size. unit: byte +func WithFileSize(min, max uint64) FilterFunc { return fileSizeFilter(min, max, true) } + +// WithoutFileSize filter file by file size. unit: byte +func WithoutFileSize(min, max uint64) FilterFunc { return fileSizeFilter(min, max, false) } + +// fileSizeFilter filter file by file size. unit: byte +func fileSizeFilter(min, max uint64, include bool) FilterFunc { return func(el Elem) bool { if el.IsDir() { return false @@ -237,18 +386,22 @@ func FileSizeFilter(min, max int64, include bool) FilterFunc { return false } - return ByteSizeCheck(fi, min, max, include) + if mathutil.InUintRange(uint64(fi.Size()), min, max) { + return include + } + return !include } } -// HumanSizeFilter filter file by file size string. eg: 1KB, 2MB, 3GB -func HumanSizeFilter(min, max string, include bool) FilterFunc { - minSize, err := strutil.ToByteSize(min) - if err != nil { - panic(err) - } +// WithHumanSize filter file by file size string. +func WithHumanSize(expr string) FilterFunc { return humanSizeFilter(expr, true) } - maxSize, err := strutil.ToByteSize(max) +// WithoutHumanSize filter file by file size string. +func WithoutHumanSize(expr string) FilterFunc { return humanSizeFilter(expr, false) } + +// humanSizeFilter filter file by file size string. eg: ">1k", "<2m", "1g~3g" +func humanSizeFilter(expr string, include bool) FilterFunc { + min, max, err := strutil.ParseSizeRange(expr, nil) if err != nil { panic(err) } @@ -263,18 +416,9 @@ func HumanSizeFilter(min, max string, include bool) FilterFunc { return false } - return ByteSizeCheck(fi, int64(minSize), int64(maxSize), include) - } -} - -// ByteSizeCheck filter file by file size. -func ByteSizeCheck(fi fs.FileInfo, min, max int64, include bool) bool { - if min > 0 && fi.Size() < min { - return !include - } - - if max > 0 && fi.Size() > max { + if mathutil.InUintRange(uint64(fi.Size()), min, max) { + return include + } return !include } - return include } diff --git a/fsutil/finder/filters_test.go b/fsutil/finder/filters_test.go index e7b850253..909030098 100644 --- a/fsutil/finder/filters_test.go +++ b/fsutil/finder/filters_test.go @@ -1,6 +1,7 @@ package finder_test import ( + "fmt" "testing" "github.com/gookit/goutil/fsutil/finder" @@ -8,23 +9,113 @@ import ( "github.com/gookit/goutil/testutil/assert" ) -func TestRegexFilterFunc(t *testing.T) { +func newMockElem(fp string, isDir ...bool) finder.Elem { + return finder.NewElem(fp, testutil.NewDirEnt(fp, isDir...)) +} + +func TestFilters_simple(t *testing.T) { + el := newMockElem("path/some.txt") + fn := finder.FilterFunc(func(el finder.Elem) bool { + return false + }) + + assert.False(t, fn(el)) + + // with name + fn = finder.WithName("some.txt") + assert.True(t, fn(el)) + fn = finder.WithName("not-exist.txt") + assert.False(t, fn(el)) + + // without name + fn = finder.WithoutName("some.txt") + assert.False(t, fn(el)) + fn = finder.WithoutName("not-exist.txt") + assert.True(t, fn(el)) + + // with ext + fn = finder.WithExt(".txt") + assert.True(t, fn(el)) + fn = finder.WithExt(".js") + assert.False(t, fn(el)) + + // without ext + fn = finder.WithoutExt(".txt") + assert.False(t, fn(el)) + fn = finder.WithoutExt(".js") + assert.True(t, fn(el)) + + // with suffix + fn = finder.WithSuffix("me.txt") + assert.True(t, fn(el)) + fn = finder.WithSuffix("not-exist.txt") + assert.False(t, fn(el)) +} + +func TestExtFilterFunc(t *testing.T) { + ent := &testutil.DirEnt{} + + fn := finder.WithExt(".log") + assert.True(t, fn(finder.NewElem("info.log", ent))) + assert.False(t, fn(finder.NewElem("info.tmp", ent))) + + fn = finder.WithoutExt(".log") + assert.False(t, fn(finder.NewElem("info.log", ent))) + assert.True(t, fn(finder.NewElem("info.tmp", ent))) +} + +func TestRegexMatch(t *testing.T) { tests := []struct { filePath string pattern string - include bool match bool }{ - {"path/to/util.go", `\.go$`, true, true}, - {"path/to/util.go", `\.go$`, false, false}, - {"path/to/util.go", `\.py$`, true, false}, - {"path/to/util.go", `\.py$`, false, true}, + {"path/to/util.go", `\.go$`, true}, + {"path/to/util.go", `\.md$`, false}, + {"path/to/util.md", `\.md$`, true}, + {"path/to/util.md", `\.go$`, false}, } ent := &testutil.DirEnt{} for _, tt := range tests { - fn := finder.RegexFilterFunc(tt.pattern, tt.include) - assert.Eq(t, tt.match, fn(finder.NewElem(tt.filePath, ent))) + el := finder.NewElem(tt.filePath, ent) + fn := finder.WithRegexMatch(tt.pattern) + assert.Eq(t, tt.match, fn(el)) } + + t.Run("exclude", func(t *testing.T) { + for _, tt := range tests { + el := finder.NewElem(tt.filePath, ent) + fn := finder.WithoutRegexMatch(tt.pattern) + assert.Eq(t, !tt.match, fn(el)) + } + }) +} + +func TestDotDirFilter(t *testing.T) { + f := finder.EmptyFinder(). + ScanDir("./testdata") + + fmt.Println("no limits:") + fmt.Println(f) + + dirName := ".dotdir" + assert.Contains(t, f.FindPaths(), dirName) + + f = finder.EmptyFinder(). + ScanDir("./testdata"). + NoDotDir() + + fmt.Println("NoDotDir limits:") + fmt.Println(f.Config()) + assert.NotContains(t, f.FindPaths(), dirName) + + f = finder.NewEmpty(). + ScanDir("./testdata"). + WithDir(finder.DotDirFilter(false)) + + fmt.Println("DotDirFilter limits:") + fmt.Println(f) + assert.NotContains(t, f.FindPaths(), dirName) } diff --git a/fsutil/finder/finder.go b/fsutil/finder/finder.go index 19b725613..2bc1d602b 100644 --- a/fsutil/finder/finder.go +++ b/fsutil/finder/finder.go @@ -7,179 +7,46 @@ import ( "strings" ) -// FileFinder struct -type FileFinder struct { +// FileFinder type alias. +type FileFinder = Finder + +// Finder struct +type Finder struct { // config for finder c *Config // last error err error - // ch - founded file elem chan + // num - founded fs elem number + num int + // ch - founded fs elem chan ch chan Elem - // caches - cache found file elem. if config.CacheResult is true + // caches - cache found fs elem. if config.CacheResult is true caches []Elem } -// NewEmpty new empty FileFinder instance -func NewEmpty() *FileFinder { return New([]string{}) } - -// EmptyFinder new empty FileFinder instance -func EmptyFinder() *FileFinder { return NewEmpty() } - // New instance with source dir paths. -func New(dirs []string, fls ...Filter) *FileFinder { +func New(dirs []string) *Finder { c := NewConfig(dirs...) - c.Filters = fls - return NewWithConfig(c) } // NewFinder new instance with source dir paths. -func NewFinder(dirPaths ...string) *FileFinder { - return New(dirPaths) -} - -// NewWithConfig new instance with config. -func NewWithConfig(c *Config) *FileFinder { - return &FileFinder{ - c: c, - } -} - -// WithConfig on the finder -func (f *FileFinder) WithConfig(c *Config) *FileFinder { - f.c = c - return f -} - -// ConfigFn the finder -func (f *FileFinder) ConfigFn(fns ...func(c *Config)) *FileFinder { - if f.c == nil { - f.c = &Config{} - } +func NewFinder(dirPaths ...string) *Finder { return New(dirPaths) } - for _, fn := range fns { - fn(f.c) - } - return f -} - -// AddDirPath add source dir for find -func (f *FileFinder) AddDirPath(dirPaths ...string) *FileFinder { - f.c.DirPaths = append(f.c.DirPaths, dirPaths...) - return f -} +// NewEmpty new empty Finder instance +func NewEmpty() *Finder { return New([]string{}) } -// AddDir add source dir for find. alias of AddDirPath() -func (f *FileFinder) AddDir(dirPaths ...string) *FileFinder { - f.c.DirPaths = append(f.c.DirPaths, dirPaths...) - return f -} - -// CacheResult cache result for find result. -func (f *FileFinder) CacheResult(enable ...bool) *FileFinder { - if len(enable) > 0 { - f.c.CacheResult = enable[0] - } else { - f.c.CacheResult = true - } - return f -} - -// ExcludeDotDir exclude dot dir names. eg: ".idea" -func (f *FileFinder) ExcludeDotDir(exclude ...bool) *FileFinder { - if len(exclude) > 0 { - f.c.ExcludeDotDir = exclude[0] - } else { - f.c.ExcludeDotDir = true - } - return f -} - -// WithoutDotDir exclude dot dir names. alias of ExcludeDotDir(). -func (f *FileFinder) WithoutDotDir(exclude ...bool) *FileFinder { - return f.ExcludeDotDir(exclude...) -} +// EmptyFinder new empty Finder instance. alias of NewEmpty() +func EmptyFinder() *Finder { return NewEmpty() } -// NoDotDir exclude dot dir names. alias of ExcludeDotDir(). -func (f *FileFinder) NoDotDir(exclude ...bool) *FileFinder { - return f.ExcludeDotDir(exclude...) -} - -// ExcludeDotFile exclude dot dir names. eg: ".gitignore" -func (f *FileFinder) ExcludeDotFile(exclude ...bool) *FileFinder { - if len(exclude) > 0 { - f.c.ExcludeDotFile = exclude[0] - } else { - f.c.ExcludeDotFile = true - } - return f -} - -// WithoutDotFile exclude dot dir names. alias of ExcludeDotFile(). -func (f *FileFinder) WithoutDotFile(exclude ...bool) *FileFinder { - return f.ExcludeDotFile(exclude...) -} - -// NoDotFile exclude dot dir names. alias of ExcludeDotFile(). -func (f *FileFinder) NoDotFile(exclude ...bool) *FileFinder { - return f.ExcludeDotFile(exclude...) -} - -// ExcludeDir exclude dir names. -func (f *FileFinder) ExcludeDir(dirs ...string) *FileFinder { - f.c.ExcludeDirs = append(f.c.ExcludeDirs, dirs...) - return f -} - -// ExcludeName exclude file names. -func (f *FileFinder) ExcludeName(files ...string) *FileFinder { - f.c.ExcludeNames = append(f.c.ExcludeNames, files...) - return f -} - -// AddFilter for filter filepath or dir path -func (f *FileFinder) AddFilter(filters ...Filter) *FileFinder { - return f.WithFilter(filters...) -} - -// WithFilter add filter func for filtering filepath or dir path -func (f *FileFinder) WithFilter(filters ...Filter) *FileFinder { - f.c.Filters = append(f.c.Filters, filters...) - return f -} - -// AddFileFilter for filter filepath -func (f *FileFinder) AddFileFilter(filters ...Filter) *FileFinder { - return f.WithFileFilter(filters...) -} - -// WithFileFilter for filter func for filtering filepath -func (f *FileFinder) WithFileFilter(filters ...Filter) *FileFinder { - f.c.FileFilters = append(f.c.FileFilters, filters...) - return f -} - -// AddDirFilter for filter file contents -func (f *FileFinder) AddDirFilter(fls ...Filter) *FileFinder { - return f.WithDirFilter(fls...) -} - -// WithDirFilter for filter func for filtering file contents -func (f *FileFinder) WithDirFilter(filters ...Filter) *FileFinder { - f.c.DirFilters = append(f.c.DirFilters, filters...) - return f -} - -// AddBodyFilter for filter file contents -func (f *FileFinder) AddBodyFilter(fls ...BodyFilter) *FileFinder { - return f.WithBodyFilter(fls...) +// NewWithConfig new instance with config. +func NewWithConfig(c *Config) *Finder { + return &Finder{c: c} } -// WithBodyFilter for filter func for filtering file contents -func (f *FileFinder) WithBodyFilter(fls ...BodyFilter) *FileFinder { - f.c.BodyFilters = append(f.c.BodyFilters, fls...) - return f -} +// +// --------- do finding --------- +// // Find files in given dir paths. will return a channel, you can use it to get the result. // @@ -189,57 +56,55 @@ func (f *FileFinder) WithBodyFilter(fls ...BodyFilter) *FileFinder { // for el := range f { // fmt.Println(el.Path()) // } -func (f *FileFinder) Find() <-chan Elem { +func (f *Finder) Find() <-chan Elem { f.find() return f.ch } +// Elems find and return founded file Elem. alias of Find() +func (f *Finder) Elems() <-chan Elem { return f.Find() } + // Results find and return founded file Elem. alias of Find() -// -// Usage: -// -// rs := NewFinder("/path/to/dir").Results() -// for el := range rs { -// fmt.Println(el.Path()) -// } -func (f *FileFinder) Results() <-chan Elem { - f.find() - return f.ch -} +func (f *Finder) Results() <-chan Elem { return f.Find() } -// FindPaths find and return founded file paths. -func (f *FileFinder) FindPaths() []string { - f.find() +// FindNames find and return founded file/dir names. +func (f *Finder) FindNames() []string { + paths := make([]string, 0, 8*len(f.c.ScanDirs)) + for el := range f.Find() { + paths = append(paths, el.Name()) + } + return paths +} - paths := make([]string, 0, 8*len(f.c.DirPaths)) - for el := range f.ch { +// FindPaths find and return founded file/dir paths. +func (f *Finder) FindPaths() []string { + paths := make([]string, 0, 8*len(f.c.ScanDirs)) + for el := range f.Find() { paths = append(paths, el.Path()) } return paths } -// Each file or dir Elem. -func (f *FileFinder) Each(fn func(el Elem)) { - f.EachElem(fn) -} +// Each founded file or dir Elem. +func (f *Finder) Each(fn func(el Elem)) { f.EachElem(fn) } -// EachElem file or dir Elem. -func (f *FileFinder) EachElem(fn func(el Elem)) { +// EachElem founded file or dir Elem. +func (f *Finder) EachElem(fn func(el Elem)) { f.find() for el := range f.ch { fn(el) } } -// EachPath file paths. -func (f *FileFinder) EachPath(fn func(filePath string)) { +// EachPath founded file paths. +func (f *Finder) EachPath(fn func(filePath string)) { f.EachElem(func(el Elem) { fn(el.Path()) }) } // EachFile each file os.File -func (f *FileFinder) EachFile(fn func(file *os.File)) { +func (f *Finder) EachFile(fn func(file *os.File)) { f.EachElem(func(el Elem) { file, err := os.Open(el.Path()) if err == nil { @@ -251,7 +116,7 @@ func (f *FileFinder) EachFile(fn func(file *os.File)) { } // EachStat each file os.FileInfo -func (f *FileFinder) EachStat(fn func(fi os.FileInfo, filePath string)) { +func (f *Finder) EachStat(fn func(fi os.FileInfo, filePath string)) { f.EachElem(func(el Elem) { fi, err := el.Info() if err == nil { @@ -263,7 +128,7 @@ func (f *FileFinder) EachStat(fn func(fi os.FileInfo, filePath string)) { } // EachContents handle each found file contents -func (f *FileFinder) EachContents(fn func(contents, filePath string)) { +func (f *Finder) EachContents(fn func(contents, filePath string)) { f.EachElem(func(el Elem) { bs, err := os.ReadFile(el.Path()) if err == nil { @@ -274,14 +139,25 @@ func (f *FileFinder) EachContents(fn func(contents, filePath string)) { }) } -// do finding -func (f *FileFinder) find() { +// prepare for find. +func (f *Finder) prepare() { f.err = nil f.ch = make(chan Elem, 8) + if f.CacheNum() == 0 { + f.num = 0 + } + if f.c == nil { f.c = NewConfig() + } else { + f.c.Init() } +} + +// do finding +func (f *Finder) find() { + f.prepare() go func() { defer close(f.ch) @@ -295,93 +171,100 @@ func (f *FileFinder) find() { } // do finding - for _, dirPath := range f.c.DirPaths { + var err error + for _, dirPath := range f.c.ScanDirs { + if f.c.UseAbsPath { + dirPath, err = filepath.Abs(dirPath) + if err != nil { + f.err = err + continue + } + } + + f.c.depth = 0 f.findDir(dirPath, f.c) } }() } // code refer filepath.glob() -func (f *FileFinder) findDir(dirPath string, c *Config) { - dfi, err := os.Stat(dirPath) - if err != nil { - return // ignore I/O error - } - if !dfi.IsDir() { - return // ignore I/O error - } - +func (f *Finder) findDir(dirPath string, c *Config) { des, err := os.ReadDir(dirPath) if err != nil { return // ignore I/O error } - c.curDepth++ + var ok bool + c.depth++ for _, ent := range des { - baseName := ent.Name() - fullPath := filepath.Join(dirPath, baseName) + name := ent.Name() + isDir := ent.IsDir() + if name[0] == '.' { + if isDir { + if c.ExcludeDotDir { + continue + } + } else if c.ExcludeDotFile { + continue + } + } - ok := false + fullPath := filepath.Join(dirPath, name) el := NewElem(fullPath, ent) // apply generic filters - for _, filter := range c.Filters { - if filter.Apply(el) { // 有一个满足即可 - ok = true - break - } - } - if !ok { + if !applyExFilters(el, c.ExFilters) { continue } - // --- dir - if ent.IsDir() { - if c.ExcludeDotDir && baseName[0] == '.' { + // --- dir: apply dir filters + if isDir { + if !applyExFilters(el, c.DirExFilters) { continue } - // apply dir filters - ok = false - for _, df := range c.DirFilters { - if df.Apply(el) { // 有一个满足即可 - ok = true - break + if len(c.Filters) > 0 { + ok = applyFilters(el, c.Filters) + if !ok && len(c.DirFilters) > 0 { + ok = applyFilters(el, c.DirFilters) } + } else { + ok = applyFilters(el, c.DirFilters) } - if ok { - if c.FindFlags&FlagDir > 0 { - if c.CacheResult { - f.caches = append(f.caches, el) - } - f.ch <- el + if ok && c.FindFlags&FlagDir > 0 { + if c.CacheResult { + f.caches = append(f.caches, el) } + f.num++ + f.ch <- el + } - // find in sub dir. - if c.curDepth < c.MaxDepth { - f.findDir(fullPath, c) - } + // find in sub dir. + if c.MaxDepth == 0 || c.depth < c.MaxDepth { + f.findDir(fullPath, c) + c.depth-- // restore depth } continue } - if c.FindFlags&FlagDir > 0 { + // --- type: file + if c.FindFlags&FlagFile == 0 { continue } - // --- type: file - if c.ExcludeDotFile && baseName[0] == '.' { + // apply file filters + if !applyExFilters(el, c.FileExFilters) { continue } - // use custom filter functions - ok = false - for _, ff := range c.FileFilters { - if ff.Apply(el) { // 有一个满足即可 - ok = true - break + if len(c.Filters) > 0 { + ok = applyFilters(el, c.Filters) + if !ok && len(c.FileFilters) > 0 { + ok = applyFilters(el, c.FileFilters) } + } else { + ok = applyFilters(el, c.FileFilters) } // write to consumer @@ -389,41 +272,75 @@ func (f *FileFinder) findDir(dirPath string, c *Config) { if c.CacheResult { f.caches = append(f.caches, el) } + f.num++ f.ch <- el } } } +func applyFilters(el Elem, fls []Filter) bool { + for _, f := range fls { + if f.Apply(el) { + return true + } + } + return len(fls) == 0 +} + +func applyExFilters(el Elem, fls []Filter) bool { + for _, f := range fls { + if f.Apply(el) { + return false + } + } + return true +} + // Reset filters config setting and results info. -func (f *FileFinder) Reset() { - c := NewConfig(f.c.DirPaths...) +func (f *Finder) Reset() { + c := NewConfig(f.c.ScanDirs...) c.ExcludeDotDir = f.c.ExcludeDotDir c.FindFlags = f.c.FindFlags c.MaxDepth = f.c.MaxDepth - c.curDepth = 0 f.c = c + f.ResetResult() +} + +// ResetResult reset result info. +func (f *Finder) ResetResult() { + f.num = 0 f.err = nil f.ch = make(chan Elem, 8) f.caches = []Elem{} } +// Num get found elem num. only valid after finding. +func (f *Finder) Num() int { + return f.num +} + // Err get last error -func (f *FileFinder) Err() error { +func (f *Finder) Err() error { return f.err } +// Caches get cached results. only valid after finding. +func (f *Finder) Caches() []Elem { + return f.caches +} + // CacheNum get -func (f *FileFinder) CacheNum() int { +func (f *Finder) CacheNum() int { return len(f.caches) } // Config get -func (f *FileFinder) Config() Config { +func (f *Finder) Config() Config { return *f.c } // String all dir paths -func (f *FileFinder) String() string { - return strings.Join(f.c.DirPaths, ",") +func (f *Finder) String() string { + return strings.Join(f.c.ScanDirs, ";") } diff --git a/fsutil/finder/finder_test.go b/fsutil/finder/finder_test.go index 252020160..3c9fe667a 100644 --- a/fsutil/finder/finder_test.go +++ b/fsutil/finder/finder_test.go @@ -6,108 +6,146 @@ import ( "testing" "github.com/gookit/goutil/fsutil/finder" - "github.com/gookit/goutil/testutil" "github.com/gookit/goutil/testutil/assert" ) -func TestEmptyFinder(t *testing.T) { - f := finder.EmptyFinder() - - f. - AddDir("./testdata"). +func TestFinder_findFile(t *testing.T) { + f := finder.EmptyFinder(). + ScanDir("./testdata"). NoDotFile(). - CacheResult(). - // NoDotDir(). - EachPath(func(filePath string) { + NoDotDir(). + WithoutExt(".jpg"). + CacheResult() + + assert.Nil(t, f.Err()) + assert.NotEmpty(t, f.String()) + assert.Eq(t, 0, f.CacheNum()) + + // find paths + assert.NotEmpty(t, f.FindPaths()) + assert.Gt(t, f.CacheNum(), 0) + assert.NotEmpty(t, f.Caches()) + + f.Each(func(elem finder.Elem) { + fmt.Println(elem) + }) + + t.Run("each elem", func(t *testing.T) { + f.EachElem(func(elem finder.Elem) { + fmt.Println(elem) + }) + }) + + t.Run("each file", func(t *testing.T) { + f.EachFile(func(file *os.File) { + fmt.Println(file.Name()) + }) + }) + + t.Run("each path", func(t *testing.T) { + f.EachPath(func(filePath string) { fmt.Println(filePath) }) + }) - assert.NotEmpty(t, f.FindPaths()) + t.Run("each stat", func(t *testing.T) { + f.EachStat(func(fi os.FileInfo, filePath string) { + fmt.Println(filePath, "=>", fi.ModTime()) + }) + }) - f.Reset() - assert.Empty(t, f.FindPaths()) + t.Run("reset", func(t *testing.T) { + f.Reset() + assert.Empty(t, f.Caches()) + assert.NotEmpty(t, f.FindPaths()) + + f.EachElem(func(elem finder.Elem) { + fmt.Println(elem) + }) + }) } -func TestNewFinder(t *testing.T) { - finder.NewFinder("./testdata"). - NoDotDir(). - EachStat(func(fi os.FileInfo, filePath string) { - fmt.Println(filePath, "=>", fi.ModTime()) +func TestFinder_OnlyFindDir(t *testing.T) { + ff := finder.NewFinder("./../../"). + OnlyFindDir(). + UseAbsPath(). + WithoutDotDir(). + WithDirName("testdata") + + ff.EachPath(func(filePath string) { + fmt.Println(filePath) + }) + assert.Gt(t, ff.Num(), 0) + assert.Eq(t, 0, ff.CacheNum()) + + t.Run("each elem", func(t *testing.T) { + ff.Each(func(elem finder.Elem) { + fmt.Println(elem) }) + }) + + ff.ResetResult() + assert.Eq(t, 0, ff.Num()) + assert.Eq(t, 0, ff.CacheNum()) + + t.Run("max depth", func(t *testing.T) { + ff.WithMaxDepth(2) + ff.EachPath(func(filePath string) { + fmt.Println(filePath) + }) + assert.Gt(t, ff.Num(), 0) + }) } -func TestDotFileFilterFunc(t *testing.T) { +func TestFileFinder_NoDotFile(t *testing.T) { f := finder.NewEmpty(). - AddDir("./testdata") + ScanDir("./testdata") assert.NotEmpty(t, f.String()) - fmt.Println("no limits:") - fmt.Println(f) - - fileName := ".env" + fileName := "testdata/.env" assert.Contains(t, f.FindPaths(), fileName) f = finder.EmptyFinder(). - AddDir("./testdata"). + ScanDir("./testdata"). NoDotFile() - - fmt.Println("NoDotFile limits:") - fmt.Println(f) assert.NotContains(t, f.FindPaths(), fileName) - f = finder.EmptyFinder(). - AddDir("./testdata"). - WithFilter(finder.DotFileFilter(false)) + t.Run("WithoutDotFile", func(t *testing.T) { + f = finder.EmptyFinder(). + ScanDir("./testdata"). + Not(finder.WithoutDotFile()) - fmt.Println("DotFileFilter limits:") - fmt.Println(f) - assert.NotContains(t, f.FindPaths(), fileName) + assert.NotContains(t, f.FindPaths(), fileName) + }) } -func TestDotDirFilterFunc(t *testing.T) { - f := finder.EmptyFinder(). - AddDir("./testdata") +func TestFileFinder_IncludeName(t *testing.T) { + f := finder.NewFinder(".").IncludeName("elem.go").WithNames([]string{"not-exist.file"}) - fmt.Println("no limits:") - fmt.Println(f) + names := f.FindNames() + assert.Len(t, names, 1) + assert.Contains(t, names, "elem.go") + assert.NotContains(t, names, "not-exist.file") - dirName := ".dotdir" - assert.Contains(t, f.FindPaths(), dirName) - - f = finder.EmptyFinder(). - AddDir("./testdata"). - NoDotDir() - - fmt.Println("NoDotDir limits:") - fmt.Println(f.Config()) - assert.NotContains(t, f.FindPaths(), dirName) - - f = finder.NewEmpty(). - AddDir("./testdata"). - WithDirFilter(finder.DotDirFilter(false)) - - fmt.Println("DotDirFilter limits:") - fmt.Println(f) - assert.NotContains(t, f.FindPaths(), dirName) -} - -var testFiles = []string{ - "info.log", - "error.log", - "cache.tmp", - "/some/path/to/info.log", - "/some/path1/to/cache.tmp", + f.Reset() + t.Run("name in subdir", func(t *testing.T) { + f.WithFileName("test.jpg") + names = f.FindNames() + assert.Len(t, names, 1) + assert.Contains(t, names, "test.jpg") + }) } -func TestExtFilterFunc(t *testing.T) { - ent := &testutil.DirEnt{} - - fn := finder.ExtFilter(true, ".log") - assert.True(t, fn(finder.NewElem("info.log", ent))) - assert.False(t, fn(finder.NewElem("info.tmp", ent))) - - fn = finder.ExtFilter(false, ".log") - assert.False(t, fn(finder.NewElem("info.log", ent))) - assert.True(t, fn(finder.NewElem("info.tmp", ent))) - +func TestFileFinder_ExcludeName(t *testing.T) { + f := finder.NewEmpty(). + AddScanDir("."). + WithMaxDepth(1). + ExcludeName("elem.go"). + WithoutNames([]string{"config.go"}) + // f.Exclude(finder.WithoutSuffix("_test.go"), finder.WithoutExt(".md")) + + names := f.FindNames() + fmt.Println(names) + assert.Contains(t, names, "filter.go") + assert.NotContains(t, names, "elem.go") }