From 544953b3558cd37c3119afabf5cfc4955a36241e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 13 Sep 2023 12:09:46 +0300 Subject: [PATCH 01/23] refactor: remove parallel walk Signed-off-by: knqyf263 --- go.mod | 1 - go.sum | 2 - pkg/fanal/artifact/local/fs.go | 2 +- pkg/fanal/artifact/local/fs_test.go | 2 +- pkg/fanal/walker/fs.go | 68 ++++++++--------------------- pkg/fanal/walker/fs_test.go | 2 +- 6 files changed, 22 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 32285a44efba..2a69c45f004b 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,6 @@ require ( github.com/owenrumney/go-sarif/v2 v2.2.0 github.com/package-url/packageurl-go v0.1.2-0.20230812223828-f8bb31c1f10b github.com/samber/lo v1.38.1 - github.com/saracen/walker v0.1.3 github.com/secure-systems-lab/go-securesystemslib v0.7.0 github.com/sigstore/rekor v1.2.1 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 3c4aef09b426..0154508cc0a0 100644 --- a/go.sum +++ b/go.sum @@ -1558,8 +1558,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g= -github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index d7c4a0573ff0..0bc5788390e6 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -56,7 +56,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a rootPath: filepath.ToSlash(filepath.Clean(rootPath)), cache: c, walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs), - opt.Slow, opt.WalkOption.ErrorCallback), + opt.WalkOption.ErrorCallback), analyzer: a, handlerManager: handlerManager, diff --git a/pkg/fanal/artifact/local/fs_test.go b/pkg/fanal/artifact/local/fs_test.go index d18048f8e97b..f0e32a3166e2 100644 --- a/pkg/fanal/artifact/local/fs_test.go +++ b/pkg/fanal/artifact/local/fs_test.go @@ -158,7 +158,7 @@ func TestArtifact_Inspect(t *testing.T) { fields: fields{ dir: "./testdata/unknown", }, - wantErr: "walk error", + wantErr: "walk dir error", }, { name: "happy path with single file", diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index 74be944e73ac..dabf100ce453 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -5,11 +5,9 @@ import ( "os" "path/filepath" - swalker "github.com/saracen/walker" "golang.org/x/xerrors" dio "github.com/aquasecurity/go-dep-parser/pkg/io" - "github.com/aquasecurity/trivy/pkg/log" ) type ErrorCallback func(pathname string, err error) error @@ -19,7 +17,7 @@ type FS struct { errCallback ErrorCallback } -func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS { +func NewFS(skipFiles, skipDirs []string, errCallback ErrorCallback) FS { if errCallback == nil { errCallback = func(pathname string, err error) error { // ignore permission errors @@ -32,78 +30,50 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F } return FS{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs, false), errCallback: errCallback, } } -// Walk walks the file tree rooted at root, calling WalkFunc for each file or +// Walk walks the file tree rooted at root, calling WalkDirFunc for each file or // directory in the tree, including root, but a directory to be ignored will be skipped. func (w FS) Walk(root string, fn WalkFunc) error { - // walk function called for every path found - walkFn := func(pathname string, fi os.FileInfo) error { - pathname = filepath.Clean(pathname) + err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error { + if err != nil { + return w.errCallback(filePath, err) + } + + filePath = filepath.Clean(filePath) // For exported rootfs (e.g. images/alpine/etc/alpine-release) - relPath, err := filepath.Rel(root, pathname) + relPath, err := filepath.Rel(root, filePath) if err != nil { return xerrors.Errorf("filepath rel (%s): %w", relPath, err) } relPath = filepath.ToSlash(relPath) - if fi.IsDir() { + info, err := d.Info() + if err != nil { + return xerrors.Errorf("file info error: %w", err) + } + + if info.IsDir() { if w.shouldSkipDir(relPath) { return filepath.SkipDir } return nil - } else if !fi.Mode().IsRegular() { + } else if !info.Mode().IsRegular() { return nil } else if w.shouldSkipFile(relPath) { return nil } - if err = fn(relPath, fi, w.fileOpener(pathname)); err != nil { + if err = fn(relPath, info, w.fileOpener(filePath)); err != nil { return xerrors.Errorf("failed to analyze file: %w", err) } return nil - } - - if w.slow { - // In series: fast, with higher CPU/memory - return w.walkSlow(root, walkFn) - } - - // In parallel: slow, with lower CPU/memory - return w.walkFast(root, walkFn) -} - -type fastWalkFunc func(pathname string, fi os.FileInfo) error - -func (w FS) walkFast(root string, walkFn fastWalkFunc) error { - // error function called for every error encountered - errorCallbackOption := swalker.WithErrorCallback(w.errCallback) - - // Multiple goroutines stat the filesystem concurrently. The provided - // walkFn must be safe for concurrent use. - log.Logger.Debugf("Walk the file tree rooted at '%s' in parallel", root) - if err := swalker.Walk(root, walkFn, errorCallbackOption); err != nil { - return xerrors.Errorf("walk error: %w", err) - } - return nil -} - -func (w FS) walkSlow(root string, walkFn fastWalkFunc) error { - log.Logger.Debugf("Walk the file tree rooted at '%s' in series", root) - err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return w.errCallback(path, err) - } - info, err := d.Info() - if err != nil { - return xerrors.Errorf("file info error: %w", err) - } - return walkFn(path, info) }) + if err != nil { return xerrors.Errorf("walk dir error: %w", err) } diff --git a/pkg/fanal/walker/fs_test.go b/pkg/fanal/walker/fs_test.go index 038e15ccc7d9..5576d3981d88 100644 --- a/pkg/fanal/walker/fs_test.go +++ b/pkg/fanal/walker/fs_test.go @@ -93,7 +93,7 @@ func TestDir_Walk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback) + w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.errCallback) err := w.Walk(tt.rootDir, tt.analyzeFn) if tt.wantErr != "" { From 6fd56ee9fa7a5d3e9e2325d737e249557bb630d9 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 20 Oct 2023 00:40:13 +0900 Subject: [PATCH 02/23] fix(walk): call error callback on all errors Signed-off-by: knqyf263 --- pkg/fanal/walker/fs.go | 26 ++++++++++++++++++++------ pkg/fanal/walker/fs_test.go | 21 ++++++++++++++++++--- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index dabf100ce453..63fd66def3ce 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -1,6 +1,7 @@ package walker import ( + "errors" "io/fs" "os" "path/filepath" @@ -20,8 +21,12 @@ type FS struct { func NewFS(skipFiles, skipDirs []string, errCallback ErrorCallback) FS { if errCallback == nil { errCallback = func(pathname string, err error) error { + switch { + // Unwrap fs.SkipDir error + case errors.Is(err, fs.SkipDir): + return fs.SkipDir // ignore permission errors - if os.IsPermission(err) { + case os.IsPermission(err): return nil } // halt traversal on any other error @@ -38,7 +43,21 @@ func NewFS(skipFiles, skipDirs []string, errCallback ErrorCallback) FS { // Walk walks the file tree rooted at root, calling WalkDirFunc for each file or // directory in the tree, including root, but a directory to be ignored will be skipped. func (w FS) Walk(root string, fn WalkFunc) error { + walkDir := w.walkDirFunc(root, fn) err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error { + if walkErr := walkDir(filePath, d, err); walkErr != nil { + return w.errCallback(filePath, walkErr) + } + return nil + }) + if err != nil { + return xerrors.Errorf("walk dir error: %w", err) + } + return nil +} + +func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc { + return func(filePath string, d fs.DirEntry, err error) error { if err != nil { return w.errCallback(filePath, err) } @@ -72,12 +91,7 @@ func (w FS) Walk(root string, fn WalkFunc) error { return xerrors.Errorf("failed to analyze file: %w", err) } return nil - }) - - if err != nil { - return xerrors.Errorf("walk dir error: %w", err) } - return nil } // fileOpener returns a function opening a file. diff --git a/pkg/fanal/walker/fs_test.go b/pkg/fanal/walker/fs_test.go index 5576d3981d88..f8e02557c6cf 100644 --- a/pkg/fanal/walker/fs_test.go +++ b/pkg/fanal/walker/fs_test.go @@ -3,6 +3,7 @@ package walker_test import ( "errors" "io" + "io/fs" "os" "strings" "testing" @@ -77,15 +78,29 @@ func TestDir_Walk(t *testing.T) { return nil }, }, - analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { - // Ignore errors + analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { return nil }, }, + { + name: "ignore analysis errors", + rootDir: "testdata/fs", + fields: fields{ + errCallback: func(pathname string, err error) error { + if errors.Is(err, fs.ErrClosed) { + return nil + } + return err + }, + }, + analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { + return fs.ErrClosed + }, + }, { name: "sad path", rootDir: "testdata/fs", - analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { return errors.New("error") }, wantErr: "failed to analyze file", From 362871a52011e97761d3cd1c838948a0359f9a71 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 2 Nov 2023 10:34:40 +0900 Subject: [PATCH 03/23] feat: add delay per file Signed-off-by: knqyf263 --- pkg/fanal/walker/fs.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index a658a86434c4..b78ff32775c2 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -5,12 +5,15 @@ import ( "io/fs" "os" "path/filepath" + "time" "golang.org/x/xerrors" dio "github.com/aquasecurity/go-dep-parser/pkg/io" ) +const walkDelay = 10 * time.Millisecond + type ErrorCallback func(pathname string, err error) error type FS struct { @@ -18,7 +21,7 @@ type FS struct { errCallback ErrorCallback } -func NewFS(skipFiles, skipDirs []string, errCallback ErrorCallback) FS { +func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS { if errCallback == nil { errCallback = func(pathname string, err error) error { switch { @@ -35,7 +38,7 @@ func NewFS(skipFiles, skipDirs []string, errCallback ErrorCallback) FS { } return FS{ - walker: newWalker(skipFiles, skipDirs, false), + walker: newWalker(skipFiles, skipDirs, slow), errCallback: errCallback, } } @@ -62,6 +65,10 @@ func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc { return w.errCallback(filePath, err) } + if w.walker.slow { + time.Sleep(walkDelay) + } + filePath = filepath.Clean(filePath) // For exported rootfs (e.g. images/alpine/etc/alpine-release) From 4972f06962045c8ad89714cb2434abf3750e8e5d Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Mon, 6 Nov 2023 12:57:12 +0900 Subject: [PATCH 04/23] feat: configure delay seconds Signed-off-by: knqyf263 --- pkg/fanal/artifact/artifact.go | 8 +------- pkg/fanal/artifact/local/fs.go | 2 +- pkg/fanal/walker/fs.go | 27 ++++++++++++++++----------- pkg/fanal/walker/fs_test.go | 26 +++++++++++++++----------- 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/pkg/fanal/artifact/artifact.go b/pkg/fanal/artifact/artifact.go index 2028b601c744..c2854a81b4e2 100644 --- a/pkg/fanal/artifact/artifact.go +++ b/pkg/fanal/artifact/artifact.go @@ -41,13 +41,7 @@ type Option struct { LicenseScannerOption analyzer.LicenseScannerOption // File walk - WalkOption WalkOption -} - -// WalkOption is a struct that allows users to define a custom walking behavior. -// This option is only available when using Trivy as an imported library and not through CLI flags. -type WalkOption struct { - ErrorCallback walker.ErrorCallback + WalkOption walker.Option } func (o *Option) Sort() { diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index 56b212786332..8b024b681d09 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -56,7 +56,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a rootPath: filepath.ToSlash(filepath.Clean(rootPath)), cache: c, walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs), - opt.WalkOption.ErrorCallback), + opt.Slow, opt.WalkOption), analyzer: a, handlerManager: handlerManager, diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index b78ff32775c2..9e2563642db5 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -12,18 +12,23 @@ import ( dio "github.com/aquasecurity/go-dep-parser/pkg/io" ) -const walkDelay = 10 * time.Millisecond - type ErrorCallback func(pathname string, err error) error +// Option is a struct that allows users to define a custom walking behavior. +// This option is only available when using Trivy as an imported library and not through CLI flags. +type Option struct { + ErrorCallback ErrorCallback + Delay time.Duration +} + type FS struct { walker - errCallback ErrorCallback + option Option } -func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS { - if errCallback == nil { - errCallback = func(pathname string, err error) error { +func NewFS(skipFiles, skipDirs []string, slow bool, opt Option) FS { + if opt.ErrorCallback == nil { + opt.ErrorCallback = func(pathname string, err error) error { switch { // Unwrap fs.SkipDir error case errors.Is(err, fs.SkipDir): @@ -38,8 +43,8 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F } return FS{ - walker: newWalker(skipFiles, skipDirs, slow), - errCallback: errCallback, + walker: newWalker(skipFiles, skipDirs, slow), + option: opt, } } @@ -49,7 +54,7 @@ func (w FS) Walk(root string, fn WalkFunc) error { walkDir := w.walkDirFunc(root, fn) err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error { if walkErr := walkDir(filePath, d, err); walkErr != nil { - return w.errCallback(filePath, walkErr) + return w.option.ErrorCallback(filePath, walkErr) } return nil }) @@ -62,11 +67,11 @@ func (w FS) Walk(root string, fn WalkFunc) error { func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc { return func(filePath string, d fs.DirEntry, err error) error { if err != nil { - return w.errCallback(filePath, err) + return err } if w.walker.slow { - time.Sleep(walkDelay) + time.Sleep(w.option.Delay) } filePath = filepath.Clean(filePath) diff --git a/pkg/fanal/walker/fs_test.go b/pkg/fanal/walker/fs_test.go index f8e02557c6cf..3de96b8e0148 100644 --- a/pkg/fanal/walker/fs_test.go +++ b/pkg/fanal/walker/fs_test.go @@ -17,9 +17,9 @@ import ( func TestDir_Walk(t *testing.T) { type fields struct { - skipFiles []string - skipDirs []string - errCallback walker.ErrorCallback + skipFiles []string + skipDirs []string + option walker.Option } tests := []struct { name string @@ -74,8 +74,10 @@ func TestDir_Walk(t *testing.T) { name: "ignore all errors", rootDir: "testdata/fs/nosuch", fields: fields{ - errCallback: func(pathname string, err error) error { - return nil + option: walker.Option{ + ErrorCallback: func(pathname string, err error) error { + return nil + }, }, }, analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { @@ -86,11 +88,13 @@ func TestDir_Walk(t *testing.T) { name: "ignore analysis errors", rootDir: "testdata/fs", fields: fields{ - errCallback: func(pathname string, err error) error { - if errors.Is(err, fs.ErrClosed) { - return nil - } - return err + option: walker.Option{ + ErrorCallback: func(pathname string, err error) error { + if errors.Is(err, fs.ErrClosed) { + return nil + } + return err + }, }, }, analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { @@ -108,7 +112,7 @@ func TestDir_Walk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.errCallback) + w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, false, tt.fields.option) err := w.Walk(tt.rootDir, tt.analyzeFn) if tt.wantErr != "" { From 7fc49c2916e40961aa4e6d54fe9c3e595fe51970 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 7 Nov 2023 12:36:58 +0900 Subject: [PATCH 05/23] feat: replace slow with parallel Signed-off-by: knqyf263 --- pkg/commands/artifact/run.go | 2 +- pkg/custom/custom.go | 19 +++++++++++ pkg/fanal/analyzer/analyzer.go | 2 +- pkg/fanal/analyzer/language/java/jar/jar.go | 8 ++--- .../analyzer/language/java/jar/jar_test.go | 2 +- pkg/fanal/artifact/artifact.go | 34 ++++++++++++++++--- pkg/fanal/artifact/image/image.go | 29 +++++----------- pkg/fanal/artifact/image/image_test.go | 5 +-- pkg/fanal/artifact/local/fs.go | 19 ++++------- pkg/fanal/artifact/sbom/sbom.go | 1 + pkg/fanal/artifact/vm/vm.go | 17 ++++------ pkg/fanal/artifact/vm/vm_test.go | 2 +- pkg/fanal/walker/fs.go | 20 +++-------- pkg/fanal/walker/fs_test.go | 9 ++--- pkg/fanal/walker/tar.go | 10 ++---- pkg/fanal/walker/tar_test.go | 2 +- pkg/fanal/walker/vm.go | 10 ++---- pkg/fanal/walker/walk.go | 7 ++-- pkg/fanal/walker/walk_test.go | 4 +-- pkg/flag/kubernetes_flags.go | 20 ----------- pkg/flag/scan_flags.go | 22 ++++++++++-- pkg/parallel/walk.go | 8 ++--- pkg/semaphore/semaphore.go | 30 ---------------- 23 files changed, 123 insertions(+), 159 deletions(-) create mode 100644 pkg/custom/custom.go delete mode 100644 pkg/semaphore/semaphore.go diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index 5baa61787a57..274d2f0a30ec 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -629,6 +629,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi SkipDirs: opts.SkipDirs, FilePatterns: opts.FilePatterns, Offline: opts.OfflineScan, + Parallel: opts.Parallel, NoProgress: opts.NoProgress || opts.Quiet, Insecure: opts.Insecure, RepoBranch: opts.RepoBranch, @@ -637,7 +638,6 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi SBOMSources: opts.SBOMSources, RekorURL: opts.RekorURL, //Platform: opts.Platform, - Slow: opts.Slow, AWSRegion: opts.Region, AWSEndpoint: opts.Endpoint, FileChecksum: fileChecksum, diff --git a/pkg/custom/custom.go b/pkg/custom/custom.go new file mode 100644 index 000000000000..9df8489dfdd3 --- /dev/null +++ b/pkg/custom/custom.go @@ -0,0 +1,19 @@ +package custom + +import ( + "time" +) + +type ErrorCallback func(pathname string, err error) error + +// Option is a struct that allows defining a custom behavior. +// This option is only available when Trivy is used as an imported library and not through CLI flags. +type Option struct { + // Delay is the amount of time to wait between each file walk. + // Default: 0 sec + Delay time.Duration + + // ErrorCallback is a function that allows users to define a custom error handling behavior while walking the filesystem. + // If not defined, the default behavior is to halt traversal on any error. + ErrorCallback ErrorCallback +} diff --git a/pkg/fanal/analyzer/analyzer.go b/pkg/fanal/analyzer/analyzer.go index ccb578be62fa..b9f1620e07c6 100644 --- a/pkg/fanal/analyzer/analyzer.go +++ b/pkg/fanal/analyzer/analyzer.go @@ -41,7 +41,7 @@ var ( // AnalyzerOptions is used to initialize analyzers type AnalyzerOptions struct { Group Group - Slow bool + Parallel int FilePatterns []string DisabledAnalyzers []Type MisconfScannerOption misconf.ScannerOption diff --git a/pkg/fanal/analyzer/language/java/jar/jar.go b/pkg/fanal/analyzer/language/java/jar/jar.go index 8e4ed4820f18..c787acb68390 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar.go +++ b/pkg/fanal/analyzer/language/java/jar/jar.go @@ -34,13 +34,13 @@ var requiredExtensions = []string{ // javaLibraryAnalyzer analyzes jar/war/ear/par files type javaLibraryAnalyzer struct { - client *javadb.DB - slow bool + client *javadb.DB + parallel int } func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { return &javaLibraryAnalyzer{ - slow: options.Slow, + parallel: options.Parallel, }, nil } @@ -75,7 +75,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po return nil } - if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil { + if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil { return nil, xerrors.Errorf("walk dir error: %w", err) } diff --git a/pkg/fanal/analyzer/language/java/jar/jar_test.go b/pkg/fanal/analyzer/language/java/jar/jar_test.go index f198c7beb041..fc59f91b17cc 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar_test.go +++ b/pkg/fanal/analyzer/language/java/jar/jar_test.go @@ -132,7 +132,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { // init java-trivy-db with skip update javadb.Init("testdata", defaultJavaDBRepository, true, false, types.RegistryOptions{Insecure: false}) - a := javaLibraryAnalyzer{slow: true} + a := javaLibraryAnalyzer{parallel: 1} ctx := context.Background() mfs := mapfs.New() diff --git a/pkg/fanal/artifact/artifact.go b/pkg/fanal/artifact/artifact.go index c2854a81b4e2..de52045d4507 100644 --- a/pkg/fanal/artifact/artifact.go +++ b/pkg/fanal/artifact/artifact.go @@ -4,9 +4,9 @@ import ( "context" "sort" + "github.com/aquasecurity/trivy/pkg/custom" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/types" - "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/misconf" ) @@ -17,13 +17,13 @@ type Option struct { SkipFiles []string SkipDirs []string FilePatterns []string + Parallel int NoProgress bool Insecure bool Offline bool AppDirs []string SBOMSources []string RekorURL string - Slow bool // Lower CPU and memory AWSRegion string AWSEndpoint string FileChecksum bool // For SPDX @@ -40,8 +40,34 @@ type Option struct { SecretScannerOption analyzer.SecretScannerOption LicenseScannerOption analyzer.LicenseScannerOption - // File walk - WalkOption walker.Option + CustomOption custom.Option +} + +func (o *Option) Init() { + if o.Parallel == 0 { + o.Parallel = 5 // Set the default value + } +} + +func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions { + return analyzer.AnalyzerOptions{ + Group: o.AnalyzerGroup, + FilePatterns: o.FilePatterns, + Parallel: o.Parallel, + DisabledAnalyzers: o.DisabledAnalyzers, + MisconfScannerOption: o.MisconfScannerOption, + SecretScannerOption: o.SecretScannerOption, + LicenseScannerOption: o.LicenseScannerOption, + } +} + +func (o *Option) ConfigAnalyzerOptions() analyzer.ConfigAnalyzerOptions { + return analyzer.ConfigAnalyzerOptions{ + FilePatterns: o.FilePatterns, + DisabledAnalyzers: o.DisabledAnalyzers, + MisconfScannerOption: o.MisconfScannerOption, + SecretScannerOption: o.SecretScannerOption, + } } func (o *Option) Sort() { diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index 87c572b32363..5d584e554ecf 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -12,6 +12,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/samber/lo" "golang.org/x/exp/slices" + "golang.org/x/sync/semaphore" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" @@ -23,7 +24,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/parallel" - "github.com/aquasecurity/trivy/pkg/semaphore" ) type Artifact struct { @@ -43,31 +43,20 @@ type LayerInfo struct { } func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { + opt.Init() + // Initialize handlers handlerManager, err := handler.NewManager(opt) if err != nil { return nil, xerrors.Errorf("handler init error: %w", err) } - a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ - Group: opt.AnalyzerGroup, - Slow: opt.Slow, - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - LicenseScannerOption: opt.LicenseScannerOption, - }) + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("analyzer group error: %w", err) } - ca, err := analyzer.NewConfigAnalyzerGroup(analyzer.ConfigAnalyzerOptions{ - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - }) + ca, err := analyzer.NewConfigAnalyzerGroup(opt.ConfigAnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("config analyzer group error: %w", err) } @@ -75,7 +64,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a return Artifact{ image: img, cache: c, - walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow), + walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs), analyzer: a, configAnalyzer: ca, handlerManager: handlerManager, @@ -215,8 +204,8 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error { var osFound types.OS - workers := lo.Ternary(a.artifactOption.Slow, 1, 5) - p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) { + p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context, + layerKey string) (any, error) { layer := layerKeyMap[layerKey] // If it is a base layer, secret scanning should not be performed. @@ -268,7 +257,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable FileChecksum: a.artifactOption.FileChecksum, } result := analyzer.NewAnalysisResult() - limit := semaphore.New(a.artifactOption.Slow) + limit := semaphore.NewWeighted(int64(a.artifactOption.Parallel)) // Prepare filesystem for post analysis composite, err := a.analyzer.PostAnalyzerFS() diff --git a/pkg/fanal/artifact/image/image_test.go b/pkg/fanal/artifact/image/image_test.go index c257fccb30c2..d65df55f2fc5 100644 --- a/pkg/fanal/artifact/image/image_test.go +++ b/pkg/fanal/artifact/image/image_test.go @@ -2035,11 +2035,8 @@ func TestArtifact_Inspect(t *testing.T) { wantErr: "put layer failed", }, { - name: "sad path, PutBlob returns an error with multiple layers and Slow enabled", + name: "sad path, PutBlob returns an error with multiple layers", imagePath: "../../test/testdata/vuln-image.tar.gz", - artifactOpt: artifact.Option{ - Slow: true, - }, missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ ArtifactID: "sha256:33f9415ed2cd5a9cef5d5144333619745b9ec0f851f0684dd45fa79c6b26a650", diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index 8b024b681d09..e0bf1bb9d583 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/opencontainers/go-digest" + "golang.org/x/sync/semaphore" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" @@ -20,7 +21,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/log" - "github.com/aquasecurity/trivy/pkg/semaphore" ) type Artifact struct { @@ -34,20 +34,14 @@ type Artifact struct { } func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { + opt.Init() + handlerManager, err := handler.NewManager(opt) if err != nil { return nil, xerrors.Errorf("handler initialize error: %w", err) } - a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ - Group: opt.AnalyzerGroup, - Slow: opt.Slow, - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - LicenseScannerOption: opt.LicenseScannerOption, - }) + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("analyzer group error: %w", err) } @@ -56,10 +50,9 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a rootPath: filepath.ToSlash(filepath.Clean(rootPath)), cache: c, walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs), - opt.Slow, opt.WalkOption), + opt.CustomOption), analyzer: a, handlerManager: handlerManager, - artifactOption: opt, }, nil } @@ -122,7 +115,7 @@ func buildPathsToSkip(base string, paths []string) []string { func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) { var wg sync.WaitGroup result := analyzer.NewAnalysisResult() - limit := semaphore.New(a.artifactOption.Slow) + limit := semaphore.NewWeighted(int64(a.artifactOption.Parallel)) opts := analyzer.AnalysisOptions{ Offline: a.artifactOption.Offline, FileChecksum: a.artifactOption.FileChecksum, diff --git a/pkg/fanal/artifact/sbom/sbom.go b/pkg/fanal/artifact/sbom/sbom.go index e788e14b48db..7981e6e793d9 100644 --- a/pkg/fanal/artifact/sbom/sbom.go +++ b/pkg/fanal/artifact/sbom/sbom.go @@ -29,6 +29,7 @@ type Artifact struct { } func NewArtifact(filePath string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { + opt.Init() return Artifact{ filePath: filepath.Clean(filePath), cache: c, diff --git a/pkg/fanal/artifact/vm/vm.go b/pkg/fanal/artifact/vm/vm.go index a9173ed00e7c..151ac140aef4 100644 --- a/pkg/fanal/artifact/vm/vm.go +++ b/pkg/fanal/artifact/vm/vm.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "golang.org/x/sync/semaphore" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" @@ -15,7 +16,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/handler" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" - "github.com/aquasecurity/trivy/pkg/semaphore" ) type Type string @@ -41,7 +41,7 @@ type Storage struct { func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) { var wg sync.WaitGroup - limit := semaphore.New(a.artifactOption.Slow) + limit := semaphore.NewWeighted(int64(a.artifactOption.Parallel)) result := analyzer.NewAnalysisResult() opts := analyzer.AnalysisOptions{ @@ -115,18 +115,13 @@ func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobI } func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { + opt.Init() + handlerManager, err := handler.NewManager(opt) if err != nil { return nil, xerrors.Errorf("handler init error: %w", err) } - a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ - Group: opt.AnalyzerGroup, - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - LicenseScannerOption: opt.LicenseScannerOption, - }) + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("analyzer group error: %w", err) } @@ -135,7 +130,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art cache: c, analyzer: a, handlerManager: handlerManager, - walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow), + walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs), artifactOption: opt, } diff --git a/pkg/fanal/artifact/vm/vm_test.go b/pkg/fanal/artifact/vm/vm_test.go index 33e83a2c4e08..4f1a20f26728 100644 --- a/pkg/fanal/artifact/vm/vm_test.go +++ b/pkg/fanal/artifact/vm/vm_test.go @@ -61,7 +61,7 @@ func TestNewArtifact(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := vm.NewArtifact(tt.target, nil, artifact.Option{}) + _, err := vm.NewArtifact(tt.target, nil, artifact.Option{Parallel: 3}) tt.wantErr(t, err, fmt.Sprintf("NewArtifact(%v, nil, nil)", tt.target)) }) } diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index 9e2563642db5..a91b0bb2fed1 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -10,23 +10,15 @@ import ( "golang.org/x/xerrors" dio "github.com/aquasecurity/go-dep-parser/pkg/io" + "github.com/aquasecurity/trivy/pkg/custom" ) -type ErrorCallback func(pathname string, err error) error - -// Option is a struct that allows users to define a custom walking behavior. -// This option is only available when using Trivy as an imported library and not through CLI flags. -type Option struct { - ErrorCallback ErrorCallback - Delay time.Duration -} - type FS struct { walker - option Option + option custom.Option } -func NewFS(skipFiles, skipDirs []string, slow bool, opt Option) FS { +func NewFS(skipFiles, skipDirs []string, opt custom.Option) FS { if opt.ErrorCallback == nil { opt.ErrorCallback = func(pathname string, err error) error { switch { @@ -43,7 +35,7 @@ func NewFS(skipFiles, skipDirs []string, slow bool, opt Option) FS { } return FS{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs), option: opt, } } @@ -70,9 +62,7 @@ func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc { return err } - if w.walker.slow { - time.Sleep(w.option.Delay) - } + time.Sleep(w.option.Delay) filePath = filepath.Clean(filePath) diff --git a/pkg/fanal/walker/fs_test.go b/pkg/fanal/walker/fs_test.go index 3de96b8e0148..280d8367b969 100644 --- a/pkg/fanal/walker/fs_test.go +++ b/pkg/fanal/walker/fs_test.go @@ -2,6 +2,7 @@ package walker_test import ( "errors" + "github.com/aquasecurity/trivy/pkg/custom" "io" "io/fs" "os" @@ -19,7 +20,7 @@ func TestDir_Walk(t *testing.T) { type fields struct { skipFiles []string skipDirs []string - option walker.Option + option custom.Option } tests := []struct { name string @@ -74,7 +75,7 @@ func TestDir_Walk(t *testing.T) { name: "ignore all errors", rootDir: "testdata/fs/nosuch", fields: fields{ - option: walker.Option{ + option: custom.Option{ ErrorCallback: func(pathname string, err error) error { return nil }, @@ -88,7 +89,7 @@ func TestDir_Walk(t *testing.T) { name: "ignore analysis errors", rootDir: "testdata/fs", fields: fields{ - option: walker.Option{ + option: custom.Option{ ErrorCallback: func(pathname string, err error) error { if errors.Is(err, fs.ErrClosed) { return nil @@ -112,7 +113,7 @@ func TestDir_Walk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, false, tt.fields.option) + w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.option) err := w.Walk(tt.rootDir, tt.analyzeFn) if tt.wantErr != "" { diff --git a/pkg/fanal/walker/tar.go b/pkg/fanal/walker/tar.go index ac4faaa070ac..0f0b1f2f694c 100644 --- a/pkg/fanal/walker/tar.go +++ b/pkg/fanal/walker/tar.go @@ -25,14 +25,10 @@ type LayerTar struct { threshold int64 } -func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar { - threshold := defaultSizeThreshold - if slow { - threshold = slowSizeThreshold - } - +func NewLayerTar(skipFiles, skipDirs []string) LayerTar { + threshold := sizeThreshold return LayerTar{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs), threshold: threshold, } } diff --git a/pkg/fanal/walker/tar_test.go b/pkg/fanal/walker/tar_test.go index 8f1cb98a8010..fed2f9efea7d 100644 --- a/pkg/fanal/walker/tar_test.go +++ b/pkg/fanal/walker/tar_test.go @@ -81,7 +81,7 @@ func TestLayerTar_Walk(t *testing.T) { f, err := os.Open("testdata/test.tar") require.NoError(t, err) - w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true) + w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs) gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn) if tt.wantErr != "" { diff --git a/pkg/fanal/walker/vm.go b/pkg/fanal/walker/vm.go index 52ddfe0b50ff..6b728c2c99b9 100644 --- a/pkg/fanal/walker/vm.go +++ b/pkg/fanal/walker/vm.go @@ -39,14 +39,10 @@ type VM struct { analyzeFn WalkFunc } -func NewVM(skipFiles, skipDirs []string, slow bool) VM { - threshold := defaultSizeThreshold - if slow { - threshold = slowSizeThreshold - } - +func NewVM(skipFiles, skipDirs []string) VM { + threshold := sizeThreshold return VM{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs), threshold: threshold, } } diff --git a/pkg/fanal/walker/walk.go b/pkg/fanal/walker/walk.go index 7205972b98e4..1f098da5b4f6 100644 --- a/pkg/fanal/walker/walk.go +++ b/pkg/fanal/walker/walk.go @@ -23,8 +23,7 @@ var ( ) const ( - defaultSizeThreshold = int64(200) << 20 // 200MB - slowSizeThreshold = int64(100) << 10 // 10KB + sizeThreshold = int64(200) << 20 // 200MB ) type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error @@ -32,10 +31,9 @@ type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) er type walker struct { skipFiles []string skipDirs []string - slow bool } -func newWalker(skipFiles, skipDirs []string, slow bool) walker { +func newWalker(skipFiles, skipDirs []string) walker { var cleanSkipFiles, cleanSkipDirs []string for _, skipFile := range skipFiles { skipFile = filepath.ToSlash(filepath.Clean(skipFile)) @@ -52,7 +50,6 @@ func newWalker(skipFiles, skipDirs []string, slow bool) walker { return walker{ skipFiles: cleanSkipFiles, skipDirs: cleanSkipDirs, - slow: slow, } } diff --git a/pkg/fanal/walker/walk_test.go b/pkg/fanal/walker/walk_test.go index 8b19a92899f1..4d52117aa1ce 100644 --- a/pkg/fanal/walker/walk_test.go +++ b/pkg/fanal/walker/walk_test.go @@ -54,7 +54,7 @@ func Test_shouldSkipFile(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprint(i), func(t *testing.T) { - w := newWalker(tc.skipFiles, nil, false) + w := newWalker(tc.skipFiles, nil) for file, skipResult := range tc.skipMap { assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file)) } @@ -115,7 +115,7 @@ func Test_shouldSkipDir(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprint(i), func(t *testing.T) { - w := newWalker(nil, tc.skipDirs, false) + w := newWalker(nil, tc.skipDirs) for dir, skipResult := range tc.skipMap { assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir)) } diff --git a/pkg/flag/kubernetes_flags.go b/pkg/flag/kubernetes_flags.go index a6e8b563a1dc..5585dd3a4521 100644 --- a/pkg/flag/kubernetes_flags.go +++ b/pkg/flag/kubernetes_flags.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/samber/lo" - "golang.org/x/xerrors" corev1 "k8s.io/api/core/v1" ) @@ -52,12 +51,6 @@ var ( Default: "", Usage: "specify k8s version to validate outdated api by it (example: 1.21.0)", } - ParallelFlag = Flag{ - Name: "parallel", - ConfigName: "kubernetes.parallel", - Default: 5, - Usage: "number (between 1-20) of goroutines enabled for parallel scanning", - } TolerationsFlag = Flag{ Name: "tolerations", ConfigName: "kubernetes.tolerations", @@ -97,7 +90,6 @@ type K8sFlagGroup struct { KubeConfig *Flag Components *Flag K8sVersion *Flag - Parallel *Flag Tolerations *Flag AllNamespaces *Flag NodeCollectorNamespace *Flag @@ -111,7 +103,6 @@ type K8sOptions struct { KubeConfig string Components []string K8sVersion string - Parallel int Tolerations []corev1.Toleration AllNamespaces bool NodeCollectorNamespace string @@ -126,7 +117,6 @@ func NewK8sFlagGroup() *K8sFlagGroup { KubeConfig: &KubeConfigFlag, Components: &ComponentsFlag, K8sVersion: &K8sVersionFlag, - Parallel: &ParallelFlag, Tolerations: &TolerationsFlag, AllNamespaces: &AllNamespaces, NodeCollectorNamespace: &NodeCollectorNamespace, @@ -146,7 +136,6 @@ func (f *K8sFlagGroup) Flags() []*Flag { f.KubeConfig, f.Components, f.K8sVersion, - f.Parallel, f.Tolerations, f.AllNamespaces, f.NodeCollectorNamespace, @@ -160,14 +149,6 @@ func (f *K8sFlagGroup) ToOptions() (K8sOptions, error) { if err != nil { return K8sOptions{}, err } - var parallel int - if f.Parallel != nil { - parallel = getInt(f.Parallel) - // check parallel flag is a valid number between 1-20 - if parallel < 1 || parallel > 20 { - return K8sOptions{}, xerrors.Errorf("unable to parse parallel value, please ensure that the value entered is a valid number between 1-20.") - } - } exludeNodeLabels := make(map[string]string) exludeNodes := getStringSlice(f.ExcludeNodes) for _, exludeNodeValue := range exludeNodes { @@ -184,7 +165,6 @@ func (f *K8sFlagGroup) ToOptions() (K8sOptions, error) { KubeConfig: getString(f.KubeConfig), Components: getStringSlice(f.Components), K8sVersion: getString(f.K8sVersion), - Parallel: parallel, Tolerations: tolerations, AllNamespaces: getBool(f.AllNamespaces), NodeCollectorNamespace: getString(f.NodeCollectorNamespace), diff --git a/pkg/flag/scan_flags.go b/pkg/flag/scan_flags.go index 66280ecb255e..3323e110fec4 100644 --- a/pkg/flag/scan_flags.go +++ b/pkg/flag/scan_flags.go @@ -1,6 +1,8 @@ package flag import ( + "golang.org/x/xerrors" + "github.com/aquasecurity/trivy/pkg/types" xstrings "github.com/aquasecurity/trivy/pkg/x/strings" ) @@ -57,6 +59,7 @@ var ( ConfigName: "scan.slow", Default: false, Usage: "scan over time with lower CPU and memory utilization", + Deprecated: true, } SBOMSourcesFlag = Flag{ Name: "sbom-sources", @@ -77,6 +80,12 @@ var ( Default: false, Usage: "include development dependencies in the report (supported: npm, yarn)", } + ParallelFlag = Flag{ + Name: "parallel", + ConfigName: "scan.parallel", + Default: 5, + Usage: "number (1-20) of goroutines enabled for parallel scanning", + } ) type ScanFlagGroup struct { @@ -86,6 +95,7 @@ type ScanFlagGroup struct { Scanners *Flag FilePatterns *Flag Slow *Flag + Parallel *Flag SBOMSources *Flag RekorURL *Flag IncludeDevDeps *Flag @@ -98,7 +108,7 @@ type ScanOptions struct { OfflineScan bool Scanners types.Scanners FilePatterns []string - Slow bool + Parallel int SBOMSources []string RekorURL string IncludeDevDeps bool @@ -112,6 +122,7 @@ func NewScanFlagGroup() *ScanFlagGroup { Scanners: &ScannersFlag, FilePatterns: &FilePatternsFlag, Slow: &SlowFlag, + Parallel: &ParallelFlag, SBOMSources: &SBOMSourcesFlag, RekorURL: &RekorURLFlag, IncludeDevDeps: &IncludeDevDepsFlag, @@ -130,6 +141,7 @@ func (f *ScanFlagGroup) Flags() []*Flag { f.Scanners, f.FilePatterns, f.Slow, + f.Parallel, f.SBOMSources, f.RekorURL, f.IncludeDevDeps, @@ -142,6 +154,12 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) { target = args[0] } + parallel := getInt(f.Parallel) + // check parallel flag is a valid number between 1-20 + if parallel < 1 || parallel > 20 { + return ScanOptions{}, xerrors.Errorf("'--parallel' must be a number between 1-20: %d", parallel) + } + return ScanOptions{ Target: target, SkipDirs: getStringSlice(f.SkipDirs), @@ -149,7 +167,7 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) { OfflineScan: getBool(f.OfflineScan), Scanners: getUnderlyingStringSlice[types.Scanner](f.Scanners), FilePatterns: getStringSlice(f.FilePatterns), - Slow: getBool(f.Slow), + Parallel: parallel, SBOMSources: getStringSlice(f.SBOMSources), RekorURL: getString(f.RekorURL), IncludeDevDeps: getBool(f.IncludeDevDeps), diff --git a/pkg/parallel/walk.go b/pkg/parallel/walk.go index 09ba2b99ce85..2ba79c428990 100644 --- a/pkg/parallel/walk.go +++ b/pkg/parallel/walk.go @@ -15,7 +15,7 @@ import ( type onFile[T any] func(string, fs.FileInfo, dio.ReadSeekerAt) (T, error) type onWalkResult[T any] func(T) error -func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool, +func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int, onFile onFile[T], onResult onWalkResult[T]) error { g, ctx := errgroup.WithContext(ctx) @@ -54,11 +54,7 @@ func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool, // Start a fixed number of goroutines to read and digest files. c := make(chan T) - limit := 10 - if slow { - limit = 1 - } - for i := 0; i < limit; i++ { + for i := 0; i < parallel; i++ { g.Go(func() error { for path := range paths { if err := walk(ctx, fsys, path, c, onFile); err != nil { diff --git a/pkg/semaphore/semaphore.go b/pkg/semaphore/semaphore.go deleted file mode 100644 index f985cc578617..000000000000 --- a/pkg/semaphore/semaphore.go +++ /dev/null @@ -1,30 +0,0 @@ -package semaphore - -import "golang.org/x/sync/semaphore" - -const defaultSize = 5 - -type options struct { - size int64 -} - -type option func(*options) - -func WithDefault(n int64) option { - return func(opts *options) { - opts.size = defaultSize - } -} - -func New(slow bool, opts ...option) *semaphore.Weighted { - o := &options{size: defaultSize} - for _, opt := range opts { - opt(o) - } - if slow { - // Process in series - return semaphore.NewWeighted(1) - } - // Process in parallel - return semaphore.NewWeighted(o.size) -} From c778768c7ce652e1d2fc381b86872712e2501043 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 9 Nov 2023 22:55:14 +0900 Subject: [PATCH 06/23] test: remove external Signed-off-by: knqyf263 --- pkg/fanal/external/config_scan.go | 79 ---------- pkg/fanal/external/config_scan_test.go | 143 ------------------ pkg/fanal/external/testdata/allow/Dockerfile | 3 - pkg/fanal/external/testdata/allow/docker.rego | 20 --- pkg/fanal/external/testdata/deny/Dockerfile | 3 - pkg/fanal/external/testdata/deny/docker.rego | 25 --- 6 files changed, 273 deletions(-) delete mode 100644 pkg/fanal/external/config_scan.go delete mode 100644 pkg/fanal/external/config_scan_test.go delete mode 100644 pkg/fanal/external/testdata/allow/Dockerfile delete mode 100644 pkg/fanal/external/testdata/allow/docker.rego delete mode 100644 pkg/fanal/external/testdata/deny/Dockerfile delete mode 100644 pkg/fanal/external/testdata/deny/docker.rego diff --git a/pkg/fanal/external/config_scan.go b/pkg/fanal/external/config_scan.go deleted file mode 100644 index 43f9ede430f4..000000000000 --- a/pkg/fanal/external/config_scan.go +++ /dev/null @@ -1,79 +0,0 @@ -package external - -import ( - "context" - "errors" - - "github.com/aquasecurity/trivy/pkg/fanal/analyzer" - "github.com/aquasecurity/trivy/pkg/fanal/applier" - "github.com/aquasecurity/trivy/pkg/fanal/artifact" - "github.com/aquasecurity/trivy/pkg/fanal/artifact/local" - "github.com/aquasecurity/trivy/pkg/fanal/cache" - "github.com/aquasecurity/trivy/pkg/fanal/types" - "github.com/aquasecurity/trivy/pkg/misconf" - - _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/all" -) - -type ConfigScanner struct { - cache cache.FSCache - policyPaths []string - dataPaths []string - namespaces []string - allowEmbedded bool -} - -func NewConfigScanner(cacheDir string, policyPaths, dataPaths, namespaces []string, allowEmbedded bool) (*ConfigScanner, error) { - // Initialize local cache - cacheClient, err := cache.NewFSCache(cacheDir) - if err != nil { - return nil, err - } - - return &ConfigScanner{ - cache: cacheClient, - policyPaths: policyPaths, - dataPaths: dataPaths, - namespaces: namespaces, - allowEmbedded: allowEmbedded, - }, nil -} - -func (s ConfigScanner) Scan(dir string) ([]types.Misconfiguration, error) { - art, err := local.NewArtifact(dir, s.cache, artifact.Option{ - MisconfScannerOption: misconf.ScannerOption{ - PolicyPaths: s.policyPaths, - DataPaths: s.dataPaths, - Namespaces: s.namespaces, - DisableEmbeddedPolicies: !s.allowEmbedded, - DisableEmbeddedLibraries: !s.allowEmbedded, - }, - }) - if err != nil { - return nil, err - } - - // Scan config files - result, err := art.Inspect(context.Background()) - if err != nil { - return nil, err - } - - // Merge layers - a := applier.NewApplier(s.cache) - mergedLayer, err := a.ApplyLayers(result.ID, result.BlobIDs) - if !errors.Is(err, analyzer.ErrUnknownOS) && !errors.Is(err, analyzer.ErrNoPkgsDetected) { - return nil, err - } - - // Do not assert successes and layer - for i := range mergedLayer.Misconfigurations { - mergedLayer.Misconfigurations[i].Layer = types.Layer{} - } - - return mergedLayer.Misconfigurations, nil -} - -func (s ConfigScanner) Close() error { - return s.cache.Close() -} diff --git a/pkg/fanal/external/config_scan_test.go b/pkg/fanal/external/config_scan_test.go deleted file mode 100644 index 094fdf41accf..000000000000 --- a/pkg/fanal/external/config_scan_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package external_test - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/aquasecurity/trivy/pkg/fanal/external" - "github.com/aquasecurity/trivy/pkg/fanal/types" - - _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/all" -) - -func TestConfigScanner_Scan(t *testing.T) { - type fields struct { - policyPaths []string - dataPaths []string - namespaces []string - } - tests := []struct { - name string - fields fields - inputDir string - want []types.Misconfiguration - }{ - { - name: "deny", - fields: fields{ - policyPaths: []string{filepath.Join("testdata", "deny")}, - namespaces: []string{"testdata"}, - }, - inputDir: filepath.Join("testdata", "deny"), - want: []types.Misconfiguration{ - { - FileType: "dockerfile", - FilePath: "Dockerfile", - Failures: types.MisconfResults{ - types.MisconfResult{ - Namespace: "testdata.xyz_200", - Query: "data.testdata.xyz_200.deny", - Message: "Old image", - PolicyMetadata: types.PolicyMetadata{ - ID: "XYZ-200", - Type: "Dockerfile Security Check", - Title: "Old FROM", - Description: "Rego module: data.testdata.xyz_200", - Severity: "LOW", - RecommendedActions: "", - References: []string(nil), - }, - CauseMetadata: types.CauseMetadata{ - Resource: "", - Provider: "Dockerfile", - Service: "general", - StartLine: 1, - EndLine: 2, - Code: types.Code{ - Lines: []types.Line{ - { - Number: 1, - Content: "FROM alpine:3.10", - Highlighted: "\x1b[38;5;64mFROM\x1b[0m\x1b[38;5;37m alpine:3.10", - IsCause: true, - Annotation: "", - Truncated: false, - FirstCause: true, - LastCause: false, - }, - { - Number: 2, - Content: "", - Highlighted: "\x1b[0m", - IsCause: true, - Annotation: "", - Truncated: false, - FirstCause: false, - LastCause: true, - }, - }, - }, - }, Traces: []string(nil), - }, - }, Warnings: types.MisconfResults(nil), - Successes: types.MisconfResults(nil), - Exceptions: types.MisconfResults(nil), - Layer: types.Layer{ - Digest: "", - DiffID: "", - }, - }, - }, - }, - { - name: "allow", - fields: fields{ - policyPaths: []string{filepath.Join("testdata", "allow")}, - namespaces: []string{"testdata"}, - }, - inputDir: filepath.Join("testdata", "allow"), - want: []types.Misconfiguration{ - { - FileType: "dockerfile", - FilePath: "Dockerfile", - Successes: types.MisconfResults{ - { - Namespace: "testdata.xyz_200", - Query: "data.testdata.xyz_200.deny", - PolicyMetadata: types.PolicyMetadata{ - ID: "XYZ-200", - Type: "Dockerfile Security Check", - Title: "Old FROM", - Description: "Rego module: data.testdata.xyz_200", - Severity: "LOW", - }, - CauseMetadata: types.CauseMetadata{ - Resource: "", - Provider: "Dockerfile", - Service: "general", - StartLine: 0, - EndLine: 0, - }, - }, - }, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s, err := external.NewConfigScanner(t.TempDir(), - tt.fields.policyPaths, tt.fields.dataPaths, tt.fields.namespaces, false) - require.NoError(t, err) - - defer func() { _ = s.Close() }() - - got, err := s.Scan(tt.inputDir) - require.NoError(t, err) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/fanal/external/testdata/allow/Dockerfile b/pkg/fanal/external/testdata/allow/Dockerfile deleted file mode 100644 index fb7daf440fc6..000000000000 --- a/pkg/fanal/external/testdata/allow/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM alpine:3.14 - -ADD foo.txt . \ No newline at end of file diff --git a/pkg/fanal/external/testdata/allow/docker.rego b/pkg/fanal/external/testdata/allow/docker.rego deleted file mode 100644 index 0ef57f138813..000000000000 --- a/pkg/fanal/external/testdata/allow/docker.rego +++ /dev/null @@ -1,20 +0,0 @@ -package testdata.xyz_200 - -__rego_metadata__ := { - "id": "XYZ-200", - "title": "Old FROM", - "version": "v1.0.0", - "severity": "LOW", - "type": "Docker Security Check", -} - -__rego_input__ := { - "combine": false, - "selector": [{"type": "dockerfile"}], -} - -deny[msg] { - input.stages[from] - from == "alpine:3.10" - msg := "Old image" -} diff --git a/pkg/fanal/external/testdata/deny/Dockerfile b/pkg/fanal/external/testdata/deny/Dockerfile deleted file mode 100644 index b3be16e28eac..000000000000 --- a/pkg/fanal/external/testdata/deny/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM alpine:3.10 - -ADD foo.txt . \ No newline at end of file diff --git a/pkg/fanal/external/testdata/deny/docker.rego b/pkg/fanal/external/testdata/deny/docker.rego deleted file mode 100644 index 6be4af8ee57e..000000000000 --- a/pkg/fanal/external/testdata/deny/docker.rego +++ /dev/null @@ -1,25 +0,0 @@ -package testdata.xyz_200 - -__rego_metadata__ := { - "id": "XYZ-200", - "title": "Old FROM", - "version": "v1.0.0", - "severity": "LOW", - "type": "Docker Security Check", -} - -__rego_input__ := { - "combine": false, - "selector": [{"type": "dockerfile"}], -} - -deny[res] { - stage := input.Stages[_] - stage.Name == "alpine:3.10" - msg := "Old image" - res := { - "msg": msg, - "startline": 1, - "endline": 2, - } -} From 7cfb534dd42677147deb67c2add593d46578cc89 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 9 Nov 2023 22:58:55 +0900 Subject: [PATCH 07/23] refactor: inject filesystem walker Signed-off-by: knqyf263 --- pkg/commands/artifact/run.go | 10 +- pkg/commands/artifact/wire_gen.go | 13 +- pkg/custom/custom.go | 19 --- pkg/fanal/artifact/artifact.go | 10 +- pkg/fanal/artifact/image/image.go | 2 +- pkg/fanal/artifact/local/fs.go | 76 ++-------- pkg/fanal/artifact/local/fs_test.go | 124 ++-------------- pkg/fanal/artifact/repo/git.go | 22 ++- pkg/fanal/artifact/repo/git_test.go | 5 +- pkg/fanal/artifact/vm/vm.go | 2 +- pkg/fanal/cache/key.go | 2 +- pkg/fanal/cache/key_test.go | 8 +- pkg/fanal/test/integration/library_test.go | 1 - pkg/fanal/walker/cached_file.go | 11 +- pkg/fanal/walker/fs.go | 151 +++++++++++++------ pkg/fanal/walker/fs_test.go | 161 +++++++++++++++------ pkg/fanal/walker/tar.go | 23 ++- pkg/fanal/walker/tar_test.go | 16 +- pkg/fanal/walker/vm.go | 31 ++-- pkg/fanal/walker/walk.go | 84 +++-------- pkg/fanal/walker/walk_test.go | 141 ++++++++++-------- pkg/flag/scan_flags.go | 12 +- pkg/parallel/walk.go | 3 + pkg/scanner/scan.go | 8 +- 24 files changed, 436 insertions(+), 499 deletions(-) delete mode 100644 pkg/custom/custom.go diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index 274d2f0a30ec..a66d4bf6f5fd 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -18,6 +18,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/artifact" "github.com/aquasecurity/trivy/pkg/fanal/cache" ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/flag" "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/log" @@ -72,6 +73,9 @@ type ScannerConfig struct { // Artifact options ArtifactOption artifact.Option + + // Walk options + WalkerOption walker.Option } type Runner interface { @@ -625,8 +629,6 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi }, ArtifactOption: artifact.Option{ DisabledAnalyzers: disabledAnalyzers(opts), - SkipFiles: opts.SkipFiles, - SkipDirs: opts.SkipDirs, FilePatterns: opts.FilePatterns, Offline: opts.OfflineScan, Parallel: opts.Parallel, @@ -665,6 +667,10 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi ClassifierConfidenceLevel: opts.LicenseConfidenceLevel, }, }, + WalkerOption: walker.Option{ + SkipFiles: opts.SkipFiles, + SkipDirs: opts.SkipDirs, + }, }, scanOptions, nil } diff --git a/pkg/commands/artifact/wire_gen.go b/pkg/commands/artifact/wire_gen.go index 9f89dca066d8..69606aea05b2 100644 --- a/pkg/commands/artifact/wire_gen.go +++ b/pkg/commands/artifact/wire_gen.go @@ -19,6 +19,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/cache" "github.com/aquasecurity/trivy/pkg/fanal/image" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/rpc/client" "github.com/aquasecurity/trivy/pkg/scanner" "github.com/aquasecurity/trivy/pkg/scanner/langpkg" @@ -82,7 +83,8 @@ func initializeFilesystemScanner(ctx context.Context, path string, artifactCache config := db.Config{} client := vulnerability.NewClient(config) localScanner := local.NewScanner(applierApplier, ospkgScanner, langpkgScanner, client) - artifactArtifact, err := local2.NewArtifact(path, artifactCache, artifactOption) + fs := walker.NewFS() + artifactArtifact, err := local2.NewArtifact(path, artifactCache, fs, artifactOption) if err != nil { return scanner.Scanner{}, nil, err } @@ -98,7 +100,8 @@ func initializeRepositoryScanner(ctx context.Context, url string, artifactCache config := db.Config{} client := vulnerability.NewClient(config) localScanner := local.NewScanner(applierApplier, ospkgScanner, langpkgScanner, client) - artifactArtifact, cleanup, err := repo.NewArtifact(url, artifactCache, artifactOption) + fs := walker.NewFS() + artifactArtifact, cleanup, err := repo.NewArtifact(url, artifactCache, fs, artifactOption) if err != nil { return scanner.Scanner{}, nil, err } @@ -185,7 +188,8 @@ func initializeRemoteArchiveScanner(ctx context.Context, filePath string, artifa func initializeRemoteFilesystemScanner(ctx context.Context, path string, artifactCache cache.ArtifactCache, remoteScanOptions client.ScannerOption, artifactOption artifact.Option) (scanner.Scanner, func(), error) { v := _wireValue clientScanner := client.NewScanner(remoteScanOptions, v...) - artifactArtifact, err := local2.NewArtifact(path, artifactCache, artifactOption) + fs := walker.NewFS() + artifactArtifact, err := local2.NewArtifact(path, artifactCache, fs, artifactOption) if err != nil { return scanner.Scanner{}, nil, err } @@ -198,7 +202,8 @@ func initializeRemoteFilesystemScanner(ctx context.Context, path string, artifac func initializeRemoteRepositoryScanner(ctx context.Context, url string, artifactCache cache.ArtifactCache, remoteScanOptions client.ScannerOption, artifactOption artifact.Option) (scanner.Scanner, func(), error) { v := _wireValue clientScanner := client.NewScanner(remoteScanOptions, v...) - artifactArtifact, cleanup, err := repo.NewArtifact(url, artifactCache, artifactOption) + fs := walker.NewFS() + artifactArtifact, cleanup, err := repo.NewArtifact(url, artifactCache, fs, artifactOption) if err != nil { return scanner.Scanner{}, nil, err } diff --git a/pkg/custom/custom.go b/pkg/custom/custom.go deleted file mode 100644 index 9df8489dfdd3..000000000000 --- a/pkg/custom/custom.go +++ /dev/null @@ -1,19 +0,0 @@ -package custom - -import ( - "time" -) - -type ErrorCallback func(pathname string, err error) error - -// Option is a struct that allows defining a custom behavior. -// This option is only available when Trivy is used as an imported library and not through CLI flags. -type Option struct { - // Delay is the amount of time to wait between each file walk. - // Default: 0 sec - Delay time.Duration - - // ErrorCallback is a function that allows users to define a custom error handling behavior while walking the filesystem. - // If not defined, the default behavior is to halt traversal on any error. - ErrorCallback ErrorCallback -} diff --git a/pkg/fanal/artifact/artifact.go b/pkg/fanal/artifact/artifact.go index de52045d4507..41f22afedd26 100644 --- a/pkg/fanal/artifact/artifact.go +++ b/pkg/fanal/artifact/artifact.go @@ -4,9 +4,9 @@ import ( "context" "sort" - "github.com/aquasecurity/trivy/pkg/custom" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/misconf" ) @@ -14,8 +14,6 @@ type Option struct { AnalyzerGroup analyzer.Group // It is empty in OSS DisabledAnalyzers []analyzer.Type DisabledHandlers []types.HandlerType - SkipFiles []string - SkipDirs []string FilePatterns []string Parallel int NoProgress bool @@ -40,7 +38,7 @@ type Option struct { SecretScannerOption analyzer.SecretScannerOption LicenseScannerOption analyzer.LicenseScannerOption - CustomOption custom.Option + WalkerOption walker.Option } func (o *Option) Init() { @@ -74,8 +72,8 @@ func (o *Option) Sort() { sort.Slice(o.DisabledAnalyzers, func(i, j int) bool { return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j] }) - sort.Strings(o.SkipFiles) - sort.Strings(o.SkipDirs) + sort.Strings(o.WalkerOption.SkipFiles) + sort.Strings(o.WalkerOption.SkipDirs) sort.Strings(o.FilePatterns) } diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index 5d584e554ecf..e166b1925105 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -64,7 +64,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a return Artifact{ image: img, cache: c, - walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs), + walker: walker.NewLayerTar(opt.WalkerOption), analyzer: a, configAnalyzer: ca, handlerManager: handlerManager, diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index 34f8682a87b1..f4fce12cabc9 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/google/wire" "github.com/opencontainers/go-digest" "golang.org/x/sync/semaphore" "golang.org/x/xerrors" @@ -20,20 +21,25 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/handler" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" - "github.com/aquasecurity/trivy/pkg/log" +) + +var ArtifactSet = wire.NewSet( + walker.NewFS, + wire.Bind(new(walker.FSWalker), new(*walker.FS)), + NewArtifact, ) type Artifact struct { rootPath string cache cache.ArtifactCache - walker walker.FS + walker walker.FSWalker analyzer analyzer.AnalyzerGroup handlerManager handler.Manager artifactOption artifact.Option } -func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { +func NewArtifact(rootPath string, c cache.ArtifactCache, w walker.FSWalker, opt artifact.Option) (artifact.Artifact, error) { opt.Init() handlerManager, err := handler.NewManager(opt) @@ -47,71 +53,15 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a } return Artifact{ - rootPath: filepath.ToSlash(filepath.Clean(rootPath)), - cache: c, - walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs), - opt.CustomOption), + rootPath: filepath.ToSlash(filepath.Clean(rootPath)), + cache: c, + walker: w, analyzer: a, handlerManager: handlerManager, artifactOption: opt, }, nil } -// buildPathsToSkip builds correct patch for skipDirs and skipFiles -func buildPathsToSkip(base string, paths []string) []string { - var relativePaths []string - absBase, err := filepath.Abs(base) - if err != nil { - log.Logger.Warnf("Failed to get an absolute path of %s: %s", base, err) - return nil - } - for _, path := range paths { - // Supports three types of flag specification. - // All of them are converted into the relative path from the root directory. - // 1. Relative skip dirs/files from the root directory - // The specified dirs and files will be used as is. - // e.g. $ trivy fs --skip-dirs bar ./foo - // The skip dir from the root directory will be `bar/`. - // 2. Relative skip dirs/files from the working directory - // The specified dirs and files wll be converted to the relative path from the root directory. - // e.g. $ trivy fs --skip-dirs ./foo/bar ./foo - // The skip dir will be converted to `bar/`. - // 3. Absolute skip dirs/files - // The specified dirs and files wll be converted to the relative path from the root directory. - // e.g. $ trivy fs --skip-dirs /bar/foo/baz ./foo - // When the working directory is - // 3.1 /bar: the skip dir will be converted to `baz/`. - // 3.2 /hoge : the skip dir will be converted to `../../bar/foo/baz/`. - - absSkipPath, err := filepath.Abs(path) - if err != nil { - log.Logger.Warnf("Failed to get an absolute path of %s: %s", base, err) - continue - } - rel, err := filepath.Rel(absBase, absSkipPath) - if err != nil { - log.Logger.Warnf("Failed to get a relative path from %s to %s: %s", base, path, err) - continue - } - - var relPath string - switch { - case !filepath.IsAbs(path) && strings.HasPrefix(rel, ".."): - // #1: Use the path as is - relPath = path - case !filepath.IsAbs(path) && !strings.HasPrefix(rel, ".."): - // #2: Use the relative path from the root directory - relPath = rel - case filepath.IsAbs(path): - // #3: Use the relative path from the root directory - relPath = rel - } - relPath = filepath.ToSlash(relPath) - relativePaths = append(relativePaths, relPath) - } - return relativePaths -} - func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) { var wg sync.WaitGroup result := analyzer.NewAnalysisResult() @@ -127,7 +77,7 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) return types.ArtifactReference{}, xerrors.Errorf("failed to prepare filesystem for post analysis: %w", err) } - err = a.walker.Walk(a.rootPath, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + err = a.walker.Walk(a.rootPath, a.artifactOption.WalkerOption, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { dir := a.rootPath // When the directory is the same as the filePath, a file was given diff --git a/pkg/fanal/artifact/local/fs_test.go b/pkg/fanal/artifact/local/fs_test.go index e83bb0b8a7c0..1b3e658d21d8 100644 --- a/pkg/fanal/artifact/local/fs_test.go +++ b/pkg/fanal/artifact/local/fs_test.go @@ -3,19 +3,16 @@ package local import ( "context" "errors" - "path/filepath" - "runtime" + "github.com/aquasecurity/trivy/pkg/fanal/walker" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" - "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/artifact" "github.com/aquasecurity/trivy/pkg/fanal/cache" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/misconf" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config/all" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/python/pip" @@ -244,7 +241,7 @@ func TestArtifact_Inspect(t *testing.T) { c := new(cache.MockArtifactCache) c.ApplyPutBlobExpectation(tt.putBlobExpectation) - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) @@ -260,107 +257,6 @@ func TestArtifact_Inspect(t *testing.T) { } } -func TestBuildPathsToSkip(t *testing.T) { - tests := []struct { - name string - oses []string - paths []string - base string - want []string - }{ - // Linux/macOS - { - name: "path - abs, base - abs, not joining paths", - oses: []string{ - "linux", - "darwin", - }, - base: "/foo", - paths: []string{"/foo/bar"}, - want: []string{"bar"}, - }, - { - name: "path - abs, base - rel", - oses: []string{ - "linux", - "darwin", - }, - base: "foo", - paths: func() []string { - abs, err := filepath.Abs("foo/bar") - require.NoError(t, err) - return []string{abs} - }(), - want: []string{"bar"}, - }, - { - name: "path - rel, base - rel, joining paths", - oses: []string{ - "linux", - "darwin", - }, - base: "foo", - paths: []string{"bar"}, - want: []string{"bar"}, - }, - { - name: "path - rel, base - rel, not joining paths", - oses: []string{ - "linux", - "darwin", - }, - base: "foo", - paths: []string{"foo/bar/bar"}, - want: []string{"bar/bar"}, - }, - { - name: "path - rel with dot, base - rel, removing the leading dot and not joining paths", - oses: []string{ - "linux", - "darwin", - }, - base: "foo", - paths: []string{"./foo/bar"}, - want: []string{"bar"}, - }, - { - name: "path - rel, base - dot", - oses: []string{ - "linux", - "darwin", - }, - base: ".", - paths: []string{"foo/bar"}, - want: []string{"foo/bar"}, - }, - // Windows - { - name: "path - rel, base - rel. Skip common prefix", - oses: []string{"windows"}, - base: "foo", - paths: []string{"foo\\bar\\bar"}, - want: []string{"bar/bar"}, - }, - { - name: "path - rel, base - dot, windows", - oses: []string{"windows"}, - base: ".", - paths: []string{"foo\\bar"}, - want: []string{"foo/bar"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if !slices.Contains(tt.oses, runtime.GOOS) { - t.Skipf("Skip path tests for %q", tt.oses) - } - got := buildPathsToSkip(tt.base, tt.paths) - assert.Equal(t, tt.want, got) - }) - } -} - var policyMetadata = types.PolicyMetadata{ ID: "TEST001", AVDID: "AVD-TEST-0001", @@ -815,7 +711,7 @@ func TestTerraformMisconfigurationScan(t *testing.T) { types.SystemFileFilteringPostHandler, } tt.artifactOpt.MisconfScannerOption.DisableEmbeddedPolicies = true - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) @@ -1081,7 +977,7 @@ func TestCloudFormationMisconfigurationScan(t *testing.T) { types.SystemFileFilteringPostHandler, } tt.artifactOpt.MisconfScannerOption.DisableEmbeddedPolicies = true - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) @@ -1316,7 +1212,7 @@ func TestDockerfileMisconfigurationScan(t *testing.T) { tt.artifactOpt.DisabledHandlers = []types.HandlerType{ types.SystemFileFilteringPostHandler, } - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) @@ -1584,7 +1480,7 @@ func TestKubernetesMisconfigurationScan(t *testing.T) { tt.artifactOpt.DisabledHandlers = []types.HandlerType{ types.SystemFileFilteringPostHandler, } - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) @@ -1841,7 +1737,7 @@ func TestAzureARMMisconfigurationScan(t *testing.T) { tt.artifactOpt.DisabledHandlers = []types.HandlerType{ types.SystemFileFilteringPostHandler, } - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) @@ -1957,7 +1853,7 @@ func TestMixedConfigurationScan(t *testing.T) { tt.artifactOpt.DisabledHandlers = []types.HandlerType{ types.SystemFileFilteringPostHandler, } - a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt) + a, err := NewArtifact(tt.fields.dir, c, walker.NewFS(), tt.artifactOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) diff --git a/pkg/fanal/artifact/repo/git.go b/pkg/fanal/artifact/repo/git.go index de1a528f65dc..03d10337d0dc 100644 --- a/pkg/fanal/artifact/repo/git.go +++ b/pkg/fanal/artifact/repo/git.go @@ -8,6 +8,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/google/wire" "github.com/hashicorp/go-multierror" "golang.org/x/xerrors" @@ -15,6 +16,13 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/artifact/local" "github.com/aquasecurity/trivy/pkg/fanal/cache" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/fanal/walker" +) + +var ArtifactSet = wire.NewSet( + walker.NewFS, + wire.Bind(new(walker.FSWalker), new(*walker.FS)), + NewArtifact, ) type Artifact struct { @@ -22,21 +30,21 @@ type Artifact struct { local artifact.Artifact } -func NewArtifact(target string, c cache.ArtifactCache, artifactOpt artifact.Option) ( +func NewArtifact(target string, c cache.ArtifactCache, w walker.FSWalker, artifactOpt artifact.Option) ( artifact.Artifact, func(), error) { var cleanup func() var errs error // Try the local repository - art, err := tryLocalRepo(target, c, artifactOpt) + art, err := tryLocalRepo(target, c, w, artifactOpt) if err == nil { return art, func() {}, nil } errs = multierror.Append(errs, err) // Try the remote git repository - art, cleanup, err = tryRemoteRepo(target, c, artifactOpt) + art, cleanup, err = tryRemoteRepo(target, c, w, artifactOpt) if err == nil { return art, cleanup, nil } @@ -64,12 +72,12 @@ func (Artifact) Clean(_ types.ArtifactReference) error { return nil } -func tryLocalRepo(target string, c cache.ArtifactCache, artifactOpt artifact.Option) (artifact.Artifact, error) { +func tryLocalRepo(target string, c cache.ArtifactCache, w walker.FSWalker, artifactOpt artifact.Option) (artifact.Artifact, error) { if _, err := os.Stat(target); err != nil { return nil, xerrors.Errorf("no such path: %w", err) } - art, err := local.NewArtifact(target, c, artifactOpt) + art, err := local.NewArtifact(target, c, w, artifactOpt) if err != nil { return nil, xerrors.Errorf("local repo artifact error: %w", err) } @@ -78,7 +86,7 @@ func tryLocalRepo(target string, c cache.ArtifactCache, artifactOpt artifact.Opt }, nil } -func tryRemoteRepo(target string, c cache.ArtifactCache, artifactOpt artifact.Option) (artifact.Artifact, func(), error) { +func tryRemoteRepo(target string, c cache.ArtifactCache, w walker.FSWalker, artifactOpt artifact.Option) (artifact.Artifact, func(), error) { cleanup := func() {} u, err := newURL(target) if err != nil { @@ -92,7 +100,7 @@ func tryRemoteRepo(target string, c cache.ArtifactCache, artifactOpt artifact.Op cleanup = func() { _ = os.RemoveAll(tmpDir) } - art, err := local.NewArtifact(tmpDir, c, artifactOpt) + art, err := local.NewArtifact(tmpDir, c, w, artifactOpt) if err != nil { return nil, cleanup, xerrors.Errorf("fs artifact: %w", err) } diff --git a/pkg/fanal/artifact/repo/git_test.go b/pkg/fanal/artifact/repo/git_test.go index ca29941a597d..2f096f833407 100644 --- a/pkg/fanal/artifact/repo/git_test.go +++ b/pkg/fanal/artifact/repo/git_test.go @@ -4,6 +4,7 @@ package repo import ( "context" + "github.com/aquasecurity/trivy/pkg/fanal/walker" "net/http/httptest" "testing" @@ -165,7 +166,7 @@ func TestNewArtifact(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, cleanup, err := NewArtifact(tt.args.target, tt.args.c, artifact.Option{ + _, cleanup, err := NewArtifact(tt.args.target, tt.args.c, walker.NewFS(), artifact.Option{ NoProgress: tt.args.noProgress, RepoBranch: tt.args.repoBranch, RepoTag: tt.args.repoTag, @@ -207,7 +208,7 @@ func TestArtifact_Inspect(t *testing.T) { fsCache, err := cache.NewFSCache(t.TempDir()) require.NoError(t, err) - art, cleanup, err := NewArtifact(tt.rawurl, fsCache, artifact.Option{}) + art, cleanup, err := NewArtifact(tt.rawurl, fsCache, walker.NewFS(), artifact.Option{}) require.NoError(t, err) defer cleanup() diff --git a/pkg/fanal/artifact/vm/vm.go b/pkg/fanal/artifact/vm/vm.go index 151ac140aef4..5ddbbb43e76a 100644 --- a/pkg/fanal/artifact/vm/vm.go +++ b/pkg/fanal/artifact/vm/vm.go @@ -130,7 +130,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art cache: c, analyzer: a, handlerManager: handlerManager, - walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs), + walker: walker.NewVM(opt.WalkerOption), artifactOption: opt, } diff --git a/pkg/fanal/cache/key.go b/pkg/fanal/cache/key.go index f9258caa866b..b11cfc4be1df 100644 --- a/pkg/fanal/cache/key.go +++ b/pkg/fanal/cache/key.go @@ -30,7 +30,7 @@ func CalcKey(id string, analyzerVersions analyzer.Versions, hookVersions map[str SkipFiles []string SkipDirs []string FilePatterns []string `json:",omitempty"` - }{id, analyzerVersions, hookVersions, artifactOpt.SkipFiles, artifactOpt.SkipDirs, artifactOpt.FilePatterns} + }{id, analyzerVersions, hookVersions, artifactOpt.WalkerOption.SkipFiles, artifactOpt.WalkerOption.SkipDirs, artifactOpt.FilePatterns} if err := json.NewEncoder(h).Encode(keyBase); err != nil { return "", xerrors.Errorf("json encode error: %w", err) diff --git a/pkg/fanal/cache/key_test.go b/pkg/fanal/cache/key_test.go index 94828e5d485e..f21910b275ae 100644 --- a/pkg/fanal/cache/key_test.go +++ b/pkg/fanal/cache/key_test.go @@ -1,6 +1,7 @@ package cache import ( + "github.com/aquasecurity/trivy/pkg/fanal/walker" "testing" "github.com/stretchr/testify/assert" @@ -230,8 +231,6 @@ func TestCalcKey(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { artifactOpt := artifact.Option{ - SkipFiles: tt.args.skipFiles, - SkipDirs: tt.args.skipDirs, FilePatterns: tt.args.patterns, MisconfScannerOption: misconf.ScannerOption{ @@ -242,6 +241,11 @@ func TestCalcKey(t *testing.T) { SecretScannerOption: analyzer.SecretScannerOption{ ConfigPath: tt.args.secretConfigPath, }, + + WalkerOption: walker.Option{ + SkipFiles: tt.args.skipFiles, + SkipDirs: tt.args.skipDirs, + }, } got, err := CalcKey(tt.args.key, tt.args.analyzerVersions, tt.args.hookVersions, artifactOpt) if tt.wantErr != "" { diff --git a/pkg/fanal/test/integration/library_test.go b/pkg/fanal/test/integration/library_test.go index e8be494a57d5..61f8da757925 100644 --- a/pkg/fanal/test/integration/library_test.go +++ b/pkg/fanal/test/integration/library_test.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package integration diff --git a/pkg/fanal/walker/cached_file.go b/pkg/fanal/walker/cached_file.go index eed1cf4bdcd9..48605ae135e3 100644 --- a/pkg/fanal/walker/cached_file.go +++ b/pkg/fanal/walker/cached_file.go @@ -19,17 +19,14 @@ type cachedFile struct { size int64 reader io.Reader - threshold int64 //　Files larger than this threshold are written to file without being read into memory. - content []byte // It will be populated if this file is small filePath string // It will be populated if this file is large } -func newCachedFile(size int64, r io.Reader, threshold int64) *cachedFile { +func newCachedFile(size int64, r io.Reader) *cachedFile { return &cachedFile{ - size: size, - reader: r, - threshold: threshold, + size: size, + reader: r, } } @@ -39,7 +36,7 @@ func newCachedFile(size int64, r io.Reader, threshold int64) *cachedFile { func (o *cachedFile) Open() (dio.ReadSeekCloserAt, error) { o.once.Do(func() { // When the file is large, it will be written down to a temp file. - if o.size >= o.threshold { + if o.size >= defaultSizeThreshold { f, err := os.CreateTemp("", "fanal-*") if err != nil { o.err = xerrors.Errorf("failed to create the temp file: %w", err) diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index a91b0bb2fed1..ed9a348bbdb8 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -5,67 +5,50 @@ import ( "io/fs" "os" "path/filepath" - "time" + "strings" "golang.org/x/xerrors" dio "github.com/aquasecurity/go-dep-parser/pkg/io" - "github.com/aquasecurity/trivy/pkg/custom" + "github.com/aquasecurity/trivy/pkg/log" ) -type FS struct { - walker - option custom.Option +var _ FSWalker = (*FS)(nil) + +type FSWalker interface { + Walk(root string, opt Option, fn WalkFunc) error } -func NewFS(skipFiles, skipDirs []string, opt custom.Option) FS { - if opt.ErrorCallback == nil { - opt.ErrorCallback = func(pathname string, err error) error { - switch { - // Unwrap fs.SkipDir error - case errors.Is(err, fs.SkipDir): - return fs.SkipDir - // ignore permission errors - case os.IsPermission(err): - return nil - } - // halt traversal on any other error - return xerrors.Errorf("unknown error with %s: %w", pathname, err) - } - } +// FS is the filesystem walker +type FS struct{} - return FS{ - walker: newWalker(skipFiles, skipDirs), - option: opt, - } +func NewFS() *FS { + return &FS{} } -// Walk walks the file tree rooted at root, calling WalkDirFunc for each file or -// directory in the tree, including root, but a directory to be ignored will be skipped. -func (w FS) Walk(root string, fn WalkFunc) error { - walkDir := w.walkDirFunc(root, fn) - err := filepath.WalkDir(root, func(filePath string, d fs.DirEntry, err error) error { - if walkErr := walkDir(filePath, d, err); walkErr != nil { - return w.option.ErrorCallback(filePath, walkErr) - } - return nil - }) - if err != nil { +// Walk walks the filesystem rooted at root, calling fn for each unfiltered file. +func (w *FS) Walk(root string, opt Option, fn WalkFunc) error { + opt.SkipFiles = w.BuildSkipPaths(root, opt.SkipFiles) + opt.SkipDirs = append(opt.SkipDirs, defaultSkipDirs...) + opt.SkipDirs = w.BuildSkipPaths(root, opt.SkipDirs) + + walkDirFunc := w.WalkDirFunc(root, fn, opt) + walkDirFunc = w.onError(walkDirFunc) + + // Walk the filesystem + if err := filepath.WalkDir(root, walkDirFunc); err != nil { return xerrors.Errorf("walk dir error: %w", err) } + return nil } -func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc { +func (w *FS) WalkDirFunc(root string, fn WalkFunc, opt Option) fs.WalkDirFunc { return func(filePath string, d fs.DirEntry, err error) error { if err != nil { return err } - time.Sleep(w.option.Delay) - - filePath = filepath.Clean(filePath) - // For exported rootfs (e.g. images/alpine/etc/alpine-release) relPath, err := filepath.Rel(root, filePath) if err != nil { @@ -78,28 +61,104 @@ func (w FS) walkDirFunc(root string, fn WalkFunc) fs.WalkDirFunc { return xerrors.Errorf("file info error: %w", err) } + // Skip unnecessary files switch { case info.IsDir(): - if w.shouldSkipDir(relPath) { + if SkipPath(relPath, opt.SkipDirs) { return filepath.SkipDir } return nil case !info.Mode().IsRegular(): return nil - case w.shouldSkipFile(relPath): + case SkipPath(relPath, opt.SkipFiles): return nil } - if err = fn(relPath, info, w.fileOpener(filePath)); err != nil { + if err = fn(relPath, info, fileOpener(filePath)); err != nil { return xerrors.Errorf("failed to analyze file: %w", err) } + + return nil + } +} + +func (w *FS) onError(wrapped fs.WalkDirFunc) fs.WalkDirFunc { + return func(filePath string, d fs.DirEntry, err error) error { + err = wrapped(filePath, d, err) + switch { + // Unwrap fs.SkipDir error + case errors.Is(err, fs.SkipDir): + return fs.SkipDir + // ignore permission errors + case os.IsPermission(err): + return nil + case err != nil: + // halt traversal on any other error + return xerrors.Errorf("unknown error with %s: %w", filePath, err) + } + return nil + } +} + +// BuildSkipPaths builds correct patch for defaultSkipDirs and skipFiles +func (w *FS) BuildSkipPaths(base string, paths []string) []string { + var relativePaths []string + absBase, err := filepath.Abs(base) + if err != nil { + log.Logger.Warnf("Failed to get an absolute path of %s: %s", base, err) return nil } + for _, path := range paths { + // Supports three types of flag specification. + // All of them are converted into the relative path from the root directory. + // 1. Relative skip dirs/files from the root directory + // The specified dirs and files will be used as is. + // e.g. $ trivy fs --skip-dirs bar ./foo + // The skip dir from the root directory will be `bar/`. + // 2. Relative skip dirs/files from the working directory + // The specified dirs and files wll be converted to the relative path from the root directory. + // e.g. $ trivy fs --skip-dirs ./foo/bar ./foo + // The skip dir will be converted to `bar/`. + // 3. Absolute skip dirs/files + // The specified dirs and files wll be converted to the relative path from the root directory. + // e.g. $ trivy fs --skip-dirs /bar/foo/baz ./foo + // When the working directory is + // 3.1 /bar: the skip dir will be converted to `baz/`. + // 3.2 /hoge : the skip dir will be converted to `../../bar/foo/baz/`. + + absSkipPath, err := filepath.Abs(path) + if err != nil { + log.Logger.Warnf("Failed to get an absolute path of %s: %s", base, err) + continue + } + rel, err := filepath.Rel(absBase, absSkipPath) + if err != nil { + log.Logger.Warnf("Failed to get a relative path from %s to %s: %s", base, path, err) + continue + } + + var relPath string + switch { + case !filepath.IsAbs(path) && strings.HasPrefix(rel, ".."): + // #1: Use the path as is + relPath = path + case !filepath.IsAbs(path) && !strings.HasPrefix(rel, ".."): + // #2: Use the relative path from the root directory + relPath = rel + case filepath.IsAbs(path): + // #3: Use the relative path from the root directory + relPath = rel + } + relPath = filepath.ToSlash(relPath) + relativePaths = append(relativePaths, relPath) + } + + relativePaths = CleanSkipPaths(relativePaths) + return relativePaths } -// fileOpener returns a function opening a file. -func (w *walker) fileOpener(pathname string) func() (dio.ReadSeekCloserAt, error) { +func fileOpener(filePath string) func() (dio.ReadSeekCloserAt, error) { return func() (dio.ReadSeekCloserAt, error) { - return os.Open(pathname) + return os.Open(filePath) } } diff --git a/pkg/fanal/walker/fs_test.go b/pkg/fanal/walker/fs_test.go index 280d8367b969..5dab0b872f2d 100644 --- a/pkg/fanal/walker/fs_test.go +++ b/pkg/fanal/walker/fs_test.go @@ -2,10 +2,11 @@ package walker_test import ( "errors" - "github.com/aquasecurity/trivy/pkg/custom" + "golang.org/x/exp/slices" "io" - "io/fs" "os" + "path/filepath" + "runtime" "strings" "testing" @@ -16,15 +17,10 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/walker" ) -func TestDir_Walk(t *testing.T) { - type fields struct { - skipFiles []string - skipDirs []string - option custom.Option - } +func TestFS_Walk(t *testing.T) { tests := []struct { name string - fields fields + option walker.Option rootDir string analyzeFn walker.WalkFunc wantErr string @@ -48,8 +44,8 @@ func TestDir_Walk(t *testing.T) { { name: "skip file", rootDir: "testdata/fs", - fields: fields{ - skipFiles: []string{"testdata/fs/bar"}, + option: walker.Option{ + SkipFiles: []string{"testdata/fs/bar"}, }, analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { if filePath == "testdata/fs/bar" { @@ -61,8 +57,8 @@ func TestDir_Walk(t *testing.T) { { name: "skip dir", rootDir: "testdata/fs/", - fields: fields{ - skipDirs: []string{"/testdata/fs/app"}, + option: walker.Option{ + SkipDirs: []string{"/testdata/fs/app"}, }, analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { if strings.HasPrefix(filePath, "testdata/fs/app") { @@ -71,37 +67,6 @@ func TestDir_Walk(t *testing.T) { return nil }, }, - { - name: "ignore all errors", - rootDir: "testdata/fs/nosuch", - fields: fields{ - option: custom.Option{ - ErrorCallback: func(pathname string, err error) error { - return nil - }, - }, - }, - analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { - return nil - }, - }, - { - name: "ignore analysis errors", - rootDir: "testdata/fs", - fields: fields{ - option: custom.Option{ - ErrorCallback: func(pathname string, err error) error { - if errors.Is(err, fs.ErrClosed) { - return nil - } - return err - }, - }, - }, - analyzeFn: func(string, os.FileInfo, analyzer.Opener) error { - return fs.ErrClosed - }, - }, { name: "sad path", rootDir: "testdata/fs", @@ -113,15 +78,115 @@ func TestDir_Walk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.option) + w := walker.NewFS() - err := w.Walk(tt.rootDir, tt.analyzeFn) + err := w.Walk(tt.rootDir, tt.option, tt.analyzeFn) if tt.wantErr != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) + assert.ErrorContains(t, err, tt.wantErr) return } assert.NoError(t, err) }) } } + +func TestFS_BuildSkipPaths(t *testing.T) { + tests := []struct { + name string + oses []string + paths []string + base string + want []string + }{ + // Linux/macOS + { + name: "path - abs, base - abs, not joining paths", + oses: []string{ + "linux", + "darwin", + }, + base: "/foo", + paths: []string{"/foo/bar"}, + want: []string{"bar"}, + }, + { + name: "path - abs, base - rel", + oses: []string{ + "linux", + "darwin", + }, + base: "foo", + paths: func() []string { + abs, err := filepath.Abs("foo/bar") + require.NoError(t, err) + return []string{abs} + }(), + want: []string{"bar"}, + }, + { + name: "path - rel, base - rel, joining paths", + oses: []string{ + "linux", + "darwin", + }, + base: "foo", + paths: []string{"bar"}, + want: []string{"bar"}, + }, + { + name: "path - rel, base - rel, not joining paths", + oses: []string{ + "linux", + "darwin", + }, + base: "foo", + paths: []string{"foo/bar/bar"}, + want: []string{"bar/bar"}, + }, + { + name: "path - rel with dot, base - rel, removing the leading dot and not joining paths", + oses: []string{ + "linux", + "darwin", + }, + base: "foo", + paths: []string{"./foo/bar"}, + want: []string{"bar"}, + }, + { + name: "path - rel, base - dot", + oses: []string{ + "linux", + "darwin", + }, + base: ".", + paths: []string{"foo/bar"}, + want: []string{"foo/bar"}, + }, + // Windows + { + name: "path - rel, base - rel. Skip common prefix", + oses: []string{"windows"}, + base: "foo", + paths: []string{"foo\\bar\\bar"}, + want: []string{"bar/bar"}, + }, + { + name: "path - rel, base - dot, windows", + oses: []string{"windows"}, + base: ".", + paths: []string{"foo\\bar"}, + want: []string{"foo/bar"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if !slices.Contains(tt.oses, runtime.GOOS) { + t.Skipf("Skip path tests for %q", tt.oses) + } + got := walker.NewFS().BuildSkipPaths(tt.base, tt.paths) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/fanal/walker/tar.go b/pkg/fanal/walker/tar.go index 0f0b1f2f694c..51f185d89c50 100644 --- a/pkg/fanal/walker/tar.go +++ b/pkg/fanal/walker/tar.go @@ -21,20 +21,19 @@ const ( var parentDir = ".." + utils.PathSeparator type LayerTar struct { - walker - threshold int64 + skipFiles []string + skipDirs []string } -func NewLayerTar(skipFiles, skipDirs []string) LayerTar { - threshold := sizeThreshold +func NewLayerTar(opt Option) LayerTar { return LayerTar{ - walker: newWalker(skipFiles, skipDirs), - threshold: threshold, + skipFiles: CleanSkipPaths(opt.SkipFiles), + skipDirs: CleanSkipPaths(opt.SkipDirs), } } func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, error) { - var opqDirs, whFiles, skipDirs []string + var opqDirs, whFiles, skippedDirs []string tr := tar.NewReader(layer) for { hdr, err := tr.Next() @@ -64,12 +63,12 @@ func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, switch hdr.Typeflag { case tar.TypeDir: - if w.shouldSkipDir(filePath) { - skipDirs = append(skipDirs, filePath) + if SkipPath(filePath, w.skipDirs) { + skippedDirs = append(skippedDirs, filePath) continue } case tar.TypeReg: - if w.shouldSkipFile(filePath) { + if SkipPath(filePath, w.skipFiles) { continue } // symlinks and hardlinks have no content in reader, skip them @@ -77,7 +76,7 @@ func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, continue } - if underSkippedDir(filePath, skipDirs) { + if underSkippedDir(filePath, skippedDirs) { continue } @@ -90,7 +89,7 @@ func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, } func (w LayerTar) processFile(filePath string, tr *tar.Reader, fi fs.FileInfo, analyzeFn WalkFunc) error { - cf := newCachedFile(fi.Size(), tr, w.threshold) + cf := newCachedFile(fi.Size(), tr) defer func() { // nolint _ = cf.Clean() diff --git a/pkg/fanal/walker/tar_test.go b/pkg/fanal/walker/tar_test.go index fed2f9efea7d..b7f0b01335e6 100644 --- a/pkg/fanal/walker/tar_test.go +++ b/pkg/fanal/walker/tar_test.go @@ -15,13 +15,9 @@ import ( ) func TestLayerTar_Walk(t *testing.T) { - type fields struct { - skipFiles []string - skipDirs []string - } tests := []struct { name string - fields fields + option walker.Option inputFile string analyzeFn walker.WalkFunc wantOpqDirs []string @@ -40,8 +36,8 @@ func TestLayerTar_Walk(t *testing.T) { { name: "skip file", inputFile: filepath.Join("testdata", "test.tar"), - fields: fields{ - skipFiles: []string{"/app/myweb/index.html"}, + option: walker.Option{ + SkipFiles: []string{"/app/myweb/index.html"}, }, analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { if filePath == "app/myweb/index.html" { @@ -55,8 +51,8 @@ func TestLayerTar_Walk(t *testing.T) { { name: "skip dir", inputFile: filepath.Join("testdata", "test.tar"), - fields: fields{ - skipDirs: []string{"/app"}, + option: walker.Option{ + SkipDirs: []string{"/app"}, }, analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { if strings.HasPrefix(filePath, "app") { @@ -81,7 +77,7 @@ func TestLayerTar_Walk(t *testing.T) { f, err := os.Open("testdata/test.tar") require.NoError(t, err) - w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs) + w := walker.NewLayerTar(tt.option) gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn) if tt.wantErr != "" { diff --git a/pkg/fanal/walker/vm.go b/pkg/fanal/walker/vm.go index 6b728c2c99b9..1cc34334d4fb 100644 --- a/pkg/fanal/walker/vm.go +++ b/pkg/fanal/walker/vm.go @@ -34,16 +34,15 @@ func AppendPermitDiskName(s ...string) { } type VM struct { - walker - threshold int64 + skipFiles []string + skipDirs []string analyzeFn WalkFunc } -func NewVM(skipFiles, skipDirs []string) VM { - threshold := sizeThreshold +func NewVM(opt Option) VM { return VM{ - walker: newWalker(skipFiles, skipDirs), - threshold: threshold, + skipFiles: CleanSkipPaths(opt.SkipFiles), + skipDirs: CleanSkipPaths(opt.SkipDirs), } } @@ -121,13 +120,13 @@ func (w *VM) fsWalk(fsys fs.FS, path string, d fs.DirEntry, err error) error { pathName := strings.TrimPrefix(filepath.Clean(path), "/") switch { case fi.IsDir(): - if w.shouldSkipDir(pathName) { + if SkipPath(pathName, w.skipDirs) { return filepath.SkipDir } return nil case !fi.Mode().IsRegular(): return nil - case w.shouldSkipFile(pathName): + case SkipPath(pathName, w.skipFiles): return nil case fi.Mode()&0x1000 == 0x1000 || fi.Mode()&0x2000 == 0x2000 || @@ -142,7 +141,7 @@ func (w *VM) fsWalk(fsys fs.FS, path string, d fs.DirEntry, err error) error { return nil } - cvf := newCachedVMFile(fsys, pathName, w.threshold) + cvf := newCachedVMFile(fsys, pathName) defer cvf.Clean() if err = w.analyzeFn(path, fi, cvf.Open); err != nil { @@ -152,18 +151,16 @@ func (w *VM) fsWalk(fsys fs.FS, path string, d fs.DirEntry, err error) error { } type cachedVMFile struct { - fs fs.FS - filePath string - threshold int64 + fs fs.FS + filePath string cf *cachedFile } -func newCachedVMFile(fsys fs.FS, filePath string, threshold int64) *cachedVMFile { +func newCachedVMFile(fsys fs.FS, filePath string) *cachedVMFile { return &cachedVMFile{ - fs: fsys, - filePath: filePath, - threshold: threshold, + fs: fsys, + filePath: filePath, } } @@ -181,7 +178,7 @@ func (cvf *cachedVMFile) Open() (dio.ReadSeekCloserAt, error) { return nil, xerrors.Errorf("file stat error: %w", err) } - cvf.cf = newCachedFile(fi.Size(), f, cvf.threshold) + cvf.cf = newCachedFile(fi.Size(), f) return cvf.cf.Open() } diff --git a/pkg/fanal/walker/walk.go b/pkg/fanal/walker/walk.go index 1f098da5b4f6..53013f6d8ac7 100644 --- a/pkg/fanal/walker/walk.go +++ b/pkg/fanal/walker/walk.go @@ -6,87 +6,47 @@ import ( "strings" "github.com/bmatcuk/doublestar/v4" + "github.com/samber/lo" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" - "github.com/aquasecurity/trivy/pkg/fanal/utils" "github.com/aquasecurity/trivy/pkg/log" ) -var ( - // These variables are exported so that a tool importing Trivy as a library can override these values. - AppDirs = []string{".git"} - SystemDirs = []string{ - "proc", - "sys", - "dev", - } -) - -const ( - sizeThreshold = int64(200) << 20 // 200MB -) - -type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error +const defaultSizeThreshold = int64(100) << 20 // 200MB -type walker struct { - skipFiles []string - skipDirs []string +var defaultSkipDirs = []string{ + "**/.git", + "proc", + "sys", + "dev", } -func newWalker(skipFiles, skipDirs []string) walker { - var cleanSkipFiles, cleanSkipDirs []string - for _, skipFile := range skipFiles { - skipFile = filepath.ToSlash(filepath.Clean(skipFile)) - skipFile = strings.TrimLeft(skipFile, "/") - cleanSkipFiles = append(cleanSkipFiles, skipFile) - } +type Option struct { + SkipFiles []string + SkipDirs []string +} - for _, skipDir := range append(skipDirs, SystemDirs...) { - skipDir = filepath.ToSlash(filepath.Clean(skipDir)) - skipDir = strings.TrimLeft(skipDir, "/") - cleanSkipDirs = append(cleanSkipDirs, skipDir) - } +type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error - return walker{ - skipFiles: cleanSkipFiles, - skipDirs: cleanSkipDirs, - } +func CleanSkipPaths(skipPaths []string) []string { + return lo.Map(skipPaths, func(skipPath string, index int) string { + skipPath = filepath.ToSlash(filepath.Clean(skipPath)) + return strings.TrimLeft(skipPath, "/") + }) } -func (w *walker) shouldSkipFile(filePath string) bool { - filePath = strings.TrimLeft(filePath, "/") +func SkipPath(path string, skipPaths []string) bool { + path = strings.TrimLeft(path, "/") // skip files - for _, pattern := range w.skipFiles { - match, err := doublestar.Match(pattern, filePath) + for _, pattern := range skipPaths { + match, err := doublestar.Match(pattern, path) if err != nil { return false // return early if bad pattern } else if match { - log.Logger.Debugf("Skipping file: %s", filePath) + log.Logger.Debugf("Skipping path: %s", path) return true } } return false } - -func (w *walker) shouldSkipDir(dir string) bool { - dir = strings.TrimLeft(dir, "/") - - // Skip application dirs (relative path) - base := filepath.Base(dir) - if utils.StringInSlice(base, AppDirs) { - return true - } - - // Skip system dirs and specified dirs (absolute path) - for _, pattern := range w.skipDirs { - if match, err := doublestar.Match(pattern, dir); err != nil { - return false // return early if bad pattern - } else if match { - log.Logger.Debugf("Skipping directory: %s", dir) - return true - } - } - - return false -} diff --git a/pkg/fanal/walker/walk_test.go b/pkg/fanal/walker/walk_test.go index 4d52117aa1ce..765f5c07d7b3 100644 --- a/pkg/fanal/walker/walk_test.go +++ b/pkg/fanal/walker/walk_test.go @@ -1,123 +1,144 @@ -package walker +package walker_test import ( "fmt" + "github.com/aquasecurity/trivy/pkg/fanal/walker" "path/filepath" "testing" "github.com/stretchr/testify/assert" ) -func Test_shouldSkipFile(t *testing.T) { - testCases := []struct { +func TestSkipFile(t *testing.T) { + tests := []struct { + name string skipFiles []string - skipMap map[string]bool + wants map[string]bool }{ { - skipFiles: []string{filepath.Join("/etc/*")}, - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): true, - filepath.Join("/etc/foo/bar"): false, + name: "single star", + skipFiles: []string{"/etc/*"}, + wants: map[string]bool{ + "/etc/foo": true, + "/etc/foo/bar": false, }, }, { - skipFiles: []string{filepath.Join("/etc/*/*")}, - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): false, - filepath.Join("/etc/foo/bar"): true, + name: "two stars", + skipFiles: []string{"/etc/*/*"}, + wants: map[string]bool{ + "/etc/foo": false, + "/etc/foo/bar": true, }, }, { - skipFiles: []string{filepath.Join("**/*.txt")}, - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): false, - filepath.Join("/etc/foo/bar"): false, - filepath.Join("/var/log/bar.txt"): true, + name: "double star", + skipFiles: []string{"**/*.txt"}, + wants: map[string]bool{ + "/etc/foo": false, + "/etc/foo/bar": false, + "/var/log/bar.txt": true, }, }, { - skipFiles: []string{filepath.Join("/etc/*/*"), filepath.Join("/var/log/*.txt")}, - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): false, - filepath.Join("/etc/foo/bar"): true, - filepath.Join("/var/log/bar.txt"): true, + name: "multiple skip files", + skipFiles: []string{"/etc/*/*", "/var/log/*.txt"}, + wants: map[string]bool{ + "/etc/foo": false, + "/etc/foo/bar": true, + "/var/log/bar.txt": true, }, }, { - skipFiles: []string{filepath.Join(`[^etc`)}, // filepath.Match returns ErrBadPattern - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): false, - filepath.Join("/etc/foo/bar"): false, + name: "error bad pattern", + skipFiles: []string{`[^etc`}, // filepath.Match returns ErrBadPattern + wants: map[string]bool{ + "/etc/foo": false, + "/etc/foo/bar": false, }, }, } - for i, tc := range testCases { - t.Run(fmt.Sprint(i), func(t *testing.T) { - w := newWalker(tc.skipFiles, nil) - for file, skipResult := range tc.skipMap { - assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for file, want := range tt.wants { + file = filepath.ToSlash(filepath.Clean(file)) + got := walker.SkipPath(file, walker.CleanSkipPaths(tt.skipFiles)) + assert.Equal(t, want, got, fmt.Sprintf("skipFiles: %s, file: %s", tt.skipFiles, file)) } }) } } -func Test_shouldSkipDir(t *testing.T) { - testCases := []struct { +func TestSkipDir(t *testing.T) { + tests := []struct { + name string skipDirs []string - skipMap map[string]bool + wants map[string]bool }{ { - skipDirs: nil, - skipMap: map[string]bool{ - ".git": true, // AppDir - "proc": true, // SystemDir - "foo.bar": false, // random directory + name: "default skip dirs", + skipDirs: []string{ + "**/.git", + "proc", + "sys", + "dev", + }, + wants: map[string]bool{ + ".git": true, + "proc": true, + "foo.bar": false, }, }, { - skipDirs: []string{filepath.Join("/*")}, - skipMap: map[string]bool{ - filepath.Join("/etc"): true, - filepath.Join("/etc/foo/bar"): false, + name: "single star", + skipDirs: []string{"/*"}, + wants: map[string]bool{ + "/etc": true, + "/etc/foo/bar": false, }, }, { + name: "two stars", skipDirs: []string{filepath.Join("/etc/*/*")}, - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): false, - filepath.Join("/etc/foo/bar"): true, + wants: map[string]bool{ + "/etc/foo": false, + "/etc/foo/bar": true, }, }, { + name: "multiple dirs", skipDirs: []string{filepath.Join("/etc/*/*"), filepath.Join("/var/log/*")}, - skipMap: map[string]bool{ + wants: map[string]bool{ filepath.Join("/etc/foo"): false, filepath.Join("/etc/foo/bar"): true, filepath.Join("/var/log/bar"): true, }, }, { - skipDirs: []string{filepath.Join(`[^etc`)}, // filepath.Match returns ErrBadPattern - skipMap: map[string]bool{ - filepath.Join("/etc/foo"): false, - filepath.Join("/etc/foo/bar"): false, - }, - }, - { + name: "double star", skipDirs: []string{"**/.terraform"}, - skipMap: map[string]bool{ + wants: map[string]bool{ ".terraform": true, "test/foo/bar/.terraform": true, }, }, + { + name: "error bad pattern", + skipDirs: []string{filepath.Join(`[^etc`)}, // filepath.Match returns ErrBadPattern + wants: map[string]bool{ + filepath.Join("/etc/foo"): false, + filepath.Join("/etc/foo/bar"): false, + }, + }, } - for i, tc := range testCases { - t.Run(fmt.Sprint(i), func(t *testing.T) { - w := newWalker(nil, tc.skipDirs) - for dir, skipResult := range tc.skipMap { - assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for dir, want := range tt.wants { + dir = filepath.ToSlash(filepath.Clean(dir)) + got := walker.SkipPath(dir, walker.CleanSkipPaths(tt.skipDirs)) + assert.Equal(t, want, got, fmt.Sprintf("defaultSkipDirs: %s, dir: %s", tt.skipDirs, dir)) } }) } diff --git a/pkg/flag/scan_flags.go b/pkg/flag/scan_flags.go index 3323e110fec4..67694810d1c0 100644 --- a/pkg/flag/scan_flags.go +++ b/pkg/flag/scan_flags.go @@ -1,8 +1,6 @@ package flag import ( - "golang.org/x/xerrors" - "github.com/aquasecurity/trivy/pkg/types" xstrings "github.com/aquasecurity/trivy/pkg/x/strings" ) @@ -84,7 +82,7 @@ var ( Name: "parallel", ConfigName: "scan.parallel", Default: 5, - Usage: "number (1-20) of goroutines enabled for parallel scanning", + Usage: "number of goroutines enabled for parallel scanning", } ) @@ -154,12 +152,6 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) { target = args[0] } - parallel := getInt(f.Parallel) - // check parallel flag is a valid number between 1-20 - if parallel < 1 || parallel > 20 { - return ScanOptions{}, xerrors.Errorf("'--parallel' must be a number between 1-20: %d", parallel) - } - return ScanOptions{ Target: target, SkipDirs: getStringSlice(f.SkipDirs), @@ -167,7 +159,7 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) { OfflineScan: getBool(f.OfflineScan), Scanners: getUnderlyingStringSlice[types.Scanner](f.Scanners), FilePatterns: getStringSlice(f.FilePatterns), - Parallel: parallel, + Parallel: getInt(f.Parallel), SBOMSources: getStringSlice(f.SBOMSources), RekorURL: getString(f.RekorURL), IncludeDevDeps: getBool(f.IncludeDevDeps), diff --git a/pkg/parallel/walk.go b/pkg/parallel/walk.go index 2ba79c428990..c6ef4bbd8eea 100644 --- a/pkg/parallel/walk.go +++ b/pkg/parallel/walk.go @@ -17,6 +17,9 @@ type onWalkResult[T any] func(T) error func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int, onFile onFile[T], onResult onWalkResult[T]) error { + if parallel == 0 { + parallel = 5 // Set the default value + } g, ctx := errgroup.WithContext(ctx) paths := make(chan string) diff --git a/pkg/scanner/scan.go b/pkg/scanner/scan.go index 931b5b0ea035..6d091484739f 100644 --- a/pkg/scanner/scan.go +++ b/pkg/scanner/scan.go @@ -48,13 +48,13 @@ var StandaloneArchiveSet = wire.NewSet( // StandaloneFilesystemSet binds filesystem dependencies var StandaloneFilesystemSet = wire.NewSet( - flocal.NewArtifact, + flocal.ArtifactSet, StandaloneSuperSet, ) // StandaloneRepositorySet binds repository dependencies var StandaloneRepositorySet = wire.NewSet( - repo.NewArtifact, + repo.ArtifactSet, StandaloneSuperSet, ) @@ -84,13 +84,13 @@ var RemoteSuperSet = wire.NewSet( // RemoteFilesystemSet binds filesystem dependencies for client/server mode var RemoteFilesystemSet = wire.NewSet( - flocal.NewArtifact, + flocal.ArtifactSet, RemoteSuperSet, ) // RemoteRepositorySet binds repository dependencies for client/server mode var RemoteRepositorySet = wire.NewSet( - repo.NewArtifact, + repo.ArtifactSet, RemoteSuperSet, ) From c05898ae1102cdb129e710fe7a9d810e264673e6 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 9 Nov 2023 23:40:01 +0900 Subject: [PATCH 08/23] docs: auto generate Signed-off-by: knqyf263 --- docs/docs/references/configuration/cli/trivy.md | 1 + docs/docs/references/configuration/cli/trivy_aws.md | 2 +- docs/docs/references/configuration/cli/trivy_filesystem.md | 2 +- docs/docs/references/configuration/cli/trivy_image.md | 2 +- docs/docs/references/configuration/cli/trivy_kubernetes.md | 3 +-- docs/docs/references/configuration/cli/trivy_repository.md | 2 +- docs/docs/references/configuration/cli/trivy_rootfs.md | 2 +- docs/docs/references/configuration/cli/trivy_sbom.md | 2 +- docs/docs/references/configuration/cli/trivy_vm.md | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs/references/configuration/cli/trivy.md b/docs/docs/references/configuration/cli/trivy.md index f11635a25992..c51752d03d2f 100644 --- a/docs/docs/references/configuration/cli/trivy.md +++ b/docs/docs/references/configuration/cli/trivy.md @@ -51,6 +51,7 @@ trivy [global flags] command [flags] target * [trivy kubernetes](trivy_kubernetes.md) - [EXPERIMENTAL] Scan kubernetes cluster * [trivy module](trivy_module.md) - Manage modules * [trivy plugin](trivy_plugin.md) - Manage plugins +* [trivy referrer](trivy_referrer.md) - Put referrers to OCI registry * [trivy repository](trivy_repository.md) - Scan a repository * [trivy rootfs](trivy_rootfs.md) - Scan rootfs * [trivy sbom](trivy_sbom.md) - Scan SBOM for vulnerabilities diff --git a/docs/docs/references/configuration/cli/trivy_aws.md b/docs/docs/references/configuration/cli/trivy_aws.md index e7e5651c91d4..2daf238b3fff 100644 --- a/docs/docs/references/configuration/cli/trivy_aws.md +++ b/docs/docs/references/configuration/cli/trivy_aws.md @@ -34,11 +34,11 @@ The following services are supported: - neptune - rds - redshift -- s3 - sns - sqs - ssm - workspaces +- s3 ``` diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index bfeb6302ce40..67d445978224 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -54,6 +54,7 @@ trivy filesystem [flags] PATH --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0") --policy-namespaces strings Rego namespaces @@ -76,7 +77,6 @@ trivy filesystem [flags] PATH --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index dd96a620e634..7592ebf3f7af 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -73,6 +73,7 @@ trivy image [flags] IMAGE_NAME --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --platform string set platform in the form os/arch if image is multi-platform capable --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0") @@ -97,7 +98,6 @@ trivy image [flags] IMAGE_NAME --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index 3fd04a287b69..d4d676552891 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -65,7 +65,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg: --node-collector-namespace string specify the namespace in which the node-collector job should be deployed (default "trivy-temp") --offline-scan do not issue API requests to identify dependencies -o, --output string output file name - --parallel int number (between 1-20) of goroutines enabled for parallel scanning (default 5) + --parallel int number of goroutines enabled for parallel scanning (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0") --policy-namespaces strings Rego namespaces @@ -87,7 +87,6 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg: --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index 984a40a28af6..2a44a4b2f199 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -54,6 +54,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0") --policy-namespaces strings Rego namespaces @@ -75,7 +76,6 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization --tag string pass the tag name to be scanned -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index bb007e1210f6..246c6ea65b64 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -56,6 +56,7 @@ trivy rootfs [flags] ROOTDIR --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0") --policy-namespaces strings Rego namespaces @@ -77,7 +78,6 @@ trivy rootfs [flags] ROOTDIR --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index d031f61e4cff..78583f7d442c 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -42,6 +42,7 @@ trivy sbom [flags] SBOM_PATH --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning (default 5) --redis-ca string redis ca file location, if using redis as cache backend --redis-cert string redis certificate file location, if using redis as cache backend --redis-key string redis key file location, if using redis as cache backend @@ -55,7 +56,6 @@ trivy sbom [flags] SBOM_PATH --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database - --slow scan over time with lower CPU and memory utilization -t, --template string output template --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index 2a2067c2c0df..a255737b5b76 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -51,6 +51,7 @@ trivy vm [flags] VM_IMAGE --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning (default 5) --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0") --redis-ca string redis ca file location, if using redis as cache backend --redis-cert string redis certificate file location, if using redis as cache backend @@ -68,7 +69,6 @@ trivy vm [flags] VM_IMAGE --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files From acf36ba67f14d6eb47e20a5f3cbd54d3534a540c Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 10 Nov 2023 10:07:31 +0900 Subject: [PATCH 09/23] docs: remove plugin reference Signed-off-by: knqyf263 --- .../configuration/cli/trivy_referrer.md | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 docs/docs/references/configuration/cli/trivy_referrer.md diff --git a/docs/docs/references/configuration/cli/trivy_referrer.md b/docs/docs/references/configuration/cli/trivy_referrer.md deleted file mode 100644 index 455eea57afdc..000000000000 --- a/docs/docs/references/configuration/cli/trivy_referrer.md +++ /dev/null @@ -1,31 +0,0 @@ -## trivy referrer - -Put referrers to OCI registry - -``` -trivy referrer [flags] -``` - -### Options - -``` - -h, --help help for referrer -``` - -### Options inherited from parent commands - -``` - --cache-dir string cache directory (default "/path/to/cache") - -c, --config string config path (default "trivy.yaml") - -d, --debug debug mode - --generate-default-config write the default config to trivy-default.yaml - --insecure allow insecure server connections - -q, --quiet suppress progress bar and log output - --timeout duration timeout (default 5m0s) - -v, --version show version -``` - -### SEE ALSO - -* [trivy](trivy.md) - Unified security scanner - From 031117fe739af85644793be1649cc375bef3a143 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 10 Nov 2023 11:01:30 +0900 Subject: [PATCH 10/23] chore(magefile): not load plugins for doc Signed-off-by: knqyf263 --- magefiles/docs.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/magefiles/docs.go b/magefiles/docs.go index 37d04343ecac..b69e813690af 100644 --- a/magefiles/docs.go +++ b/magefiles/docs.go @@ -3,6 +3,8 @@ package main import ( + "os" + "github.com/spf13/cobra/doc" "github.com/aquasecurity/trivy/pkg/commands" @@ -16,6 +18,9 @@ func main() { flag.CacheDirFlag.Default = "/path/to/cache" flag.ModuleDirFlag.Default = "$HOME/.trivy/modules" + // Set a dummy path not to load plugins + os.Setenv("XDG_DATA_HOME", os.TempDir()) + cmd := commands.NewApp() cmd.DisableAutoGenTag = true if err := doc.GenMarkdownTree(cmd, "./docs/docs/references/configuration/cli"); err != nil { From 87b903cebadace9ef4f7897a7a911eff44609220 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 10 Nov 2023 11:02:37 +0900 Subject: [PATCH 11/23] docs: update Signed-off-by: knqyf263 --- docs/docs/references/configuration/cli/trivy.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/docs/references/configuration/cli/trivy.md b/docs/docs/references/configuration/cli/trivy.md index c51752d03d2f..f11635a25992 100644 --- a/docs/docs/references/configuration/cli/trivy.md +++ b/docs/docs/references/configuration/cli/trivy.md @@ -51,7 +51,6 @@ trivy [global flags] command [flags] target * [trivy kubernetes](trivy_kubernetes.md) - [EXPERIMENTAL] Scan kubernetes cluster * [trivy module](trivy_module.md) - Manage modules * [trivy plugin](trivy_plugin.md) - Manage plugins -* [trivy referrer](trivy_referrer.md) - Put referrers to OCI registry * [trivy repository](trivy_repository.md) - Scan a repository * [trivy rootfs](trivy_rootfs.md) - Scan rootfs * [trivy sbom](trivy_sbom.md) - Scan SBOM for vulnerabilities From 3031612785f50ec689618e561b44f15ab981c7c4 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 10 Nov 2023 13:49:14 +0900 Subject: [PATCH 12/23] fix: pass walker options Signed-off-by: knqyf263 --- pkg/commands/artifact/run.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index a66d4bf6f5fd..bc90ac0db54d 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -73,9 +73,6 @@ type ScannerConfig struct { // Artifact options ArtifactOption artifact.Option - - // Walk options - WalkerOption walker.Option } type Runner interface { @@ -666,10 +663,12 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi Full: opts.LicenseFull, ClassifierConfidenceLevel: opts.LicenseConfidenceLevel, }, - }, - WalkerOption: walker.Option{ - SkipFiles: opts.SkipFiles, - SkipDirs: opts.SkipDirs, + + // For file walking + WalkerOption: walker.Option{ + SkipFiles: opts.SkipFiles, + SkipDirs: opts.SkipDirs, + }, }, }, scanOptions, nil } From 487e2ab4be067fb74ec80d0cfb754a6f89d6c079 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 10 Nov 2023 13:49:24 +0900 Subject: [PATCH 13/23] chore: show diff Signed-off-by: knqyf263 --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b000f9dace39..a1f8c59400b6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,6 +56,7 @@ jobs: run: | mage docs:generate if [ -n "$(git status --porcelain)" ]; then + git status echo "Run 'mage docs:generate' and push it" exit 1 fi From 42e2cb144d7fd9fc8f3b5f2222a45800f4ae5f08 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 10 Nov 2023 15:10:35 +0900 Subject: [PATCH 14/23] docs: sort aws services Signed-off-by: knqyf263 --- docs/docs/references/configuration/cli/trivy_aws.md | 4 ++-- pkg/commands/app.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docs/references/configuration/cli/trivy_aws.md b/docs/docs/references/configuration/cli/trivy_aws.md index 5bdca5268519..eb7e5a144ac1 100644 --- a/docs/docs/references/configuration/cli/trivy_aws.md +++ b/docs/docs/references/configuration/cli/trivy_aws.md @@ -26,6 +26,7 @@ The following services are supported: - elb - emr - iam +- kinesis - kms - lambda - mq @@ -33,12 +34,11 @@ The following services are supported: - neptune - rds - redshift +- s3 - sns - sqs - ssm - workspaces -- kinesis -- s3 ``` diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 36dc8be3d38a..4ed06963f851 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "sort" "strings" "time" @@ -971,6 +972,7 @@ func NewAWSCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { } services := awsScanner.AllSupportedServices() + sort.Strings(services) cmd := &cobra.Command{ Use: "aws [flags]", From 1918ac99aa50676130ad04ef53ba482f0196ac99 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 16 Nov 2023 13:56:22 +0900 Subject: [PATCH 15/23] fix: lint fixes Signed-off-by: knqyf263 --- pkg/fanal/artifact/image/image.go | 2 +- pkg/fanal/artifact/local/fs.go | 2 +- pkg/fanal/artifact/vm/vm.go | 4 ++-- pkg/parallel/walk.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index 5c1d3ae62ef3..dcae1f19aee7 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -3,7 +3,6 @@ package image import ( "context" "errors" - "github.com/aquasecurity/trivy/pkg/semaphore" "io" "os" "reflect" @@ -24,6 +23,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/parallel" + "github.com/aquasecurity/trivy/pkg/semaphore" ) type Artifact struct { diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index 2416e0375779..5896196c0e48 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/json" - "github.com/aquasecurity/trivy/pkg/semaphore" "os" "path" "path/filepath" @@ -21,6 +20,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/handler" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" + "github.com/aquasecurity/trivy/pkg/semaphore" ) var ( diff --git a/pkg/fanal/artifact/vm/vm.go b/pkg/fanal/artifact/vm/vm.go index 415fa4799e50..8228a91fe41e 100644 --- a/pkg/fanal/artifact/vm/vm.go +++ b/pkg/fanal/artifact/vm/vm.go @@ -2,13 +2,12 @@ package vm import ( "context" - semaphore "github.com/aquasecurity/trivy/pkg/semaphore" - "github.com/google/wire" "io" "os" "strings" "sync" + "github.com/google/wire" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" @@ -17,6 +16,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/handler" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" + semaphore "github.com/aquasecurity/trivy/pkg/semaphore" ) type Type string diff --git a/pkg/parallel/walk.go b/pkg/parallel/walk.go index 0286c9749455..3f3291b01238 100644 --- a/pkg/parallel/walk.go +++ b/pkg/parallel/walk.go @@ -20,7 +20,7 @@ type onWalkResult[T any] func(T) error func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int, onFile onFile[T], onResult onWalkResult[T]) error { if parallel == 0 { - parallel = 5 // Set the default value + parallel = defaultParallel // Set the default value } g, ctx := errgroup.WithContext(ctx) From b7d9f647cd461f28b057e302cd9324ec8badbded Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 16 Nov 2023 14:14:11 +0900 Subject: [PATCH 16/23] test: fix mock signature Signed-off-by: knqyf263 --- pkg/fanal/artifact/vm/vm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/artifact/vm/vm_test.go b/pkg/fanal/artifact/vm/vm_test.go index c60fdb430b1e..6a932dc54f82 100644 --- a/pkg/fanal/artifact/vm/vm_test.go +++ b/pkg/fanal/artifact/vm/vm_test.go @@ -36,7 +36,7 @@ type mockWalker struct { root string } -func (m *mockWalker) Walk(_ *io.SectionReader, _ string, fn walker.WalkFunc) error { +func (m *mockWalker) Walk(_ *io.SectionReader, _ string, _ walker.Option, fn walker.WalkFunc) error { return filepath.WalkDir(m.root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err From ae0399ca95ee78b6959396dbd0448581d8f63d70 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Thu, 16 Nov 2023 14:49:00 +0900 Subject: [PATCH 17/23] refactor: remove option.Init() Signed-off-by: knqyf263 --- pkg/fanal/artifact/artifact.go | 6 ------ pkg/fanal/artifact/sbom/sbom.go | 1 - 2 files changed, 7 deletions(-) diff --git a/pkg/fanal/artifact/artifact.go b/pkg/fanal/artifact/artifact.go index 41f22afedd26..b8245f585389 100644 --- a/pkg/fanal/artifact/artifact.go +++ b/pkg/fanal/artifact/artifact.go @@ -41,12 +41,6 @@ type Option struct { WalkerOption walker.Option } -func (o *Option) Init() { - if o.Parallel == 0 { - o.Parallel = 5 // Set the default value - } -} - func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions { return analyzer.AnalyzerOptions{ Group: o.AnalyzerGroup, diff --git a/pkg/fanal/artifact/sbom/sbom.go b/pkg/fanal/artifact/sbom/sbom.go index 7981e6e793d9..e788e14b48db 100644 --- a/pkg/fanal/artifact/sbom/sbom.go +++ b/pkg/fanal/artifact/sbom/sbom.go @@ -29,7 +29,6 @@ type Artifact struct { } func NewArtifact(filePath string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) { - opt.Init() return Artifact{ filePath: filepath.Clean(filePath), cache: c, From 61eddabf40cad2b1bf383d2ed4dd015a77bd9d8d Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 16 Apr 2024 08:31:43 +0400 Subject: [PATCH 18/23] chore: replace run.skip-* with issues.exclude-* Signed-off-by: knqyf263 --- .golangci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 5f959a4cc339..8a4d089f9524 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -89,15 +89,15 @@ linters: run: go: '1.22' - skip-files: + +issues: + exclude-files: - ".*_mock.go$" - ".*_test.go$" - "integration/*" - "examples/*" - skip-dirs: + exclude-dirs: - "pkg/iac/scanners/terraform/parser/funcs" # copies of Terraform functions - -issues: exclude-rules: - linters: - gosec From 025e73f586028786ef28dbb26e45fcc695a9285e Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 16 Apr 2024 08:36:27 +0400 Subject: [PATCH 19/23] fix: linter issues Signed-off-by: knqyf263 --- pkg/fanal/analyzer/pkg/dpkg/copyright.go | 2 +- pkg/fanal/walker/fs.go | 2 +- pkg/flag/report_flags.go | 2 +- pkg/iac/rules/register.go | 2 +- pkg/iac/scanners/azure/functions/first.go | 2 +- pkg/iac/scanners/azure/functions/last.go | 2 +- pkg/k8s/commands/namespace.go | 2 +- pkg/mapfs/fs.go | 2 +- pkg/report/table/vulnerability.go | 2 +- pkg/sbom/cyclonedx/marshal.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/fanal/analyzer/pkg/dpkg/copyright.go b/pkg/fanal/analyzer/pkg/dpkg/copyright.go index 193fd5efe8d6..1f50088f86b7 100644 --- a/pkg/fanal/analyzer/pkg/dpkg/copyright.go +++ b/pkg/fanal/analyzer/pkg/dpkg/copyright.go @@ -88,7 +88,7 @@ func (a *dpkgLicenseAnalyzer) parseCopyright(r xio.ReadSeekerAt) ([]types.Licens l := strings.TrimSpace(line[8:]) l = normalizeLicense(l) - if len(l) > 0 { + if l != "" { for _, lic := range licensing.SplitLicenses(l) { lic = licensing.Normalize(lic) if !slices.Contains(licenses, lic) { diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index 54ab3e977a2b..8255585192d9 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -2,7 +2,6 @@ package walker import ( "errors" - xio "github.com/aquasecurity/trivy/pkg/x/io" "io/fs" "os" "path/filepath" @@ -11,6 +10,7 @@ import ( "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/log" + xio "github.com/aquasecurity/trivy/pkg/x/io" ) // FS is the filesystem walker diff --git a/pkg/flag/report_flags.go b/pkg/flag/report_flags.go index c079e6bb256a..852710248972 100644 --- a/pkg/flag/report_flags.go +++ b/pkg/flag/report_flags.go @@ -261,7 +261,7 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) { } func loadComplianceTypes(compliance string) (spec.ComplianceSpec, error) { - if len(compliance) > 0 && !slices.Contains(types.SupportedCompliances, compliance) && !strings.HasPrefix(compliance, "@") { + if compliance != "" && !slices.Contains(types.SupportedCompliances, compliance) && !strings.HasPrefix(compliance, "@") { return spec.ComplianceSpec{}, xerrors.Errorf("unknown compliance : %v", compliance) } diff --git a/pkg/iac/rules/register.go b/pkg/iac/rules/register.go index f34cdf904da3..0170a8d63979 100755 --- a/pkg/iac/rules/register.go +++ b/pkg/iac/rules/register.go @@ -125,7 +125,7 @@ func GetFrameworkRules(fw ...framework.Framework) []ruleTypes.RegisteredRule { } func GetSpecRules(spec string) []ruleTypes.RegisteredRule { - if len(spec) > 0 { + if spec != "" { return coreRegistry.getSpecRules(spec) } diff --git a/pkg/iac/scanners/azure/functions/first.go b/pkg/iac/scanners/azure/functions/first.go index 3415b453ffe3..91e2aece8e26 100644 --- a/pkg/iac/scanners/azure/functions/first.go +++ b/pkg/iac/scanners/azure/functions/first.go @@ -9,7 +9,7 @@ func First(args ...interface{}) interface{} { switch cType := container.(type) { case string: - if len(cType) > 0 { + if cType != "" { return string(cType[0]) } case interface{}: diff --git a/pkg/iac/scanners/azure/functions/last.go b/pkg/iac/scanners/azure/functions/last.go index 8466ec6b669f..84f54fb335fa 100644 --- a/pkg/iac/scanners/azure/functions/last.go +++ b/pkg/iac/scanners/azure/functions/last.go @@ -9,7 +9,7 @@ func Last(args ...interface{}) interface{} { switch cType := container.(type) { case string: - if len(cType) > 0 { + if cType != "" { return string(cType[len(cType)-1]) } case interface{}: diff --git a/pkg/k8s/commands/namespace.go b/pkg/k8s/commands/namespace.go index 6d828d4efdf9..b6c85e8dae00 100644 --- a/pkg/k8s/commands/namespace.go +++ b/pkg/k8s/commands/namespace.go @@ -36,7 +36,7 @@ func namespaceRun(ctx context.Context, opts flag.Options, cluster k8s.Cluster) e } func getNamespace(opts flag.Options, currentNamespace string) string { - if len(opts.K8sOptions.Namespace) > 0 { + if opts.K8sOptions.Namespace != "" { return opts.K8sOptions.Namespace } diff --git a/pkg/mapfs/fs.go b/pkg/mapfs/fs.go index 471730cc533e..c809f394b6fd 100644 --- a/pkg/mapfs/fs.go +++ b/pkg/mapfs/fs.go @@ -235,7 +235,7 @@ func (m *FS) RemoveAll(path string) error { func cleanPath(path string) string { // Convert the volume name like 'C:' into dir like 'C\' - if vol := filepath.VolumeName(path); len(vol) > 0 { + if vol := filepath.VolumeName(path); vol != "" { newVol := strings.TrimSuffix(vol, ":") newVol = fmt.Sprintf("%s%c", newVol, filepath.Separator) path = strings.Replace(path, vol, newVol, 1) diff --git a/pkg/report/table/vulnerability.go b/pkg/report/table/vulnerability.go index 9b85a1f4aab1..1984838d3709 100644 --- a/pkg/report/table/vulnerability.go +++ b/pkg/report/table/vulnerability.go @@ -124,7 +124,7 @@ func (r *vulnerabilityRenderer) setVulnerabilityRows(tw *table.Table, vulns []ty title = strings.Join(splitTitle[:12], " ") + "..." } - if len(v.PrimaryURL) > 0 { + if v.PrimaryURL != "" { if r.isTerminal { title = tml.Sprintf("%s\n%s", title, v.PrimaryURL) } else { diff --git a/pkg/sbom/cyclonedx/marshal.go b/pkg/sbom/cyclonedx/marshal.go index 9465b790bdd2..a8d96e4b12e3 100644 --- a/pkg/sbom/cyclonedx/marshal.go +++ b/pkg/sbom/cyclonedx/marshal.go @@ -298,7 +298,7 @@ func (*Marshaler) Properties(properties []core.Property) *[]cdx.Property { cdxProps := make([]cdx.Property, 0, len(properties)) for _, property := range properties { namespace := Namespace - if len(property.Namespace) > 0 { + if property.Namespace != "" { namespace = property.Namespace } From 264e77d9a4061fde6094554902eaff0bb0ef9dfe Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 16 Apr 2024 08:56:44 +0400 Subject: [PATCH 20/23] chore: bump golangci-lint Signed-off-by: knqyf263 --- .github/workflows/test.yaml | 4 ++-- magefiles/magefile.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7bfc2c6f0f4c..a552c341e8d3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,8 +45,8 @@ jobs: id: lint uses: golangci/golangci-lint-action@v4.0.0 with: - version: v1.54 - args: --deadline=30m --out-format=line-number + version: v1.57 + args: --timeout=30m --out-format=line-number skip-cache: true # https://github.com/golangci/golangci-lint-action/issues/244#issuecomment-1052197778 if: matrix.operating-system == 'ubuntu-latest' diff --git a/magefiles/magefile.go b/magefiles/magefile.go index fbb612b9d855..9e456f9bdf40 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -61,7 +61,7 @@ func (Tool) Wire() error { // GolangciLint installs golangci-lint func (Tool) GolangciLint() error { - const version = "v1.54.2" + const version = "v1.57.2" if exists(filepath.Join(GOBIN, "golangci-lint")) { return nil } From 15cdb30f4f5dace96b2e5ad5678d728de5b8b8f1 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Tue, 16 Apr 2024 12:44:38 +0400 Subject: [PATCH 21/23] chore: remove a debug line Signed-off-by: knqyf263 --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a552c341e8d3..04be731a0781 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -66,7 +66,6 @@ jobs: run: | mage docs:generate if [ -n "$(git status --porcelain)" ]; then - git status echo "Run 'mage docs:generate' and push it" exit 1 fi From 5717829b94165ce2ddf102730afafe87668f5165 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 17 Apr 2024 22:00:05 +0400 Subject: [PATCH 22/23] fix: use skip dirs from options Signed-off-by: knqyf263 --- pkg/fanal/walker/vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/walker/vm.go b/pkg/fanal/walker/vm.go index b97d50ed83f5..6ff32564cc0d 100644 --- a/pkg/fanal/walker/vm.go +++ b/pkg/fanal/walker/vm.go @@ -48,7 +48,7 @@ func NewVM() *VM { func (w *VM) Walk(vreader *io.SectionReader, root string, opt Option, fn WalkFunc) error { w.skipFiles = opt.SkipFiles - w.skipDirs = append(w.skipDirs, defaultSkipDirs...) + w.skipDirs = append(opt.SkipDirs, defaultSkipDirs...) // This function will be called on each file. w.analyzeFn = fn From e8190608d6f0f29c873ba8d43c529968f1149843 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Wed, 17 Apr 2024 22:01:34 +0400 Subject: [PATCH 23/23] refactor: remove an import alias Signed-off-by: knqyf263 --- pkg/fanal/artifact/vm/vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/artifact/vm/vm.go b/pkg/fanal/artifact/vm/vm.go index 8228a91fe41e..5b9aae130f4c 100644 --- a/pkg/fanal/artifact/vm/vm.go +++ b/pkg/fanal/artifact/vm/vm.go @@ -16,7 +16,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/handler" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" - semaphore "github.com/aquasecurity/trivy/pkg/semaphore" + "github.com/aquasecurity/trivy/pkg/semaphore" ) type Type string