refactor: remove parallel walk

.github/workflows/test.yaml

@@ -56,6 +56,7 @@ jobs:
         run: |
           mage docs:generate
           if [ -n "$(git status --porcelain)" ]; then
+            git status
             echo "Run 'mage docs:generate' and push it"
             exit 1
           fi

go.mod

@@ -85,7 +85,6 @@ require (
 	github.com/package-url/packageurl-go v0.1.2
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
 	github.com/samber/lo v1.39.0
-	github.com/saracen/walker v0.1.3
 	github.com/secure-systems-lab/go-securesystemslib v0.8.0
 	github.com/sigstore/rekor v1.2.2
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -1575,8 +1575,6 @@ github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
 github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
-github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g=
-github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=

pkg/commands/artifact/inject.go

@@ -5,8 +5,6 @@ package artifact
 
 import (
 	"context"
-	"github.com/aquasecurity/trivy/pkg/fanal/artifact/vm"
-
 	"github.com/google/wire"
 
 	"github.com/aquasecurity/trivy/pkg/fanal/artifact"
@@ -57,7 +55,7 @@ func initializeSBOMScanner(ctx context.Context, filePath string, artifactCache c
 }
 
 func initializeVMScanner(ctx context.Context, filePath string, artifactCache cache.ArtifactCache,
-	localArtifactCache cache.LocalArtifactCache, walker vm.Walker, artifactOption artifact.Option) (
+	localArtifactCache cache.LocalArtifactCache, artifactOption artifact.Option) (
 	scanner.Scanner, func(), error) {
 	wire.Build(scanner.StandaloneVMSet)
 	return scanner.Scanner{}, nil, nil
@@ -108,7 +106,7 @@ func initializeRemoteSBOMScanner(ctx context.Context, path string, artifactCache
 
 // initializeRemoteVMScanner is for vm scanning in client/server mode
 func initializeRemoteVMScanner(ctx context.Context, path string, artifactCache cache.ArtifactCache,
-	walker vm.Walker, remoteScanOptions client.ScannerOption, artifactOption artifact.Option) (scanner.Scanner, func(), error) {
+	remoteScanOptions client.ScannerOption, artifactOption artifact.Option) (scanner.Scanner, func(), error) {
 	wire.Build(scanner.RemoteVMSet)
 	return scanner.Scanner{}, nil, nil
 }

pkg/commands/artifact/run.go

@@ -19,6 +19,7 @@ import (
 	"github.com/aquasecurity/trivy/pkg/fanal/artifact"
 	"github.com/aquasecurity/trivy/pkg/fanal/cache"
 	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
+	"github.com/aquasecurity/trivy/pkg/fanal/walker"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/aquasecurity/trivy/pkg/javadb"
 	"github.com/aquasecurity/trivy/pkg/log"
@@ -646,9 +647,8 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 		},
 		ArtifactOption: artifact.Option{
 			DisabledAnalyzers: disabledAnalyzers(opts),
-			SkipFiles:         opts.SkipFiles,
-			SkipDirs:          opts.SkipDirs,
 			FilePatterns:      opts.FilePatterns,
+			Parallel:          opts.Parallel,
 			Offline:           opts.OfflineScan,
 			NoProgress:        opts.NoProgress || opts.Quiet,
 			Insecure:          opts.Insecure,
@@ -658,7 +658,6 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			SBOMSources:       opts.SBOMSources,
 			RekorURL:          opts.RekorURL,
 			//Platform:          opts.Platform,
-			Parallel:     opts.Parallel,
 			AWSRegion:    opts.Region,
 			AWSEndpoint:  opts.Endpoint,
 			FileChecksum: fileChecksum,
@@ -685,6 +684,12 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 				Full:                      opts.LicenseFull,
 				ClassifierConfidenceLevel: opts.LicenseConfidenceLevel,
 			},
+
+			// For file walking
+			WalkerOption: walker.Option{
+				SkipFiles: opts.SkipFiles,
+				SkipDirs:  opts.SkipDirs,
+			},
 		},
 	}, scanOptions, nil
 }

pkg/commands/artifact/scanner.go

@@ -5,7 +5,6 @@ import (
 
 	"golang.org/x/xerrors"
 
-	"github.com/aquasecurity/trivy/pkg/fanal/walker"
 	"github.com/aquasecurity/trivy/pkg/scanner"
 )
 
@@ -110,10 +109,7 @@ func sbomRemoteScanner(ctx context.Context, conf ScannerConfig) (scanner.Scanner
 
 // vmStandaloneScanner initializes a VM scanner in standalone mode
 func vmStandaloneScanner(ctx context.Context, conf ScannerConfig) (scanner.Scanner, func(), error) {
-	// TODO: The walker should be initialized in initializeVMScanner after https://github.com/aquasecurity/trivy/pull/5180
-	w := walker.NewVM(conf.ArtifactOption.SkipFiles, conf.ArtifactOption.SkipDirs)
-	s, cleanup, err := initializeVMScanner(ctx, conf.Target, conf.ArtifactCache, conf.LocalArtifactCache,
-		w, conf.ArtifactOption)
+	s, cleanup, err := initializeVMScanner(ctx, conf.Target, conf.ArtifactCache, conf.LocalArtifactCache, conf.ArtifactOption)
 	if err != nil {
 		return scanner.Scanner{}, func() {}, xerrors.Errorf("unable to initialize a vm scanner: %w", err)
 	}
@@ -122,9 +118,7 @@ func vmStandaloneScanner(ctx context.Context, conf ScannerConfig) (scanner.Scann
 
 // vmRemoteScanner initializes a VM scanner in client/server mode
 func vmRemoteScanner(ctx context.Context, conf ScannerConfig) (scanner.Scanner, func(), error) {
-	// TODO: The walker should be initialized in initializeVMScanner after https://github.com/aquasecurity/trivy/pull/5180
-	w := walker.NewVM(conf.ArtifactOption.SkipFiles, conf.ArtifactOption.SkipDirs)
-	s, cleanup, err := initializeRemoteVMScanner(ctx, conf.Target, conf.ArtifactCache, w, conf.ServerOption, conf.ArtifactOption)
+	s, cleanup, err := initializeRemoteVMScanner(ctx, conf.Target, conf.ArtifactCache, conf.ServerOption, conf.ArtifactOption)
 	if err != nil {
 		return scanner.Scanner{}, func() {}, xerrors.Errorf("unable to initialize a remote vm scanner: %w", err)
 	}

pkg/fanal/artifact/artifact.go

@@ -14,16 +14,14 @@ type Option struct {
 	AnalyzerGroup     analyzer.Group // It is empty in OSS
 	DisabledAnalyzers []analyzer.Type
 	DisabledHandlers  []types.HandlerType
-	SkipFiles         []string
-	SkipDirs          []string
 	FilePatterns      []string
+	Parallel          int
 	NoProgress        bool
 	Insecure          bool
 	Offline           bool
 	AppDirs           []string
 	SBOMSources       []string
 	RekorURL          string
-	Parallel          int
 	AWSRegion         string
 	AWSEndpoint       string
 	FileChecksum      bool // For SPDX
@@ -40,14 +38,7 @@ type Option struct {
 	SecretScannerOption  analyzer.SecretScannerOption
 	LicenseScannerOption analyzer.LicenseScannerOption
 
-	// File walk
-	WalkOption WalkOption
-}
-
-// WalkOption is a struct that allows users to define a custom walking behavior.
-// This option is only available when using Trivy as an imported library and not through CLI flags.
-type WalkOption struct {
-	ErrorCallback walker.ErrorCallback
+	WalkerOption walker.Option
 }
 
 func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions {
@@ -75,8 +66,8 @@ func (o *Option) Sort() {
 	sort.Slice(o.DisabledAnalyzers, func(i, j int) bool {
 		return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j]
 	})
-	sort.Strings(o.SkipFiles)
-	sort.Strings(o.SkipDirs)
+	sort.Strings(o.WalkerOption.SkipFiles)
+	sort.Strings(o.WalkerOption.SkipDirs)
 	sort.Strings(o.FilePatterns)
 }
 

pkg/fanal/artifact/image/image.go

@@ -62,7 +62,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
 	return Artifact{
 		image:          img,
 		cache:          c,
-		walker:         walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
+		walker:         walker.NewLayerTar(opt.WalkerOption),
 		analyzer:       a,
 		configAnalyzer: ca,
 		handlerManager: handlerManager,
@@ -202,7 +202,8 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
 	layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error {
 
 	var osFound types.OS
-	p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
+	p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context,
+		layerKey string) (any, error) {
 		layer := layerKeyMap[layerKey]
 
 		// If it is a base layer, secret scanning should not be performed.

pkg/fanal/artifact/local/fs.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/google/wire"
 	"github.com/opencontainers/go-digest"
 	"golang.org/x/xerrors"
 
@@ -19,21 +20,34 @@ import (
 	"github.com/aquasecurity/trivy/pkg/fanal/handler"
 	"github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/fanal/walker"
-	"github.com/aquasecurity/trivy/pkg/log"
 	"github.com/aquasecurity/trivy/pkg/semaphore"
 )
 
+var (
+	ArtifactSet = wire.NewSet(
+		walker.NewFS,
+		wire.Bind(new(Walker), new(*walker.FS)),
+		NewArtifact,
+	)
+
+	_ Walker = (*walker.FS)(nil)
+)
+
+type Walker interface {
+	Walk(root string, opt walker.Option, fn walker.WalkFunc) error
+}
+
 type Artifact struct {
 	rootPath       string
 	cache          cache.ArtifactCache
-	walker         walker.FS
+	walker         Walker
 	analyzer       analyzer.AnalyzerGroup
 	handlerManager handler.Manager
 
 	artifactOption artifact.Option
 }
 
-func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (artifact.Artifact, error) {
+func NewArtifact(rootPath string, c cache.ArtifactCache, w Walker, opt artifact.Option) (artifact.Artifact, error) {
 	handlerManager, err := handler.NewManager(opt)
 	if err != nil {
 		return nil, xerrors.Errorf("handler initialize error: %w", err)
@@ -45,72 +59,15 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
 	}
 
 	return Artifact{
-		rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
-		cache:    c,
-		walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
-			opt.Parallel, opt.WalkOption.ErrorCallback),
+		rootPath:       filepath.ToSlash(filepath.Clean(rootPath)),
+		cache:          c,
+		walker:         w,
 		analyzer:       a,
 		handlerManager: handlerManager,
-
 		artifactOption: opt,
 	}, nil
 }
 
-// buildPathsToSkip builds correct patch for skipDirs and skipFiles
-func buildPathsToSkip(base string, paths []string) []string {
-	var relativePaths []string
-	absBase, err := filepath.Abs(base)
-	if err != nil {
-		log.Logger.Warnf("Failed to get an absolute path of %s: %s", base, err)
-		return nil
-	}
-	for _, path := range paths {
-		// Supports three types of flag specification.
-		// All of them are converted into the relative path from the root directory.
-		// 1. Relative skip dirs/files from the root directory
-		//     The specified dirs and files will be used as is.
-		//       e.g. $ trivy fs --skip-dirs bar ./foo
-		//     The skip dir from the root directory will be `bar/`.
-		// 2. Relative skip dirs/files from the working directory
-		//     The specified dirs and files wll be converted to the relative path from the root directory.
-		//       e.g. $ trivy fs --skip-dirs ./foo/bar ./foo
-		//     The skip dir will be converted to `bar/`.
-		// 3. Absolute skip dirs/files
-		//     The specified dirs and files wll be converted to the relative path from the root directory.
-		//       e.g. $ trivy fs --skip-dirs /bar/foo/baz ./foo
-		//     When the working directory is
-		//       3.1 /bar: the skip dir will be converted to `baz/`.
-		//       3.2 /hoge : the skip dir will be converted to `../../bar/foo/baz/`.
-
-		absSkipPath, err := filepath.Abs(path)
-		if err != nil {
-			log.Logger.Warnf("Failed to get an absolute path of %s: %s", base, err)
-			continue
-		}
-		rel, err := filepath.Rel(absBase, absSkipPath)
-		if err != nil {
-			log.Logger.Warnf("Failed to get a relative path from %s to %s: %s", base, path, err)
-			continue
-		}
-
-		var relPath string
-		switch {
-		case !filepath.IsAbs(path) && strings.HasPrefix(rel, ".."):
-			// #1: Use the path as is
-			relPath = path
-		case !filepath.IsAbs(path) && !strings.HasPrefix(rel, ".."):
-			// #2: Use the relative path from the root directory
-			relPath = rel
-		case filepath.IsAbs(path):
-			// #3: Use the relative path from the root directory
-			relPath = rel
-		}
-		relPath = filepath.ToSlash(relPath)
-		relativePaths = append(relativePaths, relPath)
-	}
-	return relativePaths
-}
-
 func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) {
 	var wg sync.WaitGroup
 	result := analyzer.NewAnalysisResult()
@@ -126,7 +83,7 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error)
 		return types.ArtifactReference{}, xerrors.Errorf("failed to prepare filesystem for post analysis: %w", err)
 	}
 
-	err = a.walker.Walk(a.rootPath, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
+	err = a.walker.Walk(a.rootPath, a.artifactOption.WalkerOption, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
 		dir := a.rootPath
 
 		// When the directory is the same as the filePath, a file was given