From 2796abe1ed403871e75c4cd4fda23b7884a6d4eb Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Tue, 6 Jun 2023 08:19:15 +0300 Subject: [PATCH] refactor: add composite fs for post-analyzers (#4556) --- pkg/fanal/analyzer/analyzer.go | 11 +-- pkg/fanal/analyzer/analyzer_test.go | 9 +-- pkg/fanal/analyzer/fs.go | 103 ++++++++++++++++++++++++++++ pkg/fanal/artifact/image/image.go | 71 +++---------------- pkg/fanal/artifact/local/fs.go | 45 +++--------- 5 files changed, 136 insertions(+), 103 deletions(-) create mode 100644 pkg/fanal/analyzer/fs.go diff --git a/pkg/fanal/analyzer/analyzer.go b/pkg/fanal/analyzer/analyzer.go index 85285d442d6a..4e42eedca83c 100644 --- a/pkg/fanal/analyzer/analyzer.go +++ b/pkg/fanal/analyzer/analyzer.go @@ -19,9 +19,7 @@ import ( aos "github.com/aquasecurity/trivy/pkg/fanal/analyzer/os" "github.com/aquasecurity/trivy/pkg/fanal/log" "github.com/aquasecurity/trivy/pkg/fanal/types" - "github.com/aquasecurity/trivy/pkg/mapfs" "github.com/aquasecurity/trivy/pkg/misconf" - "github.com/aquasecurity/trivy/pkg/syncx" ) var ( @@ -467,9 +465,9 @@ func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo) // and passes it to the respective post-analyzer. // The obtained results are merged into the "result". // This function may be called concurrently and must be thread-safe. -func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, files *syncx.Map[Type, *mapfs.FS], result *AnalysisResult, opts AnalysisOptions) error { +func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult, opts AnalysisOptions) error { for _, a := range ag.postAnalyzers { - fsys, ok := files.Load(a.Type()) + fsys, ok := compositeFS.Get(a.Type()) if !ok { continue } @@ -504,6 +502,11 @@ func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, files *syncx.Map[Type, return nil } +// PostAnalyzerFS returns a composite filesystem that contains multiple filesystems for each post-analyzer +func (ag AnalyzerGroup) PostAnalyzerFS() (*CompositeFS, error) { + return NewCompositeFS(ag) +} + func (ag AnalyzerGroup) filePatternMatch(analyzerType Type, filePath string) bool { for _, pattern := range ag.filePatterns[analyzerType] { if pattern.MatchString(filePath) { diff --git a/pkg/fanal/analyzer/analyzer_test.go b/pkg/fanal/analyzer/analyzer_test.go index db85f7c5803d..d527374d8c2b 100644 --- a/pkg/fanal/analyzer/analyzer_test.go +++ b/pkg/fanal/analyzer/analyzer_test.go @@ -18,7 +18,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/mapfs" - "github.com/aquasecurity/trivy/pkg/syncx" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/imgconf/apk" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/java/jar" @@ -608,10 +607,12 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) { require.NoError(t, err) // Create a virtual filesystem - files := new(syncx.Map[analyzer.Type, *mapfs.FS]) + composite, err := analyzer.NewCompositeFS(analyzer.AnalyzerGroup{}) + require.NoError(t, err) + mfs := mapfs.New() require.NoError(t, mfs.CopyFilesUnder(tt.dir)) - files.Store(tt.analyzerType, mfs) + composite.Set(tt.analyzerType, mfs) if tt.analyzerType == analyzer.TypeJar { // init java-trivy-db with skip update @@ -620,7 +621,7 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) { ctx := context.Background() got := new(analyzer.AnalysisResult) - err = a.PostAnalyze(ctx, files, got, analyzer.AnalysisOptions{}) + err = a.PostAnalyze(ctx, composite, got, analyzer.AnalysisOptions{}) require.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/fs.go b/pkg/fanal/analyzer/fs.go new file mode 100644 index 000000000000..66b458c79d0d --- /dev/null +++ b/pkg/fanal/analyzer/fs.go @@ -0,0 +1,103 @@ +package analyzer + +import ( + "errors" + "io" + "io/fs" + "os" + "path" + "path/filepath" + + "golang.org/x/xerrors" + + "github.com/aquasecurity/trivy/pkg/mapfs" + "github.com/aquasecurity/trivy/pkg/syncx" +) + +// CompositeFS contains multiple filesystems for post-analyzers +type CompositeFS struct { + group AnalyzerGroup + dir string + files *syncx.Map[Type, *mapfs.FS] +} + +func NewCompositeFS(group AnalyzerGroup) (*CompositeFS, error) { + tmpDir, err := os.MkdirTemp("", "analyzer-fs-*") + if err != nil { + return nil, xerrors.Errorf("unable to create temporary directory: %w", err) + } + + return &CompositeFS{ + group: group, + dir: tmpDir, + files: new(syncx.Map[Type, *mapfs.FS]), + }, nil +} + +// CopyFileToTemp takes a file path and information, opens the file, copies its contents to a temporary file +func (c *CompositeFS) CopyFileToTemp(opener Opener, info os.FileInfo) (string, error) { + // Create a temporary file to which the file in the layer will be copied + // so that all the files will not be loaded into memory + f, err := os.CreateTemp(c.dir, "file-*") + if err != nil { + return "", xerrors.Errorf("create temp error: %w", err) + } + defer f.Close() + + // Open a file in the layer + r, err := opener() + if err != nil { + return "", xerrors.Errorf("file open error: %w", err) + } + defer r.Close() + + // Copy file content into the temporary file + if _, err = io.Copy(f, r); err != nil { + return "", xerrors.Errorf("copy error: %w", err) + } + + if err = os.Chmod(f.Name(), info.Mode()); err != nil { + return "", xerrors.Errorf("chmod error: %w", err) + } + + return f.Name(), nil +} + +// CreateLink creates a link in the virtual filesystem that corresponds to a real file. +// The linked virtual file will have the same path as the real file path provided. +func (c *CompositeFS) CreateLink(analyzerTypes []Type, virtualPath, realPath string, setRoot bool) error { + // Create fs.FS for each post-analyzer that wants to analyze the current file + for _, t := range analyzerTypes { + // Since filesystem scanning may require access outside the specified path, (e.g. Terraform modules) + // it allows "../" access with "WithUnderlyingRoot". + var opts []mapfs.Option + if setRoot { + opts = append(opts, mapfs.WithUnderlyingRoot(filepath.Dir(realPath))) + } + mfs, _ := c.files.LoadOrStore(t, mapfs.New(opts...)) + if d := path.Dir(virtualPath); d != "." { + if err := mfs.MkdirAll(d, os.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { + return xerrors.Errorf("mapfs mkdir error: %w", err) + } + } + if err := mfs.WriteFile(virtualPath, realPath); err != nil { + return xerrors.Errorf("mapfs write error: %w", err) + } + } + return nil +} + +// Set sets the fs.FS for the specified post-analyzer +func (c *CompositeFS) Set(t Type, fs *mapfs.FS) { + c.files.Store(t, fs) +} + +// Get returns the fs.FS for the specified post-analyzer +func (c *CompositeFS) Get(t Type) (*mapfs.FS, bool) { + return c.files.Load(t) +} + +// Cleanup removes the temporary directory +func (c *CompositeFS) Cleanup() error { + return os.RemoveAll(c.dir) +} diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index fb46245fbaea..0795934fd7c4 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -4,9 +4,7 @@ import ( "context" "errors" "io" - "io/fs" "os" - "path/filepath" "reflect" "strings" "sync" @@ -24,10 +22,8 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/log" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" - "github.com/aquasecurity/trivy/pkg/mapfs" "github.com/aquasecurity/trivy/pkg/parallel" "github.com/aquasecurity/trivy/pkg/semaphore" - "github.com/aquasecurity/trivy/pkg/syncx" ) type Artifact struct { @@ -274,12 +270,11 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable limit := semaphore.New(a.artifactOption.Slow) // Prepare filesystem for post analysis - files := new(syncx.Map[analyzer.Type, *mapfs.FS]) - tmpDir, err := os.MkdirTemp("", "layers-*") + composite, err := a.analyzer.PostAnalyzerFS() if err != nil { - return types.BlobInfo{}, xerrors.Errorf("mkdir temp error: %w", err) + return types.BlobInfo{}, xerrors.Errorf("unable to get post analysis filesystem: %w", err) } - defer os.RemoveAll(tmpDir) + defer composite.Cleanup() // Walk a tar layer opqDirs, whFiles, err := a.walker.Walk(rc, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { @@ -288,8 +283,13 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable } // Build filesystem for post analysis - if err = a.buildFS(tmpDir, filePath, info, opener, files); err != nil { - return xerrors.Errorf("failed to build filesystem: %w", err) + tmpFilePath, err := composite.CopyFileToTemp(opener, info) + if err != nil { + return xerrors.Errorf("failed to copy file to temp: %w", err) + } + analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info) + if err = composite.CreateLink(analyzerTypes, filePath, tmpFilePath, false); err != nil { + return xerrors.Errorf("failed to write a file: %w", err) } return nil @@ -302,7 +302,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable wg.Wait() // Post-analysis - if err = a.analyzer.PostAnalyze(ctx, files, result, opts); err != nil { + if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil { return types.BlobInfo{}, xerrors.Errorf("post analysis error: %w", err) } @@ -337,55 +337,6 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable return blobInfo, nil } -// buildFS creates filesystem for post analysis -func (a Artifact) buildFS(tmpDir, filePath string, info os.FileInfo, opener analyzer.Opener, - files *syncx.Map[analyzer.Type, *mapfs.FS]) error { - // Get all post-analyzers that want to analyze the file - atypes := a.analyzer.RequiredPostAnalyzers(filePath, info) - if len(atypes) == 0 { - return nil - } - - // Create a temporary file to which the file in the layer will be copied - // so that all the files will not be loaded into memory - f, err := os.CreateTemp(tmpDir, "layer-file-*") - if err != nil { - return xerrors.Errorf("create temp error: %w", err) - } - defer f.Close() - - // Open a file in the layer - r, err := opener() - if err != nil { - return xerrors.Errorf("file open error: %w", err) - } - defer r.Close() - - // Copy file content into the temporary file - if _, err = io.Copy(f, r); err != nil { - return xerrors.Errorf("copy error: %w", err) - } - - if err = os.Chmod(f.Name(), info.Mode()); err != nil { - return xerrors.Errorf("chmod error: %w", err) - } - - // Create fs.FS for each post-analyzer that wants to analyze the current file - for _, at := range atypes { - fsys, _ := files.LoadOrStore(at, mapfs.New()) - if dir := filepath.Dir(filePath); dir != "." { - if err := fsys.MkdirAll(dir, os.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { - return xerrors.Errorf("mapfs mkdir error: %w", err) - } - } - err = fsys.WriteFile(filePath, f.Name()) - if err != nil { - return xerrors.Errorf("mapfs write error: %w", err) - } - } - return nil -} - func (a Artifact) diffIDs(configFile *v1.ConfigFile) []string { if configFile == nil { return nil diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index b1d70021bff6..912bf625ff22 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -4,8 +4,6 @@ import ( "context" "crypto/sha256" "encoding/json" - "errors" - "io/fs" "os" "path/filepath" "strings" @@ -21,9 +19,7 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/walker" "github.com/aquasecurity/trivy/pkg/log" - "github.com/aquasecurity/trivy/pkg/mapfs" "github.com/aquasecurity/trivy/pkg/semaphore" - "github.com/aquasecurity/trivy/pkg/syncx" ) type Artifact struct { @@ -132,9 +128,12 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) } // Prepare filesystem for post analysis - files := new(syncx.Map[analyzer.Type, *mapfs.FS]) + composite, err := a.analyzer.PostAnalyzerFS() + if err != nil { + return types.ArtifactReference{}, xerrors.Errorf("failed to prepare filesystem for post analysis: %w", err) + } - err := a.walker.Walk(a.rootPath, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + err = a.walker.Walk(a.rootPath, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { dir := a.rootPath // When the directory is the same as the filePath, a file was given @@ -143,13 +142,14 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) dir, filePath = filepath.Split(a.rootPath) } - if err := a.analyzer.AnalyzeFile(ctx, &wg, limit, result, dir, filePath, info, opener, nil, opts); err != nil { + if err = a.analyzer.AnalyzeFile(ctx, &wg, limit, result, dir, filePath, info, opener, nil, opts); err != nil { return xerrors.Errorf("analyze file (%s): %w", filePath, err) } // Build filesystem for post analysis - if err := a.buildFS(dir, filePath, info, files); err != nil { - return xerrors.Errorf("failed to build filesystem: %w", err) + analyzerTypes := a.analyzer.RequiredPostAnalyzers(filePath, info) + if err = composite.CreateLink(analyzerTypes, filePath, filepath.Join(dir, filePath), true); err != nil { + return xerrors.Errorf("failed to create link: %w", err) } return nil @@ -162,7 +162,7 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) wg.Wait() // Post-analysis - if err = a.analyzer.PostAnalyze(ctx, files, result, opts); err != nil { + if err = a.analyzer.PostAnalyze(ctx, composite, result, opts); err != nil { return types.ArtifactReference{}, xerrors.Errorf("post analysis error: %w", err) } @@ -231,28 +231,3 @@ func (a Artifact) calcCacheKey(blobInfo types.BlobInfo) (string, error) { return cacheKey, nil } - -// buildFS creates filesystem for post analysis -func (a Artifact) buildFS(dir, filePath string, info os.FileInfo, files *syncx.Map[analyzer.Type, *mapfs.FS]) error { - // Get all post-analyzers that want to analyze the file - atypes := a.analyzer.RequiredPostAnalyzers(filePath, info) - if len(atypes) == 0 { - return nil - } - - // Create fs.FS for each post-analyzer that wants to analyze the current file - for _, at := range atypes { - // Since filesystem scanning may require access outside the specified path, (e.g. Terraform modules) - // it allows "../" access with "WithUnderlyingRoot". - mfs, _ := files.LoadOrStore(at, mapfs.New(mapfs.WithUnderlyingRoot(dir))) - if d := filepath.Dir(filePath); d != "." { - if err := mfs.MkdirAll(d, os.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { - return xerrors.Errorf("mapfs mkdir error: %w", err) - } - } - if err := mfs.WriteFile(filePath, filepath.Join(dir, filePath)); err != nil { - return xerrors.Errorf("mapfs write error: %w", err) - } - } - return nil -}