Skip to content

Commit

Permalink
feat: allow restricting filesystem walk to specific folders
Browse files Browse the repository at this point in the history
  • Loading branch information
lebauce committed Nov 10, 2023
1 parent e2fb3dd commit 1d2c30a
Show file tree
Hide file tree
Showing 24 changed files with 167 additions and 15 deletions.
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_config.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ trivy config [flags] DIR
--include-non-failures include successes and exceptions, available with '--scanners config'
--k8s-version string specify k8s version to validate outdated api by it (example: 1.21.0)
--module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules")
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0")
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_filesystem.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ trivy filesystem [flags] PATH
--module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules")
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0")
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_image.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ trivy image [flags] IMAGE_NAME
--module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules")
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--platform string set platform in the form os/arch if image is multi-platform capable
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_kubernetes.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--no-progress suppress progress bar
--node-collector-namespace string specify the namespace in which the node-collector job should be deployed (default "trivy-temp")
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--parallel int number (between 1-20) of goroutines enabled for parallel scanning (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_repository.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules")
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0")
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_rootfs.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ trivy rootfs [flags] ROOTDIR
--module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules")
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0")
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_sbom.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ trivy sbom [flags] SBOM_PATH
--list-all-pkgs enabling the option will output all packages regardless of vulnerability
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--redis-ca string redis ca file location, if using redis as cache backend
--redis-cert string redis certificate file location, if using redis as cache backend
Expand Down
1 change: 1 addition & 0 deletions docs/docs/references/configuration/cli/trivy_vm.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ trivy vm [flags] VM_IMAGE
--module-dir string specify directory to the wasm modules that will be loaded (default "$HOME/.trivy/modules")
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
--only-dirs strings specify the directories where the traversal is allowed
-o, --output string output file name
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/defsec:0")
--redis-ca string redis ca file location, if using redis as cache backend
Expand Down
1 change: 1 addition & 0 deletions pkg/commands/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,7 @@ func NewConfigCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command {
// Enable only '--skip-dirs' and '--skip-files' and disable other flags
SkipDirs: &flag.SkipDirsFlag,
SkipFiles: &flag.SkipFilesFlag,
OnlyDirs: &flag.OnlyDirsFlag,
FilePatterns: &flag.FilePatternsFlag,
}

Expand Down
1 change: 1 addition & 0 deletions pkg/commands/artifact/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
DisabledAnalyzers: disabledAnalyzers(opts),
SkipFiles: opts.SkipFiles,
SkipDirs: opts.SkipDirs,
OnlyDirs: opts.OnlyDirs,
FilePatterns: opts.FilePatterns,
Offline: opts.OfflineScan,
NoProgress: opts.NoProgress || opts.Quiet,
Expand Down
2 changes: 2 additions & 0 deletions pkg/fanal/artifact/artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type Option struct {
DisabledHandlers []types.HandlerType
SkipFiles []string
SkipDirs []string
OnlyDirs []string
FilePatterns []string
NoProgress bool
Insecure bool
Expand Down Expand Up @@ -57,6 +58,7 @@ func (o *Option) Sort() {
sort.Strings(o.SkipFiles)
sort.Strings(o.SkipDirs)
sort.Strings(o.FilePatterns)
sort.Strings(o.OnlyDirs)
}

type Artifact interface {
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/artifact/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
return Artifact{
image: img,
cache: c,
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.OnlyDirs, opt.Slow),
analyzer: a,
configAnalyzer: ca,
handlerManager: handlerManager,
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/artifact/local/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
return Artifact{
rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
cache: c,
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs), buildPathsToSkip(rootPath, opt.OnlyDirs),
opt.Slow, opt.WalkOption.ErrorCallback),
analyzer: a,
handlerManager: handlerManager,
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/artifact/vm/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
cache: c,
analyzer: a,
handlerManager: handlerManager,
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.OnlyDirs, opt.Slow),
artifactOption: opt,
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/cache/key.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ func CalcKey(id string, analyzerVersions analyzer.Versions, hookVersions map[str
HookVersions map[string]int
SkipFiles []string
SkipDirs []string
OnlyDirs []string `json:",omitempty"`
FilePatterns []string `json:",omitempty"`
}{id, analyzerVersions, hookVersions, artifactOpt.SkipFiles, artifactOpt.SkipDirs, artifactOpt.FilePatterns}
}{id, analyzerVersions, hookVersions, artifactOpt.SkipFiles, artifactOpt.SkipDirs, artifactOpt.OnlyDirs, artifactOpt.FilePatterns}

if err := json.NewEncoder(h).Encode(keyBase); err != nil {
return "", xerrors.Errorf("json encode error: %w", err)
Expand Down
4 changes: 2 additions & 2 deletions pkg/fanal/walker/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type FS struct {
errCallback ErrorCallback
}

func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS {
func NewFS(skipFiles, skipDirs, onlyDirs []string, slow bool, errCallback ErrorCallback) FS {
if errCallback == nil {
errCallback = func(pathname string, err error) error {
// ignore permission errors
Expand All @@ -32,7 +32,7 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F
}

return FS{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs, onlyDirs, slow),
errCallback: errCallback,
}
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/walker/fs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func TestDir_Walk(t *testing.T) {
type fields struct {
skipFiles []string
skipDirs []string
onlyDirs []string
errCallback walker.ErrorCallback
}
tests := []struct {
Expand Down Expand Up @@ -93,7 +94,7 @@ func TestDir_Walk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback)
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.onlyDirs, true, tt.fields.errCallback)

err := w.Walk(tt.rootDir, tt.analyzeFn)
if tt.wantErr != "" {
Expand Down
4 changes: 2 additions & 2 deletions pkg/fanal/walker/tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ type LayerTar struct {
threshold int64
}

func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar {
func NewLayerTar(skipFiles, skipDirs, onlyDirs []string, slow bool) LayerTar {
threshold := defaultSizeThreshold
if slow {
threshold = slowSizeThreshold
}

return LayerTar{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs, onlyDirs, slow),
threshold: threshold,
}
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/fanal/walker/tar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func TestLayerTar_Walk(t *testing.T) {
type fields struct {
skipFiles []string
skipDirs []string
onlyDirs []string
}
tests := []struct {
name string
Expand Down Expand Up @@ -81,7 +82,7 @@ func TestLayerTar_Walk(t *testing.T) {
f, err := os.Open("testdata/test.tar")
require.NoError(t, err)

w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true)
w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, tt.fields.onlyDirs, true)

gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn)
if tt.wantErr != "" {
Expand Down
4 changes: 2 additions & 2 deletions pkg/fanal/walker/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,14 @@ type VM struct {
analyzeFn WalkFunc
}

func NewVM(skipFiles, skipDirs []string, slow bool) VM {
func NewVM(skipFiles, skipDirs, onlyDirs []string, slow bool) VM {
threshold := defaultSizeThreshold
if slow {
threshold = slowSizeThreshold
}

return VM{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs, onlyDirs, slow),
threshold: threshold,
}
}
Expand Down
43 changes: 42 additions & 1 deletion pkg/fanal/walker/walk.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,16 @@ type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) er
type walker struct {
skipFiles []string
skipDirs []string
onlyDirs []pattern
slow bool
}

func newWalker(skipFiles, skipDirs []string, slow bool) walker {
type pattern struct {
pattern string
base string
}

func newWalker(skipFiles, skipDirs, onlyDirs []string, slow bool) walker {
var cleanSkipFiles, cleanSkipDirs []string
for _, skipFile := range skipFiles {
skipFile = filepath.ToSlash(filepath.Clean(skipFile))
Expand All @@ -49,9 +55,18 @@ func newWalker(skipFiles, skipDirs []string, slow bool) walker {
cleanSkipDirs = append(cleanSkipDirs, skipDir)
}

var cleanOnlyDirs []pattern
for _, onlyDir := range onlyDirs {
onlyDir = filepath.ToSlash(filepath.Clean(onlyDir))
onlyDir = strings.TrimLeft(onlyDir, "/")
base, _ := doublestar.SplitPattern(onlyDir)
cleanOnlyDirs = append(cleanOnlyDirs, pattern{base: base, pattern: onlyDir})
}

return walker{
skipFiles: cleanSkipFiles,
skipDirs: cleanSkipDirs,
onlyDirs: cleanOnlyDirs,
slow: slow,
}
}
Expand All @@ -69,6 +84,21 @@ func (w *walker) shouldSkipFile(filePath string) bool {
return true
}
}

if len(w.onlyDirs) > 0 {
dir := filepath.Dir(filePath)
for _, onlyDir := range w.onlyDirs {
match, err := doublestar.Match(onlyDir.pattern, dir)
if err != nil {
return false // return early if bad pattern
} else if match {
log.Logger.Debugf("Skipping file: %s", filePath)
return false
}
}
return true
}

return false
}

Expand All @@ -91,5 +121,16 @@ func (w *walker) shouldSkipDir(dir string) bool {
}
}

if dir != "." && len(w.onlyDirs) > 0 {
for _, onlyDir := range w.onlyDirs {
if onlyDir.base != "." && !strings.HasPrefix(onlyDir.base, dir) && !strings.HasPrefix(dir, onlyDir.base) {
return true
}
}

log.Logger.Debugf("Skipping directory: %s", dir)
return false
}

return false
}
86 changes: 84 additions & 2 deletions pkg/fanal/walker/walk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func Test_shouldSkipFile(t *testing.T) {

for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(tc.skipFiles, nil, false)
w := newWalker(tc.skipFiles, nil, nil, false)
for file, skipResult := range tc.skipMap {
assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file))
}
Expand Down Expand Up @@ -115,10 +115,92 @@ func Test_shouldSkipDir(t *testing.T) {

for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(nil, tc.skipDirs, false)
w := newWalker(nil, tc.skipDirs, nil, false)
for dir, skipResult := range tc.skipMap {
assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir))
}
})
}
}

func Test_onlyDir(t *testing.T) {
testCases := []struct {
skipDirs []string
onlyDirs []string
skipMap map[string][2]bool
}{
{
skipDirs: nil,
onlyDirs: []string{"/etc/**"},
skipMap: map[string][2]bool{
"/etc/foo/bar": {false, false},
"/var/log/bar": {true, true},
},
},
{
skipDirs: nil,
onlyDirs: []string{"/**"},
skipMap: map[string][2]bool{
"/foo": {false, false},
"/etc/foo": {false, false},
"/var/log/bar": {false, false},
},
},
{
skipDirs: nil,
onlyDirs: []string{"/*"},
skipMap: map[string][2]bool{
"/foo": {false, false},
"/etc/foo": {false, false},
"/etc/foo/bar": {true, false},
},
},
{
skipDirs: nil,
onlyDirs: []string{"/etc/foo/*"},
skipMap: map[string][2]bool{
"/etc/foo/bar": {true, false},
"/etc/foo2/bar": {true, false},
"/var/etc/foo2/bar": {true, true},
},
},
{
onlyDirs: []string{"/*/bar"},
skipMap: map[string][2]bool{
"/etc/bar/bar": {false, false},
"/etc/foo/bar": {true, false},
"/etc/bar/foo": {false, false},
"/etc/foo/bar/foo": {true, false},
},
},
{
onlyDirs: []string{"**/bar"},
skipMap: map[string][2]bool{
"/etc/bar/bar": {false, false},
"/etc/foo/bar": {true, false},
"/etc/bar/foo": {false, false},
"/etc/foo/bar/foo": {false, false},
},
},
{
onlyDirs: []string{"**/foo/*/bar"},
skipMap: map[string][2]bool{
"/foo/foo/bar/bar": [2]bool{false, false},
"/etc/foo/bar": {true, false},
"/etc/bar/foo": {true, false},
"/etc/foo/bar/bar": {true, false},
},
},
}

for i, tc := range testCases {
t.Run(fmt.Sprint(i), func(t *testing.T) {
w := newWalker(nil, tc.skipDirs, tc.onlyDirs, false)
for file, skipResult := range tc.skipMap {
// todo: add check on skip dir
assert.Equal(t, skipResult[0], w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, file))
assert.Equal(t, skipResult[1], w.shouldSkipDir(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, file))
}
})
}
}
Loading

0 comments on commit 1d2c30a

Please sign in to comment.