Skip to content

Commit

Permalink
[chore] Log non-fatal errors encountered during a scan (#3612)
Browse files Browse the repository at this point in the history 
* [chore] Log non-fatal errors encountered during a scan

* Update tests

* Update tests
  • Loading branch information
@mcastorina
mcastorina authored Nov 19, 2024
1 parent dfcce42 commit 6a77b73
Show file tree
Hide file tree
Showing 22 changed files with 113 additions and 114 deletions.
40 changes: 25 additions & 15 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
}
}()

var ref sources.JobProgressRef
switch cmd {
case gitScan.FullCommand():
gitCfg := sources.GitConfig{
Expand All @@ -645,7 +646,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Bare: *gitScanBare,
ExcludeGlobs: *gitScanExcludeGlobs,
}
if err = eng.ScanGit(ctx, gitCfg); err != nil {
if ref, err = eng.ScanGit(ctx, gitCfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
}
case githubScan.FullCommand():
Expand Down Expand Up @@ -674,7 +675,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
CommentsTimeframeDays: *githubCommentsTimeframeDays,
Filter: filter,
}
if err := eng.ScanGitHub(ctx, cfg); err != nil {
if ref, err = eng.ScanGitHub(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
}
case githubExperimentalScan.FullCommand():
Expand All @@ -685,7 +686,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
CollisionThreshold: *githubExperimentalCollisionThreshold,
DeleteCachedData: *githubExperimentalDeleteCache,
}
if err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {
if ref, err = eng.ScanGitHubExperimental(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
}
case gitlabScan.FullCommand():
Expand All @@ -702,7 +703,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
ExcludeRepos: *gitlabScanExcludeRepos,
Filter: filter,
}
if err := eng.ScanGitLab(ctx, cfg); err != nil {
if ref, err = eng.ScanGitLab(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
}
case filesystemScan.FullCommand():
Expand All @@ -717,7 +718,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
IncludePathsFile: *filesystemScanIncludePaths,
ExcludePathsFile: *filesystemScanExcludePaths,
}
if err = eng.ScanFileSystem(ctx, cfg); err != nil {
if ref, err = eng.ScanFileSystem(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
}
case s3Scan.FullCommand():
Expand All @@ -731,7 +732,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
}
if err := eng.ScanS3(ctx, cfg); err != nil {
if ref, err = eng.ScanS3(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
}
case syslogScan.FullCommand():
Expand All @@ -743,15 +744,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
KeyPath: *syslogTLSKey,
Concurrency: *concurrency,
}
if err := eng.ScanSyslog(ctx, cfg); err != nil {
if ref, err = eng.ScanSyslog(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
}
case circleCiScan.FullCommand():
if err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
if ref, err = eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
}
case travisCiScan.FullCommand():
if err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
if ref, err = eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
}
case gcsScan.FullCommand():
Expand All @@ -768,7 +769,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Concurrency: *concurrency,
MaxObjectSize: int64(*gcsMaxObjectSize),
}
if err := eng.ScanGCS(ctx, cfg); err != nil {
if ref, err = eng.ScanGCS(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
}
case dockerScan.FullCommand():
Expand All @@ -777,7 +778,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Images: *dockerScanImages,
UseDockerKeychain: *dockerScanToken == "",
}
if err := eng.ScanDocker(ctx, cfg); err != nil {
if ref, err = eng.ScanDocker(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
}
case postmanScan.FullCommand():
Expand Down Expand Up @@ -814,7 +815,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
WorkspacePaths: *postmanWorkspacePaths,
EnvironmentPaths: *postmanEnvironmentPaths,
}
if err := eng.ScanPostman(ctx, cfg); err != nil {
if ref, err = eng.ScanPostman(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
}
case elasticsearchScan.FullCommand():
Expand All @@ -830,7 +831,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
SinceTimestamp: *elasticsearchSinceTimestamp,
BestEffortScan: *elasticsearchBestEffortScan,
}
if err := eng.ScanElasticsearch(ctx, cfg); err != nil {
if ref, err = eng.ScanElasticsearch(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
}
case jenkinsScan.FullCommand():
Expand All @@ -840,7 +841,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Username: *jenkinsUsername,
Password: *jenkinsPassword,
}
if err := eng.ScanJenkins(ctx, cfg); err != nil {
if ref, err = eng.ScanJenkins(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
}
case huggingfaceScan.FullCommand():
Expand Down Expand Up @@ -873,7 +874,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
IncludePrs: *huggingfaceIncludePrs,
Concurrency: *concurrency,
}
if err := eng.ScanHuggingface(ctx, cfg); err != nil {
if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
}
default:
Expand All @@ -885,6 +886,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)
}

// Print any errors reported during the scan.
if errs := ref.Snapshot().Errors; len(errs) > 0 {
errMsgs := make([]string, len(errs))
for i := 0; i < len(errs); i++ {
errMsgs[i] = errs[i].Error()
}
ctx.Logger().Error(nil, "encountered errors during scan", "errors", errMsgs)
}

if *printAvgDetectorTime {
printAverageDetectorTime(eng)
}
Expand Down
10 changes: 5 additions & 5 deletions pkg/engine/circleci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ import (

"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
)

// ScanCircleCI scans CircleCI logs.
func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
func (e *Engine) ScanCircleCI(ctx context.Context, token string) (sources.JobProgressRef, error) {
connection := &sourcespb.CircleCI{
Credential: &sourcespb.CircleCI_Token{
Token: token,
Expand All @@ -23,16 +24,15 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Circle CI connection")
return err
return sources.JobProgressRef{}, err
}

sourceName := "trufflehog - Circle CI"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, circleci.SourceType)

circleSource := &circleci.Source{}
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return err
return sources.JobProgressRef{}, err
}
_, err = e.sourceManager.Run(ctx, sourceName, circleSource)
return err
return e.sourceManager.Run(ctx, sourceName, circleSource)
}
9 changes: 4 additions & 5 deletions pkg/engine/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
)

// ScanDocker scans a given docker connection.
func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {
func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Docker{Images: c.Images}

switch {
Expand All @@ -29,16 +29,15 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal gitlab connection")
return err
return sources.JobProgressRef{}, err
}

sourceName := "trufflehog - docker"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, docker.SourceType)

dockerSource := &docker.Source{}
if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return err
return sources.JobProgressRef{}, err
}
_, err = e.sourceManager.Run(ctx, sourceName, dockerSource)
return err
return e.sourceManager.Run(ctx, sourceName, dockerSource)
}
9 changes: 4 additions & 5 deletions pkg/engine/elasticsearch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
)

// ScanElasticsearch scans a Elasticsearch installation.
func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) error {
func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Elasticsearch{
Nodes: c.Nodes,
Username: c.Username,
Expand All @@ -31,16 +31,15 @@ func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchC
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Elasticsearch connection")
return err
return sources.JobProgressRef{}, err
}

sourceName := "trufflehog - Elasticsearch"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, elasticsearch.SourceType)

elasticsearchSource := &elasticsearch.Source{}
if err := elasticsearchSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return err
return sources.JobProgressRef{}, err
}
_, err = e.sourceManager.Run(ctx, sourceName, elasticsearchSource)
return err
return e.sourceManager.Run(ctx, sourceName, elasticsearchSource)
}
16 changes: 8 additions & 8 deletions pkg/engine/engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func TestEngine_DuplicateSecrets(t *testing.T) {
e.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{absPath}}
if err := e.ScanFileSystem(ctx, cfg); err != nil {
if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}

Expand Down Expand Up @@ -372,7 +372,7 @@ even more`,
eng.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
err = eng.ScanFileSystem(ctx, cfg)
_, err = eng.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)

assert.NoError(t, eng.Finish(ctx))
Expand Down Expand Up @@ -420,7 +420,7 @@ func TestEngine_VersionedDetectorsVerifiedSecrets(t *testing.T) {
e.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
if err := e.ScanFileSystem(ctx, cfg); err != nil {
if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}

Expand Down Expand Up @@ -490,7 +490,7 @@ func TestEngine_CustomDetectorsDetectorsVerifiedSecrets(t *testing.T) {
e.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
if err := e.ScanFileSystem(ctx, cfg); err != nil {
if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}

Expand Down Expand Up @@ -540,7 +540,7 @@ func TestVerificationOverlapChunk(t *testing.T) {
e.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{absPath}}
if err := e.ScanFileSystem(ctx, cfg); err != nil {
if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}

Expand Down Expand Up @@ -630,7 +630,7 @@ func TestVerificationOverlapChunkFalsePositive(t *testing.T) {
e.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{absPath}}
err = e.ScanFileSystem(ctx, cfg)
_, err = e.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)

// Wait for all the chunks to be processed.
Expand Down Expand Up @@ -678,7 +678,7 @@ func TestRetainFalsePositives(t *testing.T) {
e.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{absPath}}
err = e.ScanFileSystem(ctx, cfg)
_, err = e.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)

// Wait for all the chunks to be processed.
Expand Down Expand Up @@ -1259,7 +1259,7 @@ def test_something():
eng.Start(ctx)

cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
err = eng.ScanFileSystem(ctx, cfg)
_, err = eng.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)

assert.NoError(t, eng.Finish(ctx))
Expand Down
9 changes: 4 additions & 5 deletions pkg/engine/filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
)

// ScanFileSystem scans a given file system.
func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Filesystem{
Paths: c.Paths,
IncludePathsFile: c.IncludePathsFile,
Expand All @@ -23,16 +23,15 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal filesystem connection")
return err
return sources.JobProgressRef{}, err
}

sourceName := "trufflehog - filesystem"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType)

fileSystemSource := &filesystem.Source{}
if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
return err
return sources.JobProgressRef{}, err
}
_, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
return err
return e.sourceManager.Run(ctx, sourceName, fileSystemSource)
}
2 changes: 1 addition & 1 deletion pkg/engine/filesystem_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func TestFilesystem(t *testing.T) {
})
assert.NoError(t, err)
e.Start(ctx)
err = e.ScanFileSystem(ctx, sources.FilesystemConfig{
_, err = e.ScanFileSystem(ctx, sources.FilesystemConfig{
Paths: []string{rootDir},
ExcludePathsFile: filepath.Join(configDir, "exclude"),
})
Expand Down
13 changes: 6 additions & 7 deletions pkg/engine/gcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ import (
)

// ScanGCS with the provided options.
func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) (sources.JobProgressRef, error) {
// Project ID is required if using any authenticated access.
if c.ProjectID == "" && !c.WithoutAuth {
return fmt.Errorf("project ID is required")
return sources.JobProgressRef{}, fmt.Errorf("project ID is required")
}

// If using unauthenticated access, the project ID is not used.
Expand All @@ -35,24 +35,23 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {

// Make sure only one auth method is selected.
if !isAuthValid(ctx, c, connection) {
return fmt.Errorf("multiple auth methods selected, please select only one")
return sources.JobProgressRef{}, fmt.Errorf("multiple auth methods selected, please select only one")
}

var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
return fmt.Errorf("failed to marshal GCS connection: %w", err)
return sources.JobProgressRef{}, fmt.Errorf("failed to marshal GCS connection: %w", err)
}

sourceName := "trufflehog - gcs"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gcs.SourceType)

gcsSource := &gcs.Source{}
if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
return err
return sources.JobProgressRef{}, err
}
_, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
return err
return e.sourceManager.Run(ctx, sourceName, gcsSource)
}

func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
Expand Down
Loading

0 comments on commit 6a77b73

Please sign in to comment.