Merge pull request mercedes-benz#3084 from mercedes-benz/feature-3078…

…-client-secrets-scan-switch-off-filter

Feature 3078 client secrets scan switches off filtering

sechub-cli/src/mercedes-benz.com/sechub/cli/constants.go

@@ -46,11 +46,29 @@ const MinimalTimeoutInSeconds = 10
 // SizeOfJobList - Number of latest jobs to print
 const SizeOfJobList = 20
 
-// DefaultSourceCodeExcludeDirPatterns - Define directory patterns to exclude from zip file:
-// - code in directories named "test" is not considered to end up in the binary
-// - also ignore ".git" directory
-// - ignore "node_modules" directories which may contain millions of lines of library code
-var DefaultSourceCodeExcludeDirPatterns = []string{"**/test/**", "**/.git/**", "**/node_modules/**"}
+// DefaultSCMDirPatterns - directories containing scm (source code management) data
+var DefaultSCMDirPatterns = []string{"**/.git/**"}
+
+// DefaultSourceCodeExcludeDirPatterns - Define directory patterns to exclude from zip file
+var DefaultSourceCodeUnwantedDirPatterns = []string{
+	"**/test/**",                   /* code in directories named "test" is not considered to end up in the binary */
+	"**/node_modules/**",           /* ignore "node_modules" directories which may contain millions of lines of library code */
+	"**/.gradle/**",                /* ignore Gradle cache */
+	"**/.idea/**", "**/.vscode/**", /* ignore IDE's directories (IntellliJ, VS Code) */
+}
+
+// DefaultSecretScanUnwantedFilePatterns - File patterns (case insensitive) to exclude from secrets scans
+// - we won't catch all, but aim for the most common ones in repos / build artifacts
+var DefaultSecretScanUnwantedFilePatterns = []string{
+	"sechub-false-positives-*.json", "sechub_report_*.json", /* SecHub files */
+	"*.a", "*.so", /* Unix libraries */
+	"*.class", "*.jar", /* Java binaries */
+	"*.gif", "*.jpeg", "*.jpg", "*.png", ".svg", /* Image files */
+	"*.tar", "*.xz", "*.zip", /* Archive files */
+}
+
+// DefaultSourceCodeExcludeDirPatterns - Exclude patterns for SAST / code scans
+var DefaultSourceCodeExcludeDirPatterns = append(DefaultSCMDirPatterns, DefaultSourceCodeUnwantedDirPatterns...)
 
 // SupportedReportFormats - Supported output formats for SecHub reports
 const ReportFormatJSON = "json"

sechub-cli/src/mercedes-benz.com/sechub/cli/controller.go

@@ -123,13 +123,15 @@ func prepareScan(context *Context) {
 		// Creating sources ZIP file
 		context.sourceZipFileName = tempFile(context, fmt.Sprintf("sourcecode-%s.zip", context.config.projectID))
 
-		// Set source code patterns in
+		// Set sources filter patterns in
 		// - data.sources
 		// - codeScan
 		// depending on
-		// - DefaultSourceCodeAllowedFilePatterns
+		// - scan type
+		//   - codeScan -> DefaultSourceCodeAllowedFilePatterns
+		//   - secretScan -> everything but blacklisted
 		// - context.config.whitelistAll (deactivates all filters)
-		adjustSourceCodePatterns(context)
+		adjustSourceFilterPatterns(context)
 
 		err := createSouceCodeZipFile(context)
 		if err != nil {

sechub-cli/src/mercedes-benz.com/sechub/cli/sechubconfig.go

@@ -7,8 +7,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
+	"slices"
 	"strings"
 	"text/template"
 
@@ -24,6 +24,7 @@ type SecHubConfig struct {
 	ProjectID  string                `json:"project"`
 	Server     string                `json:"server"`
 	CodeScan   CodeScanConfig        `json:"codeScan"`
+	SecretScan SecretScanConfig      `json:"secretScan"`
 	Data       DataSectionScanConfig `json:"data"`
 }
 
@@ -50,10 +51,18 @@ type NamedBinariesScanConfig struct {
 // CodeScanConfig - definition of a code scan
 type CodeScanConfig struct {
 	Use []string `json:"use"`
-	// From here: kept for backward compatibility. Take "use" in conjunction with data section.
+	////////////////////////////////
+	// From here: Deprecated/legacy
+	// Kept for backward compatibility. Take "use" in conjunction with data section.
 	FileSystem         FileSystemConfig `json:"fileSystem"`
 	Excludes           []string         `json:"excludes"`
 	SourceCodePatterns []string         `json:"additionalFilenameExtensions"`
+	////////////////////////////////
+}
+
+// SecretScanConfig - definition of a secrets scan
+type SecretScanConfig struct {
+	Use []string `json:"use"`
 }
 
 // FileSystemConfig contains data for defined files+folders
@@ -110,7 +119,7 @@ func newSecHubConfigurationFromFile(context *Context, filePath string) (SecHubCo
 
 	/* read text content as "unfilled byte value". This will be used for debug outputs,
 	   so we do not have passwords etc. accidently leaked. We limit read to maximum allowed bytes */
-	context.inputForContentProcessing, err = ioutil.ReadAll(io.LimitReader(jsonFile, MaximumBytesOfSecHubConfig))
+	context.inputForContentProcessing, err = io.ReadAll(io.LimitReader(jsonFile, MaximumBytesOfSecHubConfig))
 
 	if sechubUtil.HandleIOError(err) {
 		showHelpHint()
@@ -149,33 +158,58 @@ func envToMap() (map[string]string, error) {
 	return envMap, err
 }
 
-func adjustSourceCodePatterns(context *Context) {
+func adjustSourceFilterPatterns(context *Context) {
 	for i, item := range context.sechubConfig.Data.Sources {
-		context.sechubConfig.Data.Sources[i].SourceCodePatterns =
-			adjustSourceCodePatternsWhitelistAll(item.SourceCodePatterns, context.config.whitelistAll)
+
+		if slices.Contains(context.sechubConfig.SecretScan.Use, item.Name) {
+			// Clear all source code patterns for secrets scans
+			context.sechubConfig.Data.Sources[i].SourceCodePatterns =
+				adjustSourceFilterPatternsWhitelistAll(item.SourceCodePatterns, true)
+		} else if slices.Contains(context.sechubConfig.CodeScan.Use, item.Name) {
+			// Append default source code patterns for code scans
+			context.sechubConfig.Data.Sources[i].SourceCodePatterns =
+				adjustSourceFilterPatternsWhitelistAll(item.SourceCodePatterns, context.config.whitelistAll)
+		}
 
 		if !context.config.ignoreDefaultExcludes {
-			// add default exclude patterns to exclude list
-			context.sechubConfig.Data.Sources[i].Excludes = append(item.Excludes, DefaultSourceCodeExcludeDirPatterns...)
+			excludePatterns := computeSourceExcludePatterns(context, item)
+			// add exclude patterns to exclude list
+			context.sechubConfig.Data.Sources[i].Excludes = append(item.Excludes, excludePatterns...)
 		}
 	}
 
+	////////////////////////////////////////////////
+	// Old/legacy support
 	// We still support the old/legacy format directly in context.sechubConfig.CodeScan:
 	if len(context.sechubConfig.CodeScan.FileSystem.Folders) > 0 {
 		context.sechubConfig.CodeScan.SourceCodePatterns =
-			adjustSourceCodePatternsWhitelistAll(context.sechubConfig.CodeScan.SourceCodePatterns, context.config.whitelistAll)
+			adjustSourceFilterPatternsWhitelistAll(context.sechubConfig.CodeScan.SourceCodePatterns, context.config.whitelistAll)
 
 		if !context.config.ignoreDefaultExcludes {
 			context.sechubConfig.CodeScan.Excludes = append(context.sechubConfig.CodeScan.Excludes, DefaultSourceCodeExcludeDirPatterns...)
 		}
 	}
+	////////////////////////////////////////////////
 }
 
-func adjustSourceCodePatternsWhitelistAll(sourceCodePatterns []string, whitelistAll bool) []string {
+func adjustSourceFilterPatternsWhitelistAll(sourceCodePatterns []string, whitelistAll bool) []string {
 	if whitelistAll {
 		return []string{""}
 	}
 
 	// build list of source code file patterns
 	return append(sourceCodePatterns, DefaultSourceCodeAllowedFilePatterns...)
 }
+
+func computeSourceExcludePatterns(context *Context, config NamedCodeScanConfig) []string {
+	var result []string
+	if slices.Contains(context.sechubConfig.SecretScan.Use, config.Name) {
+		// On secrets scan we add a bunch of exclude patterns (binaries, image files etc.)
+		result = DefaultSourceCodeUnwantedDirPatterns
+		result = append(result, DefaultSCMDirPatterns...)
+		result = append(result, DefaultSecretScanUnwantedFilePatterns...)
+	} else if slices.Contains(context.sechubConfig.CodeScan.Use, config.Name) {
+		result = DefaultSourceCodeExcludeDirPatterns
+	}
+	return result
+}