v1.0.X-performance

Hardened (#159) - performance, sync, path filtering and whitelisting improvements

cmd/api.go

@@ -168,7 +168,7 @@ func processWithRegex(policy Policy, data []byte, rgPath string) error {
 }
 func executeAssureForAPI(policy Policy, rgPath, filePath string) (bool, error) {
  // Create a temporary file to store the search patterns
- searchPatternFile, err := createSearchPatternFile(policy.Regex)
+ searchPatternFile, err := createSearchPatternFile(policy.Regex, NormalizeFilename(policy.ID))
  if err != nil {
  return false, fmt.Errorf("error creating search pattern file: %w", err)
  }

cmd/assure.go

@@ -6,6 +6,7 @@ import (
  "os"
  "os/exec"
  "path/filepath"
+ "sync"
 )
 
 // ProcessAssureType handles the assurance process for policies of type "assure"
@@ -23,7 +24,7 @@ func ProcessAssureType(policy Policy, rgPath string, targetDir string, filePaths
 
 func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure []string) error {
  // Create a temporary file to store the search patterns
- searchPatternFile, err := createSearchPatternFile(policy.Regex)
+ searchPatternFile, err := createSearchPatternFile(policy.Regex, NormalizeFilename(policy.ID))
  if err != nil {
  log.Error().Err(err).Msg("error creating search pattern file")
  return fmt.Errorf("error creating search pattern file: %w", err)
@@ -49,6 +50,8 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
  writer := bufio.NewWriter(jsonoutfile)
  defer writer.Flush()
 
+ processedIgnorePatterns := processIgnorePatterns(policyData.Config.Flags.Ignore)
+
  codePatternAssureJSON := []string{
  "--pcre2",
  "--no-heading",
@@ -64,39 +67,24 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
  return fmt.Errorf("no target directory defined")
  }
 
- // Append the file targets
- if len(filesToAssure) > 0 {
- codePatternAssureJSON = append(codePatternAssureJSON, filesToAssure...)
- } else if policy.FilePattern == "" {
- codePatternAssureJSON = append(codePatternAssureJSON, targetDir)
+ codePatternAssureJSON = append(codePatternAssureJSON, processedIgnorePatterns...)
+
+ matchesFound := true
+
+ // Parallel execution for large file sets
+ if policy.FilePattern == "" {
+ log.Warn().Str("policy", policy.ID).Msg("ASSURE Policy without a filepattern is suboptimal")
+ }
+ if len(filesToAssure) > 25 {
+ matchesFound, err = executeParallelAssure(rgPath, codePatternAssureJSON, filesToAssure, writer)
  } else {
- return fmt.Errorf("no files matched policy pattern")
+ matchesFound, err = executeSingleAssure(rgPath, codePatternAssureJSON, filesToAssure, targetDir, policy, writer)
  }
 
- // Execute the ripgrep command for JSON output
- cmdJSON := exec.Command(rgPath, codePatternAssureJSON...)
- cmdJSON.Stdout = writer
- cmdJSON.Stderr = os.Stderr
+ if err != nil {
 
- log.Debug().Msgf("Creating JSON output for assure policy %s... ", policy.ID)
- err = cmdJSON.Run()
+ log.Error().Err(err).Msg("error executing ripgrep batch")
 
- // Check if ripgrep found any matches
- matchesFound := true
- if err != nil {
- if exitError, ok := err.(*exec.ExitError); ok {
- // Exit code 1 in ripgrep means "no matches found"
- if exitError.ExitCode() == 1 {
- matchesFound = false
- err = nil // Reset error as this is the expected outcome for assure
- } else {
- log.Error().Err(err).Msg("error executing ripgrep for JSON output")
- return fmt.Errorf("error executing ripgrep for JSON output: %w", err)
- }
- } else {
- log.Error().Err(err).Msg("error executing ripgrep for JSON output")
- return fmt.Errorf("error executing ripgrep for JSON output: %w", err)
- }
  }
 
  // Patch the JSON output file
@@ -107,6 +95,7 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
  }
 
  log.Debug().Msgf("JSON output for assure policy %s written to: %s ", policy.ID, jsonOutputFile)
+ log.Debug().Msgf("Scanned ~%d files for policy %s", len(filesToAssure), policy.ID)
 
  // Determine the status based on whether matches were found
  status := "NOT FOUND"
@@ -143,3 +132,115 @@ func executeAssure(policy Policy, rgPath string, targetDir string, filesToAssure
 
  return nil
 }
+
+func executeParallelAssure(rgPath string, baseArgs []string, filesToScan []string, writer *bufio.Writer) (bool, error) {
+
+ const batchSize = 25
+ matched := true
+ var wg sync.WaitGroup
+ errChan := make(chan error, len(filesToScan)/batchSize+1)
+ var mu sync.Mutex
+
+ for i := 0; i < len(filesToScan); i += batchSize {
+ end := i + batchSize
+ if end > len(filesToScan) {
+ end = len(filesToScan)
+ }
+ batch := filesToScan[i:end]
+
+ // log.Debug().Msgf("RGM: %v", batch)
+
+ wg.Add(1)
+ go func(batch []string) {
+ defer wg.Done()
+ args := append(baseArgs, batch...)
+ cmd := exec.Command(rgPath, args...)
+ output, err := cmd.Output()
+
+ if err != nil {
+
+ if exitError, ok := err.(*exec.ExitError); ok {
+ // Exit code 1 in ripgrep means "no matches found"
+ if exitError.ExitCode() == 1 {
+ matched = false
+ err = nil // Reset error as this is the expected outcome for assure
+ }
+ }
+
+ if exitError, ok := err.(*exec.ExitError); ok && exitError.ExitCode() != 1 {
+ errChan <- fmt.Errorf("error executing ripgrep: %w", err)
+ return
+ }
+ }
+
+ mu.Lock()
+ _, writeErr := writer.Write(output)
+ if writeErr == nil {
+ writeErr = writer.Flush()
+ }
+ mu.Unlock()
+
+ if writeErr != nil {
+ errChan <- fmt.Errorf("error writing output: %w", writeErr)
+ }
+ }(batch)
+ }
+
+ wg.Wait()
+ close(errChan)
+
+ for err := range errChan {
+ if err != nil {
+ return matched, err
+ }
+ }
+
+ return matched, nil
+}
+
+func executeSingleAssure(rgPath string, baseArgs []string, filesToScan []string, targetDir string, policy Policy, writer *bufio.Writer) (bool, error) {
+
+ if len(filesToScan) > 0 {
+ baseArgs = append(baseArgs, filesToScan...)
+ } else {
+ log.Error().Str("policy", policy.ID).Msgf("no files matched policy pattern on target : %s", targetDir)
+ }
+
+ matched := true
+
+ // log.Debug().Msgf("RGS: %v", baseArgs)
+
+ cmdJSON := exec.Command(rgPath, baseArgs...)
+ cmdJSON.Stdout = writer
+ cmdJSON.Stderr = os.Stderr
+
+ log.Debug().Msgf("Creating JSON output for policy %s... ", policy.ID)
+ err := cmdJSON.Run()
+ if err != nil {
+ if exitError, ok := err.(*exec.ExitError); ok {
+
+ if exitError.ExitCode() == 1 {
+ matched = false
+ err = nil // Reset error as this is the expected outcome for assure
+ }
+
+ if exitError.ExitCode() == 2 {
+ log.Warn().Msgf("RG exited with code 2")
+ log.Debug().Msgf("RG Error Args: %v", baseArgs)
+ if len(exitError.Stderr) > 0 {
+ log.Debug().Msgf("RG exited with code 2 stderr: %s", string(exitError.Stderr))
+ }
+ }
+ if exitError.ExitCode() != 1 {
+ log.Error().Err(err).Msg("error executing ripgrep for JSON output")
+ return matched, fmt.Errorf("error executing ripgrep for JSON output: %w", err)
+ }
+
+ } else {
+ log.Error().Err(err).Msg("error executing ripgrep for JSON output")
+ return matched, fmt.Errorf("error executing ripgrep for JSON output: %w", err)
+ }
+ }
+
+ return matched, nil
+}

cmd/audit.go

@@ -209,9 +209,11 @@ func processPolicy(policy Policy, allFileInfos []FileInfo, rgPath string) {
 
  if policy.Type == "json" || policy.Type == "yaml" || policy.Type == "ini" || policy.Type == "scan" || policy.Type == "assure" {
 
- log.Debug().Msgf(" Processing files for policy %s: ", policy.ID)
- for _, file := range filesToProcess {
- log.Debug().Msgf(" %s: %s ", file.Path, file.Hash)
+ log.Debug().Str("policy", policy.ID).Msgf(" Processing files for policy %s ", policy.ID)
+ if len(filesToProcess) < 15 {
+ for _, file := range filesToProcess {
+ log.Debug().Str("policy", policy.ID).Msgf(" %s: %s ", file.Path, file.Hash)
+ }
  }
 
  normalizedID := NormalizeFilename(policy.ID)

cmd/aux.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+ "bufio"
  "crypto/sha256"
  "encoding/hex"
  "encoding/json"
@@ -22,29 +23,40 @@ import (
  "gopkg.in/yaml.v3"
 )
 
-// patchJSONOutputFile reads the ripgrep JSON output, patches it to create valid JSON, and writes it back to the file
 func patchJSONOutputFile(filePath string) error {
- // Read the contents of the file
- content, err := os.ReadFile(filePath)
+ // Open the file
+ file, err := os.Open(filePath)
  if err != nil {
- return fmt.Errorf("error reading JSON file: %v", err)
+ return fmt.Errorf("error opening JSON file: %v", err)
  }
+ defer file.Close()
 
- // Split the input into separate JSON objects
- objects := strings.Split(string(content), "}\n{")
+ var validObjects []map[string]interface{}
 
- // Add the missing brackets to create a JSON array
- jsonArray := "[" + strings.Join(objects, "},\n{") + "]"
+ // Read the file line by line
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" {
+ continue
+ }
 
- // Parse the JSON array to validate it
- var parsed []interface{}
- err = json.Unmarshal([]byte(jsonArray), &parsed)
- if err != nil {
- return fmt.Errorf("error parsing JSON: %v", err)
+ // Try to parse each line as a separate JSON object
+ var obj map[string]interface{}
+ if err := json.Unmarshal([]byte(line), &obj); err == nil {
+ validObjects = append(validObjects, obj)
+ } else {
+ // Log the error and skip this line
+ fmt.Printf("Skipping invalid JSON line: %s\n", line)
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return fmt.Errorf("error reading file: %v", err)
  }
 
- // Re-marshal the parsed data to get a properly formatted JSON string
- validJSON, err := json.MarshalIndent(parsed, "", " ")
+ // Marshal the array of objects into a properly formatted JSON string
+ validJSON, err := json.MarshalIndent(validObjects, "", " ")
  if err != nil {
  return fmt.Errorf("error marshaling JSON: %v", err)
  }
@@ -334,7 +346,7 @@ func createOutputDirectories(isObserve bool) error {
  for _, dir := range dirs {
  if outputDir != "" {
  dir = filepath.Join(outputDir, dir)
- log.Debug().Msgf("Creating directory: %s", dir)
+ // log.Debug().Msgf("Creating directory: %s", dir)
 
  }
  if err := os.MkdirAll(dir, 0755); err != nil {
@@ -350,7 +362,7 @@ func cleanupOutputDirectories() error {
  if outputDir != "" {
  for i, dir := range dirsToClean {
  dirsToClean[i] = filepath.Join(outputDir, dir)
- log.Debug().Msgf("Cleaning up directories: %v", dirsToClean)
+ // log.Debug().Msgf("Cleaning up directories: %v", dirsToClean)
  }
  }
  var wg sync.WaitGroup
@@ -367,7 +379,7 @@ func cleanupOutputDirectories() error {
  return
  }
 
- log.Debug().Msgf("Cleaned up directory: %s ", d)
+ // log.Debug().Msgf("Cleaned up directory: %s ", d)
  }(dir)
  }