diff --git a/cmd/squealer/main.go b/cmd/squealer/main.go index 6d20742..a4c33a5 100644 --- a/cmd/squealer/main.go +++ b/cmd/squealer/main.go @@ -1,139 +1,14 @@ package main import ( - "fmt" - "math" - "os" - + "github.com/owenrumney/squealer/internal/app/squealer/cmd" log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/owenrumney/squealer/internal/app/squealer/formatters" - "github.com/owenrumney/squealer/internal/app/squealer/mertics" - "github.com/owenrumney/squealer/internal/app/squealer/scan" - "github.com/owenrumney/squealer/pkg/config" -) - -var rootcmd = &cobra.Command{ - Use: "squealer", - Short: "Search for secrets and squeal about them", - Long: `Telling tales on your secret leaking`, - Run: squeal, -} - -var ( - redacted = false - concise = false - noGit = false - debug = false - everything = false - configFilePath string - fromHash string - toHash string - commitListFile string - format string + "os" ) -func init() { - log.SetFormatter(&log.TextFormatter{}) - log.SetOutput(os.Stderr) - log.SetLevel(log.InfoLevel) -} - -func squeal(_ *cobra.Command, args []string) { - if concise { - log.SetLevel(log.FatalLevel) - } - - if debug { - log.SetLevel(log.DebugLevel) - } - - var basePath = "./" - if len(args) > 0 { - basePath = args[0] - } - cfg, err := config.LoadConfig(configFilePath) - if err != nil { - fail(err) - } - - scanner := getScanner(cfg, basePath) - transgressions, err := scanner.Scan() - if err != nil { - fail(err) - } - - output, err := formatters.GetFormatter(format).PrintTransgressions(transgressions, redacted) - if err != nil { - log.WithError(err).Error(err.Error()) - } - - fmt.Printf(output) - - metrics := scanner.GetMetrics() - if !concise { - _, _ = fmt.Fprint(os.Stderr, printMetrics(metrics)) - } - - exitCode := int(math.Min(float64(metrics.TransgressionsReported), 1)) - - log.Infof("Exit code: %d", exitCode) - os.Exit(exitCode) -} - -func getScanner(cfg *config.Config, basePath string) scan.Scanner { - scanner, err := scan.NewScanner(scan.ScannerConfig{ - Cfg: cfg, - Basepath: basePath, - Redacted: redacted, - NoGit: noGit, - FromHash: fromHash, - ToHash: toHash, - Everything: everything, - CommitListFile: commitListFile, - }) - if err != nil { - fail(err) - } - return scanner -} - -func printMetrics(metrics *mertics.Metrics) string { - duration, _ := metrics.Duration() - return fmt.Sprintf(` -Processing: - duration: %4.2fs - commits: %d - commit files: %d - -transgressionMap: - identified: %d - ignored: %d - reported: %d - -`, - duration, - metrics.CommitsProcessed, - metrics.FilesProcessed, - metrics.TransgressionsFound, - metrics.TransgressionsIgnored, - metrics.TransgressionsReported) -} - func main() { - rootcmd.PersistentFlags().BoolVar(&redacted, "redacted", redacted, "Display the results redacted.") - rootcmd.PersistentFlags().BoolVar(&concise, "concise", concise, "Reduced output.") - rootcmd.PersistentFlags().BoolVar(&noGit, "no-git", noGit, "Scan as a directory rather than a git history.") - rootcmd.PersistentFlags().BoolVar(&debug, "debug", debug, "Include debug output.") - rootcmd.PersistentFlags().BoolVar(&everything, "everything", everything, "Scan all commits.... everywhere.") - rootcmd.PersistentFlags().StringVar(&configFilePath, "config-file", configFilePath, "Path to the config file with the rules.") - rootcmd.PersistentFlags().StringVar(&fromHash, "from-hash", fromHash, "The hash to work back to from the starting hash.") - rootcmd.PersistentFlags().StringVar(&toHash, "to-hash", toHash, "The most recent hash to start with.") - rootcmd.PersistentFlags().StringVar(&format, "output-format", format, "The format that the output should come in (default, json, sarif.") - rootcmd.PersistentFlags().StringVar(&commitListFile, "commits-file", commitListFile, "Provide a file with the commits to check per line (git rev-list master..HEAD)") - if err := rootcmd.Execute(); err != nil { + if err := cmd.Root().Execute(); err != nil { fail(err) } } diff --git a/internal/app/squealer/cmd/root.go b/internal/app/squealer/cmd/root.go new file mode 100644 index 0000000..96e12f1 --- /dev/null +++ b/internal/app/squealer/cmd/root.go @@ -0,0 +1,142 @@ +package cmd + +import ( + "fmt" + "math" + "os" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/owenrumney/squealer/internal/pkg/formatters" + "github.com/owenrumney/squealer/internal/pkg/mertics" + "github.com/owenrumney/squealer/internal/pkg/scan" + "github.com/owenrumney/squealer/pkg/config" +) + +var ( + redacted = false + concise = false + noGit = false + debug = false + everything = false + configFilePath string + fromHash string + toHash string + commitListFile string + format string +) + +func Root() *cobra.Command { + rootCommand := &cobra.Command{ + Use: "squealer", + Short: "Search for secrets and squeal about them", + Long: `Telling tales on your secret leaking`, + RunE: squeal, + } + return rootCommand +} + +func init() { + + log.SetFormatter(&log.TextFormatter{}) + log.SetOutput(os.Stderr) + log.SetLevel(log.InfoLevel) +} + +func configureFlags(command *cobra.Command) { + command.PersistentFlags().BoolVar(&redacted, "redacted", redacted, "Display the results redacted.") + command.PersistentFlags().BoolVar(&concise, "concise", concise, "Reduced output.") + command.PersistentFlags().BoolVar(&noGit, "no-git", noGit, "Scan as a directory rather than a git history.") + command.PersistentFlags().BoolVar(&debug, "debug", debug, "Include debug output.") + command.PersistentFlags().BoolVar(&everything, "everything", everything, "Scan all commits.... everywhere.") + command.PersistentFlags().StringVar(&configFilePath, "config-file", configFilePath, "Path to the config file with the rules.") + command.PersistentFlags().StringVar(&fromHash, "from-hash", fromHash, "The hash to work back to from the starting hash.") + command.PersistentFlags().StringVar(&toHash, "to-hash", toHash, "The most recent hash to start with.") + command.PersistentFlags().StringVar(&format, "output-format", format, "The format that the output should come in (default, json, sarif.") + command.PersistentFlags().StringVar(&commitListFile, "commits-file", commitListFile, "Provide a file with the commits to check per line (git rev-list master..HEAD)") + +} + +func squeal(_ *cobra.Command, args []string) error { + if concise { + log.SetLevel(log.FatalLevel) + } + + if debug { + log.SetLevel(log.DebugLevel) + } + + var basePath = "./" + if len(args) > 0 { + basePath = args[0] + } + cfg, err := config.LoadConfig(configFilePath) + if err != nil { + return err + } + + scanner, err := getScanner(cfg, basePath) + if err != nil { + return err + } + transgressions, err := scanner.Scan() + if err != nil { + return err + } + + output, err := formatters.GetFormatter(format).PrintTransgressions(transgressions, redacted) + if err != nil { + log.WithError(err).Error(err.Error()) + } + + fmt.Printf(output) + + metrics := scanner.GetMetrics() + if !concise { + _, _ = fmt.Fprint(os.Stderr, printMetrics(metrics)) + } + + exitCode := int(math.Min(float64(metrics.TransgressionsReported), 1)) + os.Exit(exitCode) + return nil +} + +func getScanner(cfg *config.Config, basePath string) (scan.Scanner, error) { + scanner, err := scan.NewScanner(scan.ScannerConfig{ + Cfg: cfg, + Basepath: basePath, + Redacted: redacted, + NoGit: noGit, + FromHash: fromHash, + ToHash: toHash, + Everything: everything, + CommitListFile: commitListFile, + }) + if err != nil { + return nil, err + } + return scanner, nil +} + +func printMetrics(metrics *mertics.Metrics) string { + duration, _ := metrics.Duration() + return fmt.Sprintf(` +Processing: + duration: %4.2fs + commits: %d + commit files: %d + +transgressionMap: + identified: %d + ignored: %d + reported: %d + +`, + duration, + metrics.CommitsProcessed, + metrics.FilesProcessed, + metrics.TransgressionsFound, + metrics.TransgressionsIgnored, + metrics.TransgressionsReported) +} diff --git a/internal/app/squealer/formatters/default.go b/internal/pkg/formatters/default.go similarity index 91% rename from internal/app/squealer/formatters/default.go rename to internal/pkg/formatters/default.go index 3d17ca9..4f040cb 100644 --- a/internal/app/squealer/formatters/default.go +++ b/internal/pkg/formatters/default.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/owenrumney/squealer/internal/pkg/match" ) type DefaultFormatter struct { diff --git a/internal/app/squealer/formatters/default_test.go b/internal/pkg/formatters/default_test.go similarity index 93% rename from internal/app/squealer/formatters/default_test.go rename to internal/pkg/formatters/default_test.go index f118613..0dfa420 100644 --- a/internal/app/squealer/formatters/default_test.go +++ b/internal/pkg/formatters/default_test.go @@ -1,10 +1,11 @@ package formatters import ( - "github.com/stretchr/testify/assert" "testing" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/stretchr/testify/assert" + + "github.com/owenrumney/squealer/internal/pkg/match" ) func TestDefaultFormatterOutput(t *testing.T) { diff --git a/internal/app/squealer/formatters/formatter.go b/internal/pkg/formatters/formatter.go similarity index 83% rename from internal/app/squealer/formatters/formatter.go rename to internal/pkg/formatters/formatter.go index 92a8734..c7cd640 100644 --- a/internal/app/squealer/formatters/formatter.go +++ b/internal/pkg/formatters/formatter.go @@ -1,7 +1,7 @@ package formatters import ( - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/owenrumney/squealer/internal/pkg/match" ) type Formatter interface { diff --git a/internal/app/squealer/formatters/formatter_test.go b/internal/pkg/formatters/formatter_test.go similarity index 93% rename from internal/app/squealer/formatters/formatter_test.go rename to internal/pkg/formatters/formatter_test.go index 6045abc..3f25ffa 100644 --- a/internal/app/squealer/formatters/formatter_test.go +++ b/internal/pkg/formatters/formatter_test.go @@ -1,10 +1,11 @@ package formatters import ( - "github.com/stretchr/testify/assert" "testing" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/stretchr/testify/assert" + + "github.com/owenrumney/squealer/internal/pkg/match" ) func TestGetFormatter(t *testing.T) { diff --git a/internal/app/squealer/formatters/json.go b/internal/pkg/formatters/json.go similarity index 96% rename from internal/app/squealer/formatters/json.go rename to internal/pkg/formatters/json.go index a847ab4..488c3d7 100644 --- a/internal/app/squealer/formatters/json.go +++ b/internal/pkg/formatters/json.go @@ -3,7 +3,7 @@ package formatters import ( "encoding/json" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/owenrumney/squealer/internal/pkg/match" ) type JsonFormatter struct { diff --git a/internal/app/squealer/formatters/json_test.go b/internal/pkg/formatters/json_test.go similarity index 95% rename from internal/app/squealer/formatters/json_test.go rename to internal/pkg/formatters/json_test.go index d58e4a3..c836e11 100644 --- a/internal/app/squealer/formatters/json_test.go +++ b/internal/pkg/formatters/json_test.go @@ -1,10 +1,11 @@ package formatters import ( - "github.com/stretchr/testify/assert" "testing" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/stretchr/testify/assert" + + "github.com/owenrumney/squealer/internal/pkg/match" ) func TestJsonFormatterOutput(t *testing.T) { diff --git a/internal/app/squealer/formatters/sarif.go b/internal/pkg/formatters/sarif.go similarity index 96% rename from internal/app/squealer/formatters/sarif.go rename to internal/pkg/formatters/sarif.go index d65ccee..5f60daa 100644 --- a/internal/app/squealer/formatters/sarif.go +++ b/internal/pkg/formatters/sarif.go @@ -6,7 +6,7 @@ import ( "github.com/owenrumney/go-sarif/sarif" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/owenrumney/squealer/internal/pkg/match" ) type SarifFormatter struct { diff --git a/internal/app/squealer/formatters/sarif_test.go b/internal/pkg/formatters/sarif_test.go similarity index 98% rename from internal/app/squealer/formatters/sarif_test.go rename to internal/pkg/formatters/sarif_test.go index 1d08dad..999b046 100644 --- a/internal/app/squealer/formatters/sarif_test.go +++ b/internal/pkg/formatters/sarif_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/owenrumney/squealer/internal/pkg/match" ) func TestSarifFormmaterOutput(t *testing.T) { diff --git a/internal/app/squealer/match/matchers.go b/internal/pkg/match/matchers.go similarity index 83% rename from internal/app/squealer/match/matchers.go rename to internal/pkg/match/matchers.go index b449e43..18ec0b0 100644 --- a/internal/app/squealer/match/matchers.go +++ b/internal/pkg/match/matchers.go @@ -10,7 +10,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" log "github.com/sirupsen/logrus" - "github.com/owenrumney/squealer/internal/app/squealer/mertics" + "github.com/owenrumney/squealer/internal/pkg/mertics" "github.com/owenrumney/squealer/pkg/config" "github.com/owenrumney/squealer/pkg/result" ) @@ -18,6 +18,7 @@ import ( type Matcher struct { test *regexp.Regexp description string + fileFilter *regexp.Regexp } type Matchers []*Matcher @@ -50,13 +51,24 @@ func NewMatcherController(cfg *config.Config, metrics *mertics.Metrics, redacted } func (mc *MatcherController) add(rule config.MatchRule) error { - compile, err := regexp.Compile(rule.Rule) + compiledTest, err := regexp.Compile(rule.Rule) if err != nil { - return fmt.Errorf("failed to compile the regex. %v", err.Error()) + return fmt.Errorf("failed to compile the regex. %w", err) } + + var compiledFileFilter *regexp.Regexp + + if rule.FileFilter != "" { + compiledFileFilter, err = regexp.Compile(rule.FileFilter) + if err != nil { + return fmt.Errorf("failed to compile the file filter regex. %w", err) + } + } + mc.matchers = append(mc.matchers, &Matcher{ - test: compile, + test: compiledTest, description: rule.Description, + fileFilter: compiledFileFilter, }) return nil } @@ -64,6 +76,11 @@ func (mc *MatcherController) add(rule config.MatchRule) error { func (mc *MatcherController) Evaluate(filename, content string, commit *object.Commit) error { log.Debugf("\tfile: %s", filename) for _, matcher := range mc.matchers { + + if matcher.fileFilter != nil && !matcher.fileFilter.MatchString(filename) { + // only match transgressions where the file is valid + continue + } if matcher.test.MatchString(content) { mc.addTransgression(&content, filename, matcher, commit) } diff --git a/internal/app/squealer/match/transgression.go b/internal/pkg/match/transgression.go similarity index 100% rename from internal/app/squealer/match/transgression.go rename to internal/pkg/match/transgression.go diff --git a/internal/app/squealer/match/transgression_map.go b/internal/pkg/match/transgression_map.go similarity index 100% rename from internal/app/squealer/match/transgression_map.go rename to internal/pkg/match/transgression_map.go diff --git a/internal/app/squealer/match/transgression_map_test.go b/internal/pkg/match/transgression_map_test.go similarity index 100% rename from internal/app/squealer/match/transgression_map_test.go rename to internal/pkg/match/transgression_map_test.go diff --git a/internal/app/squealer/match/transgression_test.go b/internal/pkg/match/transgression_test.go similarity index 100% rename from internal/app/squealer/match/transgression_test.go rename to internal/pkg/match/transgression_test.go diff --git a/internal/app/squealer/mertics/metrics.go b/internal/pkg/mertics/metrics.go similarity index 100% rename from internal/app/squealer/mertics/metrics.go rename to internal/pkg/mertics/metrics.go diff --git a/internal/app/squealer/mertics/metrics_test.go b/internal/pkg/mertics/metrics_test.go similarity index 100% rename from internal/app/squealer/mertics/metrics_test.go rename to internal/pkg/mertics/metrics_test.go diff --git a/internal/app/squealer/scan/directory_scanner.go b/internal/pkg/scan/directory_scanner.go similarity index 91% rename from internal/app/squealer/scan/directory_scanner.go rename to internal/pkg/scan/directory_scanner.go index 3615f16..21003f2 100644 --- a/internal/app/squealer/scan/directory_scanner.go +++ b/internal/pkg/scan/directory_scanner.go @@ -5,8 +5,8 @@ import ( "os" "path/filepath" - "github.com/owenrumney/squealer/internal/app/squealer/match" - "github.com/owenrumney/squealer/internal/app/squealer/mertics" + "github.com/owenrumney/squealer/internal/pkg/match" + "github.com/owenrumney/squealer/internal/pkg/mertics" ) type directoryScanner struct { diff --git a/internal/app/squealer/scan/git_scanner.go b/internal/pkg/scan/git_scanner.go similarity index 98% rename from internal/app/squealer/scan/git_scanner.go rename to internal/pkg/scan/git_scanner.go index e7e1e2c..7c38291 100644 --- a/internal/app/squealer/scan/git_scanner.go +++ b/internal/pkg/scan/git_scanner.go @@ -11,8 +11,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" log "github.com/sirupsen/logrus" - "github.com/owenrumney/squealer/internal/app/squealer/match" - "github.com/owenrumney/squealer/internal/app/squealer/mertics" + "github.com/owenrumney/squealer/internal/pkg/match" + "github.com/owenrumney/squealer/internal/pkg/mertics" ) type CommitFile struct { diff --git a/internal/app/squealer/scan/scanner.go b/internal/pkg/scan/scanner.go similarity index 92% rename from internal/app/squealer/scan/scanner.go rename to internal/pkg/scan/scanner.go index f90ae24..e9d56f9 100644 --- a/internal/app/squealer/scan/scanner.go +++ b/internal/pkg/scan/scanner.go @@ -8,8 +8,8 @@ import ( log "github.com/sirupsen/logrus" - "github.com/owenrumney/squealer/internal/app/squealer/match" - "github.com/owenrumney/squealer/internal/app/squealer/mertics" + "github.com/owenrumney/squealer/internal/pkg/match" + "github.com/owenrumney/squealer/internal/pkg/mertics" "github.com/owenrumney/squealer/pkg/config" ) diff --git a/internal/app/squealer/scan/scanner_test.go b/internal/pkg/scan/scanner_test.go similarity index 100% rename from internal/app/squealer/scan/scanner_test.go rename to internal/pkg/scan/scanner_test.go diff --git a/internal/app/squealer/scan/signals_unix.go b/internal/pkg/scan/signals_unix.go similarity index 100% rename from internal/app/squealer/scan/signals_unix.go rename to internal/pkg/scan/signals_unix.go diff --git a/internal/app/squealer/scan/signals_windows.go b/internal/pkg/scan/signals_windows.go similarity index 100% rename from internal/app/squealer/scan/signals_windows.go rename to internal/pkg/scan/signals_windows.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 89c8230..d3a4cdc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -10,6 +10,8 @@ type Config struct { type MatchRule struct { Rule string `yaml:"rule" json:"rule"` Description string `yaml:"description" json:"description"` + FileFilter string `yaml:"file" json:"file"` + Entropy string } type RuleException struct { diff --git a/pkg/config/default.go b/pkg/config/default.go index 31108a5..416cfbd 100644 --- a/pkg/config/default.go +++ b/pkg/config/default.go @@ -40,9 +40,382 @@ func DefaultConfig() *Config { Description: "Slack OAuth Token", }, { - Rule: `(?im)password\s?[:=]\s?"?.+"?`, + Rule: `(?im)password\s?[:=]\s?"?[^\s?\$\{].+"?`, Description: "Password literal text", }, + { + Description: "AWS Manager ID", + Rule: `(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}`, + }, + { + Description: "AWS Manager ID", + Rule: `(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}`, + }, + { + Description: "AWS Secret Key", + Rule: `(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]`, + }, + { + Description: "AWS MWS key", + Rule: `amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`, + }, + { + Description: "Facebook Secret Key", + Rule: `(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]`, + }, + { + Description: "Facebook Client ID", + Rule: `(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]`, + }, + { + Description: "Twitter Secret Key", + Rule: `(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]`, + }, + { + Description: "Twitter Client ID", + Rule: `(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]`, + }, + { + Description: "Github", + Rule: `(?i)github.{0,3}((?i)token|api|key).{0,10}?(?-i)([0-9a-zA-Z]{35,40})`, + }, + { + Description: "LinkedIn Client ID", + Rule: `(?i)linkedin(.{0,20})?(?-i)['\"][0-9a-z]{12}['\"]`, + }, + { + Description: "LinkedIn Secret Key", + Rule: `(?i)linkedin(.{0,20})?['\"][0-9a-z]{16}['\"]`, + }, + { + Description: "NPM Token", + Rule: `//registry.npmjs.org/:_authToken=[0-9a-f-]+`, + }, + { + Description: "Slack", + Rule: `xox[baprs]-([0-9a-zA-Z]{10,48})?`, + }, + { + Description: "EC", + Rule: `-----BEGIN EC PRIVATE KEY-----`, + }, + { + Description: "DSA", + Rule: `-----BEGIN DSA PRIVATE KEY-----`, + }, + { + Description: "OPENSSH", + Rule: `-----BEGIN OPENSSH PRIVATE KEY-----`, + }, + { + Description: "RSA", + Rule: `-----BEGIN RSA PRIVATE KEY-----`, + }, + { + Description: "PGP", + Rule: `-----BEGIN PGP PRIVATE KEY BLOCK-----`, + }, + { + Description: "Google API key", + Rule: `AIza[0-9A-Za-z\\-_]{35}`, + }, + { + Description: "Heroku API key", + Rule: `(?i)heroku(.{0,20})?['"][0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['"]`, + }, + { + Description: "MailChimp API key", + Rule: `(?i)(mailchimp|mc)(.{0,20})?['"][0-9a-f]{32}-us[0-9]{1,2}['"]`, + }, + { + Description: "Mailgun API key", + Rule: `(?i)(mailgun|mg)(.{0,20})?['"][0-9a-z]{32}['"]`, + }, + { + Description: "PayPal Braintree access token", + Rule: `access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}`, + }, + { + Description: "Picatic API key", + Rule: `sk_live_[0-9a-z]{32}`, + }, + { + Description: "Slack Webhook", + Rule: `https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}`, + }, + { + Description: "Stripe API key", + Rule: `(?i)stripe(.{0,20})?['\"][sk|rk]_live_[0-9a-zA-Z]{24}`, + }, + { + Description: "Square access token", + Rule: `sq0atp-[0-9A-Za-z\-_]{22}`, + }, + { + Description: "Square OAuth secret", + Rule: `sq0csp-[0-9A-Za-z\\-_]{43}`, + }, + { + Description: "Twilio API key", + Rule: `(?i)twilio(.{0,20})?['\"][0-9a-f]{32}['\"]`, + }, + { + Description: "Dynatrace token", + Rule: `dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64}`, + }, + { + Description: "Shopify shared secret", + Rule: `shpss_[a-fA-F0-9]{32}`, + }, + { + Description: "Shopify access token", + Rule: `shpat_[a-fA-F0-9]{32}`, + }, + { + Description: "Shopify custom app access token", + Rule: `shpca_[a-fA-F0-9]{32}`, + }, + { + Description: "Shopify private app access token", + Rule: `shppa_[a-fA-F0-9]{32}`, + }, + { + Description: "Env Var", + Rule: `(?i)(apikey|secret|password|certificate_osx_p12|certificate_password|codacy_project_token|coveralls_api_token|coveralls_repo_token|coveralls_token|coverity_scan_token|cypress_record_key|database_password|db_password|deploy_password|deploy_token|digitalocean_access_token|docker_hub_password|docker_password|dockerhub_password|sonatype_password|firebase_api_token|firebase_token|firefox_secret|flask_secret_key|fossa_api_key|gcloud_service_key|gcr_password|gh_api_key|gh_next_oauth_client_secret|gh_next_unstable_oauth_client_secret|gh_oauth_client_secret|gpg_private_key|gpg_passphrase|gpg_secret_keys|heroku_api_key|okta_client_token|pypi_password|sonatype_nexus_password|travis_token|refresh_token|client_id|client_secret)(.*)?[(:=](\s)?['\"][0-9a-zA-Z-_!$^%=]{10,120}['\")]`, + Entropy: "4.2,7.0", + }, + { + Description: "Static key", + Rule: `(?i)(cookieParser)(.*)?[(](\s)?['\"][0-9a-zA-Z-_!$^%=]{5,20}['\")]`, + FileFilter: `\.(js|ts)$`, + }, + { + Description: "WP-Config", + Rule: `define(.{0,20})?(DB_CHARSET|NONCE_SALT|LOGGED_IN_SALT|AUTH_SALT|NONCE_KEY|DB_HOST|DB_PASSWORD|AUTH_KEY|SECURE_AUTH_KEY|LOGGED_IN_KEY|DB_NAME|DB_USER)(.{0,20})?['|"].{10,120}['|"]`, + }, + { + Description: "Shell history", + Rule: `.`, + FileFilter: `\.?(bash_|zsh_|z|mysql_|psql_|irb_)?history$`, + }, + { + Description: "Postgres password", + Rule: `.`, + FileFilter: `\.pgpass$`, + }, + { + Description: "OpenVPN", + Rule: `.`, + FileFilter: `\.ovpn$`, + }, + { + Description: "Unknown Key", + Rule: `.`, + FileFilter: `\.key$`, + }, + { + Description: "Keychain file", + Rule: `.`, + FileFilter: `\.(kdb|agilekeychain|keychain|kwallet|git-credentials|proftpdpasswd|dockercfg|credentials)$`, + }, + { + Description: "s3cfg", + Rule: `.`, + FileFilter: `\.s3cfg$`, + }, + { + Description: "secret token", + Rule: `.`, + FileFilter: `(omniauth|secret_token|carrierwave)\.rb$`, + }, + { + Description: "GitCredential", + Rule: `https?://.+:.+@.*`, + FileFilter: `\.gitCredentials$`, + }, + { + Description: "KeyStoreFile", + Rule: `.`, + FileFilter: `\.keystore$`, + }, + { + Description: "Base64EncodedCertificateInCode", + Rule: `['">;=]MII[a-z0-9/+]{200}`, + FileFilter: `\.(?:cs|ini|json|ps1|publishsettings|template|trd|ts|xml)$`, + }, + { + Description: "Base64EncodedCertificateInFile", + Rule: `MII[A-Za-z0-9/+]{60}`, + FileFilter: `\.(?:cert|cer)$`, + }, + { + Description: "PublishSettings", + Rule: `userPWD="[a-zA-Z0-9\+\/]{60}"`, + FileFilter: `(?i)(publishsettings|\.pubxml$)`, + }, + { + Description: "PemFile 1", + FileFilter: `\.pem$`, + Rule: `-{5}BEGIN(?: (?:[dr]sa|ec|openssh))? PRIVATE KEY-{5}`, + }, + { + Description: "AspNetMachineKeyInConfig 1", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `]+(?:decryptionKey\s*\=\s*"[a-fA-F0-9]{48,}|validationKey\s*\=\s*"[a-fA-F0-9]{48,})[^>]+>`, + }, + { + Description: "AspNetMachineKeyInConfig 2", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `(?:decryptionKey|validationKey)="[a-zA-Z0-9]+"`, + }, + { + Description: "SqlConnectionStringInConfig 1", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `(?i)(?:connection[sS]tring|connString)[^=]*=["'][^"']*[pP]assword\s*=\s*[^\s;][^"']*(?:'|")`, + }, + { + Description: "SqlConnectionStringInConfig 2 / CSCAN0043 SqlConnectionStringInCode", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties|policy_and_key\.hpp|AccountConfig\.h)$|hubot`, + Rule: `(?i)(?:User ID|uid|UserId).*(?:Password|[^a-z]pwd)=[^'\$%<@'";\[\{][^;/"]{4,128}(?:;|")`, + }, + { + Description: "StorageAccountKeyInConfig 1", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `[^a-z0-9/\+\._\-\$,\\][a-z0-9/+]{86}==`, + }, + { + Description: "StorageAccountKeyInCode 1", + FileFilter: `(?:\.(?:cs|js|ts|cpp)|policy_and_key\.hpp|AccountConfig\.h)$`, + Rule: `[^a-z0-9/\+\._\-\$,\\][a-z0-9/+]{86}==`, + }, + { + Description: "SharedAccessSignatureInCode 1", + FileFilter: `(?:\.(?:cs|js|ts|cpp)|policy_and_key\.hpp|AccountConfig\.h)$`, + Rule: `[^a-z0-9/\+\._\-\$,\\][a-z0-9/+]{43}=[^{@]`, + }, + { + Description: "SharedAccessSignatureInCode 2", + FileFilter: `(?:\.(?:cs|js|ts|cpp)|policy_and_key\.hpp|AccountConfig\.h)$`, + Rule: `[^a-z0-9/\+\._\-\$,\\][a-z0-9%]{43,53}%3d[^a-z0-9%]`, + }, + { + Description: "SharedAccessSignatureInConfig 1", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `[^a-z0-9/\+\._\-\$,\\][a-z0-9/+]{43}=[^{@]`, + }, + { + Description: "SharedAccessSignatureInConfig 2", + FileFilter: `\.(?:xml|pubxml|definitions|ps1|wadcfgx|ccf|config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `[^a-z0-9/\+\._\-\$,\\][a-z0-9%]{43,53}%3d[^a-z0-9%]`, + }, + { + Description: "GeneralSecretInConfig 1", + FileFilter: `\.(?:config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `]*/>`, + }, + { + Description: "GeneralSecretInConfig 2", + FileFilter: `\.(?:config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: ``, + }, + { + Description: "GeneralSecretInConfig 3", + FileFilter: `\.(?:config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `|[^>]*>.*?)`, + }, + { + Description: "GeneralSecretInConfig 4", + FileFilter: `\.(?:config|cscfg|json|js|txt|cpp|sql|dtsx|md|java|FF|template|settings|ini|BF|ste|isml|test|ts|resx|Azure|sample|backup|rd|hpp|psm1|cshtml|htm|bat|waz|yml|Beta|py|sh|m|php|xaml|keys|cmd|rds|loadtest|properties)$|hubot`, + Rule: `.+`, + }, + { + Description: "ScriptPassword 1", + FileFilter: `(?:\.cmd|\.ps|\.ps1|\.psm1)$`, + Rule: `\s-Password\s+(?:"[^"]*"|'[^']*')`, + }, + { + Description: "ScriptPassword 2", + FileFilter: `(?:\.cmd|\.ps|\.ps1|\.psm1)$`, + Rule: `\s-Password\s+[^$\(\)\[\{<\-\r?\n]+\s*(?:\r?\n|\-)`, + }, + { + Description: "ExternalApiSecret", + FileFilter: `\.cs$|\.cpp$|\.c$`, + Rule: `(private\sconst\sstring\sAccessTokenSecret|private\sconst\sstring\saccessToken|private\sconst\sstring\sconsumerSecret|private\sconst\sstring\sconsumerKey|pageAccessToken|private\sstring\stwilioAccountSid|private\sstring\stwilioAuthToken)\s=\s".*";`, + }, + { + Description: "DefaultPasswordContexts 1", + FileFilter: `\.(?:ps1|psm1|)$`, + Rule: `ConvertTo-SecureString(?:\s*-String)?\s*"[^"\r?\n]+"`, + }, + { + Description: "DefaultPasswordContexts 2", + FileFilter: `\.(?:cs|xml|config|json|ts|cfg|txt|ps1|bat|cscfg|publishsettings|cmd|psm1|aspx|asmx|vbs|added_cluster|clean|pubxml|ccf|ini|svd|sql|c|xslt|csv|FF|ExtendedTests|settings|cshtml|template|trd|argfile)$|(config|certificate|publish|UT)\.js$|(commands|user|tests)\.cpp$`, + Rule: `new\sX509Certificate2\([^()]*,\s*"[^"\r?\n]+"[^)]*\)`, + }, + { + Description: "DefaultPasswordContexts 3", + FileFilter: `\.(?:cs|xml|config|json|ts|cfg|txt|ps1|bat|cscfg|publishsettings|cmd|psm1|aspx|asmx|vbs|added_cluster|clean|pubxml|ccf|ini|svd|sql|c|xslt|csv|FF|ExtendedTests|settings|cshtml|template|trd|argfile)$|(config|certificate|publish|UT)\.js$|(commands|user|tests)\.cpp$`, + Rule: `AdminPassword\s*=\s*"[^"\r?\n]+"`, + }, + { + Description: "DefaultPasswordContexts 4", + FileFilter: `\.(?:cs|xml|config|json|ts|cfg|txt|ps1|bat|cscfg|publishsettings|cmd|psm1|aspx|asmx|vbs|added_cluster|clean|pubxml|ccf|ini|svd|sql|c|xslt|csv|FF|ExtendedTests|settings|cshtml|template|trd|argfile)$|(config|certificate|publish|UT)\.js$|(commands|user|tests)\.cpp$`, + Rule: `(?i).+`, + }, + { + Description: "DefaultPasswordContexts 5", + FileFilter: `\.(?:cs|xml|config|json|ts|cfg|txt|ps1|bat|cscfg|publishsettings|cmd|psm1|aspx|asmx|vbs|added_cluster|clean|pubxml|ccf|ini|svd|sql|c|xslt|csv|FF|ExtendedTests|settings|cshtml|template|trd|argfile)$|(config|certificate|publish|UT)\.js$|(commands|user|tests)\.cpp$`, + Rule: `ClearTextPassword"?\s*[:=]\s*"[^"\r?\n]+"`, + }, + { + Description: "DefaultPasswordContexts 6", + FileFilter: `\.(?:cs|xml|config|json|ts|cfg|txt|ps1|bat|cscfg|publishsettings|cmd|psm1|aspx|asmx|vbs|added_cluster|clean|pubxml|ccf|ini|svd|sql|c|xslt|csv|FF|ExtendedTests|settings|cshtml|template|trd|argfile)$|(config|certificate|publish|UT)\.js$|(commands|user|tests)\.cpp$`, + Rule: `certutil.*?\-p\s+("[^"%]+"|'[^'%]+'|[^"']\S*\s)`, + }, + { + Description: "DefaultPasswordContexts 7", + FileFilter: `\.(?:cs|xml|config|json|cfg|txt|ps1|bat|cscfg|publishsettings|cmd|psm1|aspx|asmx|vbs|added_cluster|clean|pubxml|ccf|ini|svd|sql|c|xslt|csv|FF|ExtendedTests|settings|cshtml|template|trd|argfile)$|(config|certificate|publish|UT)\.js$|(commands|user|tests)\.cpp$`, + Rule: `password\s*=\s*N?(["][^"\r?\n]{4,}["]|['][^'\r?\n]{4,}['])`, + }, + { + Description: "DomainPassword", + Rule: `new(?:-object)?\s+System.Net.NetworkCredential\(?:.*?,\s*"[^"]+"`, + FileFilter: `\.cs$|\.c$|\.cpp$|\.ps1$|\.ps$|\.cmd$|\.bat$|\.log$|\.psd$|\.psm1$`, + }, + { + Description: "VstsPersonalAccessToken 1", + FileFilter: `\.(?:cs|ps1|bat|config|xml|json)$`, + Rule: `(?i)(?:AccessToken|pat|token).*?[':="][a-z0-9]{52}(?:'|"|\s|[\r?\n]+)`, + }, + { + Description: "VstsPersonalAccessToken 1", + FileFilter: `\.(?:cs|ps1|bat|config|xml|json)$`, + Rule: `(?i)(?:AccessToken|pat|token).*?[':="][a-z0-9/+]{70}==(?:'|"|\s|[\r?\n]+)`, + }, + { + Description: "OAuthToken 1", + FileFilter: `\.(?:config|js|json|txt|cs|xml|java|py)$`, + Rule: `eyj[a-z0-9\-_%]+\.eyj[a-z0-9\-_%]+\.[a-z0-9\-_%]+`, + }, + { + Description: "OAuthToken 2", + FileFilter: `\.(?:config|js|json|txt|cs|xml|java|py)$`, + Rule: `refresh_token["']?\s*[:=]\s*["']?(?:[a-z0-9_]+-)+[a-z0-9_]+["']?`, + }, + { + Description: "AnsibleVault", + FileFilter: `\.yml$`, + Rule: `\$ANSIBLE_VAULT;[0-9]\.[0-9];AES256[\r?\n]+[0-9]+`, + }, + { + Description: "SlackToken 1", + Rule: `xoxp-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+-[a-z0-9]+`, + }, + { + Description: "SlackToken 2", + Rule: `xoxb-[a-z0-9]+-[a-z0-9]+`, + }, }, IgnorePaths: []string{ "vendor", diff --git a/pkg/squealer/string_scanner.go b/pkg/squealer/string_scanner.go index 22772b7..1fbc8e0 100644 --- a/pkg/squealer/string_scanner.go +++ b/pkg/squealer/string_scanner.go @@ -1,7 +1,7 @@ package squealer import ( - "github.com/owenrumney/squealer/internal/app/squealer/match" + "github.com/owenrumney/squealer/internal/pkg/match" "github.com/owenrumney/squealer/pkg/config" "github.com/owenrumney/squealer/pkg/result" ) diff --git a/tests/repo_test.go b/tests/repo_test.go index f1d7bba..207b6c4 100644 --- a/tests/repo_test.go +++ b/tests/repo_test.go @@ -3,7 +3,7 @@ package tests import ( "testing" - "github.com/owenrumney/squealer/internal/app/squealer/scan" + "github.com/owenrumney/squealer/internal/pkg/scan" "github.com/owenrumney/squealer/pkg/config" "github.com/stretchr/testify/assert" )