feat: Add openai check (#88)

- Add colour to output
- Add openai secret check
- Add a gif to Readme
- Tidy up check list

Signed-off-by: Owen Rumney <owen@owenrumney.co.uk>

.github/workflows/build.yml

@@ -16,11 +16,8 @@ jobs:
     - uses: actions/setup-go@v2
       with:
         stable: 'false'
-        go-version: '1.17.0' # The Go version to download (if necessary) and use.
+        go-version: '1.20.0' # The Go version to download (if necessary) and use.
     - run: go version
 
     - name: Run test
-      run: make test
-
-    - name: Check quality
-      run: make quality
+      run: make test

.github/workflows/releases.yml

@@ -12,10 +12,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v3
       with:
         stable: 'false'
-        go-version: '1.17.0' # The Go version to download (if necessary) and use.
+        go-version: '1.20.0' # The Go version to download (if necessary) and use.
     - run: go version
 
     - name: Login to Docker Hub

Makefile

@@ -20,5 +20,5 @@ image:
 
 .PHONY: quality
 quality:
-	which golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.0
+	which golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.0
 	golangci-lint run --timeout 3m --verbose

README.md

@@ -8,37 +8,61 @@
 [![Github Release](https://img.shields.io/github/release/owenrumney/squealer.svg)](https://github.com/owenrumney/squealer/releases)
 [![GitHub All Releases](https://img.shields.io/github/downloads/owenrumney/squealer/total)](https://github.com/owenrumney/squealer/releases)
 
-Squealer scans a git repository or filesystem for secrets that are being leaked deep within the commit history. 
-
-The built-in configuration has the following checks;
-
-AWS
-- access key id
-- access secret key
-
-Github
-- github token
-
-Slack
-- slack token OAUTH
-- webhook url
-
-
-Other
-- Asymmetric Private Key
-
-Sometimes we have secrets committed to our projects, generally we can invalidate them and move on. If squealer is telling tales about a secret that you are aware of and has been mitigated, you can use the `exception` rule found in the output to register it as ignored.
+Squealer scans a git repository or filesystem for secrets that are being leaked deep within the commit history.
+
+![Squealer](.github/images/squealer.gif)
+
+The built-in configuration includes the following checks;
+
+- AnsibleVault
+- AWS Manager ID
+- AWS MWS key
+- AWS Secret Key
+- Base64 Encoded Certificates
+- DomainPassword
+- DSA
+- Dynatrace token
+- Facebook Creds
+- GitCredential
+- Github
+- Google API key
+- Heroku API key
+- Keychain file
+- KeyStoreFile
+- LinkedIn Creds
+- MailChimp API key
+- Mailgun API key
+- NPM Token
+- OAuth Token
+- OpenAI Secret Key
+- OPENSSH
+- OpenVPN
+- Password literal text
+- PayPal Braintree access token
+- PGP
+- Picatic API key
+- Postgres password
+- PublishSettings
+- RSA
+- Shopify credentials
+- Slack credentials
+- SQL Connection Strings
+- Square credentials
+- Stripe API key
+- Twilio API key
+- Twitter credentials
+
+Sometimes we have secrets committed to our projects, generally, we can invalidate them and move on. If Squealer is telling tales about a secret that you are aware of and has been mitigated, you can use the `exception` rule found in the output to register it as ignored.
 
 ## Installation
 
 ```bash
 curl -s "https://raw.githubusercontent.com/owenrumney/squealer/main/scripts/install.sh" | bash
 ```
 
-
 ## Usage
 
-Squealer is intended to be run either locally or as part of a CI process. 
+Squealer is intended to be run either locally or as part of a CI process.
 
 ```shell
 ./squealer --help
@@ -60,41 +84,53 @@ Flags:
       --to-hash string         The most recent hash to start with.
 ```
 
+### Scan remote repositories
+
+```shell
+squealer git@github.com:owenrumney/woopsie.git
+```
+
+### Scan local repositories as a directory
+
+```shell
+squealer --no-git /path/to/repo
+```
+
 ### Config File
 
 ```yaml
 rules:
-- rule: (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}
-  description: Check for AWS Access Key Id
-- rule: (?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]
-  description: Check for AWS Secret Access Key
-- rule: (?i)github[_\-\.]?token[\s:,="\]']+?(?-i)[0-9a-zA-Z]{35,40}
-  description: Check for Github Token 
-- rule: https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}
-  description: Check for Slack webhook
-- rule: xox[baprs]-([0-9a-zA-Z]{10,48})?
-  description: Check for Slack token
-- rule: '-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'
-  description: Check for Private Asymetric Key
+  - rule: (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}
+    description: Check for AWS Access Key Id
+  - rule: (?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]
+    description: Check for AWS Secret Access Key
+  - rule: (?i)github[_\-\.]?token[\s:,="\]']+?(?-i)[0-9a-zA-Z]{35,40}
+    description: Check for Github Token
+  - rule: https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}
+    description: Check for Slack webhook
+  - rule: xox[baprs]-([0-9a-zA-Z]{10,48})?
+    description: Check for Slack token
+  - rule: "-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----"
+    description: Check for Private Asymetric Key
 ignore_paths:
-- vendor
-- node_modules
+  - vendor
+  - node_modules
 ignore_extensions:
-- .zip
-- .png
-- .jpg
-- .pdf
-- .xls
-- .doc
-- .docx
+  - .zip
+  - .png
+  - .jpg
+  - .pdf
+  - .xls
+  - .doc
+  - .docx
 exceptions:
-- exception: release/update.go:D2IDetI6aidl58GE6dv5uAaWmXM=
-  reason: This is a webhook that we got rid of - can be ignored in this file
+  - exception: release/update.go:D2IDetI6aidl58GE6dv5uAaWmXM=
+    reason: This is a webhook that we got rid of - can be ignored in this file
 ```
 
 ### Config breakdown
 
-The config file is made up of the `rules`, `ignore_prefixes`, `ignore_extensions` and `exceptions`. 
+The config file is made up of the `rules`, `ignore_prefixes`, `ignore_extensions` and `exceptions`.
 
 #### rules
 
@@ -106,7 +142,7 @@ Ignore paths are folders that you don't want to look ing - generally `vendor` an
 
 #### ignore_extensions
 
-Ignore extensions have the file types that won't be scanned. Binaries are automatically ignored.
+Ignore extensions that have these file types that won't be scanned. Binaries are automatically ignored.
 
 #### exceptions
 
@@ -153,7 +189,6 @@ INFO[0002] Exit code: 1
 
 It's worth noting that these are known because they're examples in the documentation for tfsec - I can add them to the `config.yaml` as exclusions y using the `Exclude rule`
 
-
 ## Using as a library
 
 Squealer can be used for scanning a specific string using either the default config or by passing your own file in.
@@ -173,7 +208,7 @@ package main
 
 import (
 	"fmt"
-	
+
 	"github.com/owenrumney/squealer/pkg/squealer"
 	"github.com/owenrumney/squealer/pkg/config"
 )
@@ -196,7 +231,7 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	
+
 	for _, t := range transgressions {
 		fmt.Printf("%s[%d]\n", t.Filename, t.LineNo)
     }
@@ -207,7 +242,6 @@ func main() {
 
 #### String Scanning
 
-
 ```go
 package main
 
@@ -231,7 +265,6 @@ func main() {
 
 ```
 
-
 ## Credits
 
 [Image by Derangedmisfit](https://derangedmisfit.newgrounds.com/)

go.mod

@@ -1,10 +1,11 @@
 module github.com/owenrumney/squealer
 
-go 1.17
+go 1.20
 
 require (
 	github.com/go-git/go-billy/v5 v5.4.1
 	github.com/go-git/go-git/v5 v5.6.1
+	github.com/liamg/tml v0.6.0
 	github.com/owenrumney/go-sarif v1.1.1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.7.0

go.sum

@@ -51,6 +51,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/liamg/tml v0.6.0 h1:yOC/Q9p9Io3J11U9LdYVIwpRTnTE1GPMNFLrygkmE2Y=
+github.com/liamg/tml v0.6.0/go.mod h1:0h4EAV/zBOsqI91EWONedjRpO8O0itjGJVd+wG5eC+E=
 github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
 github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
@@ -79,6 +81,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/app/squealer/cmd/root.go

@@ -98,7 +98,7 @@ Processing:
   commits:      %d
   commit files: %d
 
-transgressionMap:
+Transgressions:
   identified:   %d
   ignored:      %d
   reported:     %d

internal/pkg/formatters/default.go

@@ -1,9 +1,9 @@
 package formatters
 
 import (
-	"fmt"
 	"strings"
 
+	"github.com/liamg/tml"
 	"github.com/owenrumney/squealer/internal/pkg/match"
 )
 
@@ -18,16 +18,16 @@ func (d DefaultFormatter) PrintTransgressions(transgressions []match.Transgressi
 		if redacted {
 			content = t.RedactedContent
 		}
-		builder.Write([]byte(fmt.Sprintf(`
-Match Description: │ %s
-Content:           │ %s
-Filename:          │ %s
-Line No:           │ %d
-Secret Hash:       │ %s
-Commit:            │ %s
-Committer:         │ %s (%s)
-Committed:         │ %s
-Exclude rule:      │ %s
+		builder.Write([]byte(tml.Sprintf(`
+<blue>Match Description:</blue> <yellow>│</yellow> %s
+<blue>Content:</blue>           <yellow>|</yellow> <red>%s</red>
+<blue>Filename:</blue>          <yellow>|</yellow> %s
+<blue>Line No:</blue>           <yellow>|</yellow> %d
+<blue>Secret Hash:</blue>       <yellow>|</yellow> %s
+<blue>Commit:</blue>            <yellow>|</yellow> %s
+<blue>Committer:</blue>         <yellow>|</yellow> %s (%s)
+<blue>Committed:</blue>         <yellow>|</yellow> %s
+<blue>Exclude rule:</blue>      <yellow>|</yellow> %s
 `, t.MatchDescription, content, t.Filename, t.LineNo, t.Hash, t.CommitHash, t.Committer, t.CommitterEmail, t.Committed, t.ExcludeRule)))
 	}
 	return builder.String(), nil

internal/pkg/match/transgression_map.go

@@ -18,7 +18,7 @@ func newTransgressions() *transgressionMap {
 
 func (t *transgressionMap) add(key string, transgression Transgression) {
 	t.Lock()
-	existing := t.internal[key];
+	existing := t.internal[key]
 	if existing == nil {
 		t.internal[key] = &transgression
 		t.counter += 1