Detectors that fail verification should still report the unverified s…

…ecret (#440) * Detectors that fail verification should still report the unverified secret * fixup - change microsoft webhook keywords, filter false positives for old github detector * fixup - fix typo
trufflesecurity · Apr 21, 2022 · b574327 · b574327
1 parent 913c75d
commit b574327
Show file tree

Hide file tree

Showing 21 changed files with 237 additions and 331 deletions.
diff --git a/pkg/detectors/dropbox/dropbox.go b/pkg/detectors/dropbox/dropbox.go
@@ -6,10 +6,9 @@ import (
 	"net/http"
 	"regexp"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
 type Scanner struct{}
@@ -53,23 +52,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
 			res, err := client.Do(req)
-			if err != nil {
-				return results, err
-			}
-			defer res.Body.Close()
-
-			// 200 means good key for get current user
-			// 400 is bad (malformed)
-			// 403 bad scope
-			if res.StatusCode == http.StatusOK {
-				s.Verified = true
+			if err == nil {
+				res.Body.Close() // The request body is unused.
+
+				// 200 means good key for get current user
+				// 400 is bad (malformed)
+				// 403 bad scope
+				if res.StatusCode == http.StatusOK {
+					s.Verified = true
+				}
 			}
 		}
 
-		if !s.Verified {
-			if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+			continue
 		}
 
 		results = append(results, s)

diff --git a/pkg/detectors/elasticemail/elasticemail.go b/pkg/detectors/elasticemail/elasticemail.go
@@ -2,14 +2,11 @@ package elasticemail
 
 import (
 	"context"
-	// "log"
-	"regexp"
-	"strings"
-
-	// "fmt"
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -56,37 +53,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			res, err := client.Do(req)
-			if err != nil {
-				continue
-			}
-			defer res.Body.Close()
-			var byteData []byte
-			_, err = res.Body.Read(byteData)
-			if err != nil {
-				continue
-			}
-
-			defer res.Body.Close()
-			data, readErr := ioutil.ReadAll(res.Body)
-			if readErr != nil {
-				continue
-			}
-			var ResVar struct {
-				Success bool `json:"success"`
-			}
-			if err := json.Unmarshal(data, &ResVar); err != nil {
-				continue
-			}
-			if ResVar.Success {
-				s1.Verified = true
-			} else {
-
-				if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-					continue
+			if err == nil {
+				data, readErr := io.ReadAll(res.Body)
+				res.Body.Close()
+				if readErr == nil {
+					var ResVar struct {
+						Success bool `json:"success"`
+					}
+					if err := json.Unmarshal(data, &ResVar); err == nil {
+						if ResVar.Success {
+							s1.Verified = true
+						}
+					}
 				}
 			}
 		}
 
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
 		results = append(results, s1)
 	}
 	return detectors.CleanResults(results), nil

diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.go
@@ -73,23 +73,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			req.Header.Add("Content-Type", "application/json; charset=utf-8")
 			req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
 			res, err := client.Do(req)
-			if err != nil {
-				break
-			}
-			defer res.Body.Close()
-			if res.StatusCode >= 200 && res.StatusCode < 300 {
+			if err == nil {
 				var userResponse userRes
 				err = json.NewDecoder(res.Body).Decode(&userResponse)
+				res.Body.Close()
 				if err == nil {
 					s.Verified = true
 				}
 			}
 		}
 
-		if !s.Verified {
-			if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+			continue
 		}
 
 		results = append(results, s)

diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.go
@@ -75,21 +75,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			req.Header.Add("Content-Type", "application/json; charset=utf-8")
 			req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
 			res, err := client.Do(req)
-			if err != nil {
-				break
-			}
-			defer res.Body.Close()
-			if res.StatusCode >= 200 && res.StatusCode < 300 {
-				var userResponse userRes
-				err = json.NewDecoder(res.Body).Decode(&userResponse)
-				if err == nil {
-					s.Verified = true
+			if err == nil {
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					var userResponse userRes
+					err = json.NewDecoder(res.Body).Decode(&userResponse)
+					res.Body.Close()
+					if err == nil {
+						s.Verified = true
+					}
 				}
 			}
 		}
 
+		if !s.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
+			continue
+		}
+
 		results = append(results, s)
 	}
 
-	return
+	return detectors.CleanResults(results), nil
 }
diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.go
@@ -6,10 +6,9 @@ import (
 	"net/http"
 	"regexp"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
 type Scanner struct{}
@@ -59,27 +58,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
 			res, err := client.Do(req)
-			if err != nil {
-				return results, err
-			}
-			defer res.Body.Close()
-
-			// 200 means good key and has `read_user` scope
-			// 403 means good key but not the right scope
-			// 401 is bad key
-			if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-				secret.Verified = true
+			if err == nil {
+				res.Body.Close() // The request body is unused.
+
+				// 200 means good key and has `read_user` scope
+				// 403 means good key but not the right scope
+				// 401 is bad key
+				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+					secret.Verified = true
+				}
 			}
 		}
 
-		if !secret.Verified {
-			if detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
+		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+			continue
 		}
 
 		results = append(results, secret)
 	}
 
-	return
+	return detectors.CleanResults(results), nil
 }
diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.go
@@ -6,10 +6,9 @@ import (
 	"net/http"
 	"regexp"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
 
 type Scanner struct{}
@@ -59,23 +58,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
 			res, err := client.Do(req)
-			if err != nil {
-				return results, err
-			}
-			defer res.Body.Close()
-
-			// 200 means good key and has `read_user` scope
-			// 403 means good key but not the right scope
-			// 401 is bad key
-			if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
-				secret.Verified = true
+			if err == nil {
+				res.Body.Close() // The request body is unused.
+
+				// 200 means good key and has `read_user` scope
+				// 403 means good key but not the right scope
+				// 401 is bad key
+				if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+					secret.Verified = true
+				}
 			}
 		}
 
-		if !secret.Verified {
-			if detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
+		if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+			continue
 		}
 
 		results = append(results, secret)

diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go
@@ -45,29 +45,26 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			Redacted:     redact,
 		}
 
-		if verify {
-			//TODO can this be verified? Possibly. Could triage verification to other DBMS strings
-			s.Verified = false
-			// client := common.SaneHttpClient()
-			// req, err := http.NewRequestWithContext(ctx, "GET", "https://jdbcci.com/api/v2/me", nil)
-			if err != nil {
-				continue
-			}
-			// req.Header.Add("Accept", "application/json;")
-			// req.Header.Add("Jdbc-Token", token)
-			// res, err := client.Do(req)
-			// if err != nil {
-			// 	break
-			// }
-			// if res.StatusCode >= 200 && res.StatusCode < 300 {
-			// 	s.Verified = true
-			// }
-		}
+		//if verify {
+		//	// TODO: can this be verified? Possibly. Could triage verification to other DBMS strings
+		//	s.Verified = false
+		//	client := common.SaneHttpClient()
+		//	req, err := http.NewRequestWithContext(ctx, "GET", "https://jdbcci.com/api/v2/me", nil)
+		//	if err != nil {
+		//		continue
+		//	}
+		//	req.Header.Add("Accept", "application/json;")
+		//	req.Header.Add("Jdbc-Token", token)
+		//	res, err := client.Do(req)
+		//	if err == nil {
+		//		if res.StatusCode >= 200 && res.StatusCode < 300 {
+		//			s.Verified = true
+		//		}
+		//	}
+		//}
 
-		if !s.Verified {
-			if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
-				continue
-			}
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+			continue
 		}
 
 		results = append(results, s)

diff --git a/pkg/detectors/mailchimp/mailchimp.go b/pkg/detectors/mailchimp/mailchimp.go
@@ -54,22 +54,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.SetBasicAuth("anystring", match)
 			res, err := client.Do(req)
-			if err != nil {
-				break
-			}
-			defer res.Body.Close()
-			if res.StatusCode == 200 {
-				s.Verified = true
-			} else {
-				s.Verified = false
-			}
+			if err == nil {
+				res.Body.Close() // The request body is unused.
 
+				if res.StatusCode == 200 {
+					s.Verified = true
+				}
+			}
 		}
 
-		if !s.Verified {
-			if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
-				continue
-			}
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+			continue
 		}
 
 		results = append(results, s)