options to scan all results

cmd/root.go

@@ -34,7 +34,9 @@ func InitializeFlags() {
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoKeywords, "no-keywords", false, "Don't search for built-in keywords")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().ManyResults, "many-results", false, "Search >100 pages with filtering hack")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().OnlyFiltered, "filtered-only", false, "Only print filtered results (language files)")
+	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().AllResults, "all-results", false, "Print all results, even if they do not contain secrets")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().JsonOutput, "json", false, "Print results in JSON format")
+	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().FastMode, "fast", false, "Skip file grepping and only return search preview")
 	rootCmd.PersistentFlags().IntVar(&app.GetFlags().Threads, "threads", 20, "Threads to dig with")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoGists, "no-gists", false, "Don't search Gists")
 	rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoRepos, "no-repos", false, "Don't search repos")

internal/app/keyword_scan.go

@@ -48,13 +48,31 @@ func ScanAndPrintResult(client *http.Client, repo RepoSearchResult) {
 	if scannedRepos[repo.Repo] {
 		return
 	}
-	base := GetRawURLForSearchResult(repo)
-	defer SearchWaitGroup.Done()
-	data, err := DownloadRawFile(client, base, repo)
-	if err != nil {
-		log.Fatal(err)
+	var resultString string
+	if !GetFlags().FastMode {
+		base := GetRawURLForSearchResult(repo)
+		defer SearchWaitGroup.Done()
+		data, err := DownloadRawFile(client, base, repo)
+		if err != nil {
+			log.Fatal(err)
+		}
+		repo.Contents = string(data)
 	}
-	resultString := string(data)
+	if GetFlags().AllResults {
+		if GetFlags().JsonOutput {
+			a, _ := json.Marshal(map[string]string{
+				"repo":    repo.Repo,
+				"file":	repo.File,
+				"content":  repo.Contents,
+			})
+			fmt.Println(string(a))
+		} else {
+			color.New(color.Faint).Println("[" + repo.Repo + "]")
+			color.New(color.Faint).Println("[" + repo.File + "]")
+			color.New(color.Faint).Println(repo.Contents)
+		}
+	} else {
+	// fmt.Println(resultString)
 	matches, score := GetMatchesForString(resultString, repo)
 	if repo.Source == "repo" && (GetFlags().DigCommits || GetFlags().DigRepo) && RepoIsUnpopular(client, repo) && score > -1 {
 		scannedRepos[repo.Repo] = true
@@ -96,6 +114,7 @@ func ScanAndPrintResult(client *http.Client, repo RepoSearchResult) {
 			}
 		}
 	}
+	}
 }
 
 // MatchKeywords takes a string and checks if it contains sensitive information using pattern matching.
@@ -117,10 +136,12 @@ func MatchKeywords(source string) (matches []Match) {
 			}
 		}
 	}
+	// fmt.Println(source)
 	// loop over regexes from database
 	for _, regex := range GetFlags().TextRegexes.Rules {
 		regexp := regex.Regex.RegExp
 		matchStrings := regexp.FindAllString(source, -1)
+		// fmt.Println(matchStrings)
 		for _, match := range matchStrings {
 			shouldMatch := !regex.SmartFiltering
 			if regex.SmartFiltering {

internal/app/options.go

@@ -16,6 +16,8 @@ type Flags struct {
 	NoFiles       bool
 	NoKeywords    bool
 	OnlyFiltered  bool
+	AllResults     bool
+	FastMode       bool
 	Threads       int
 	Debug         bool
 	LegacySearch  bool

internal/app/search.go

@@ -21,6 +21,7 @@ type RepoSearchResult struct {
 	File          string
 	Raw           string
 	Source        string
+	Contents        string
 	Query         string
 	URL           string
 	searchOptions *SearchOptions
@@ -95,8 +96,6 @@ func Search(query string, client *http.Client) (results []RepoSearchResult, err
 
 // SearchGitHub searches GitHub code results for the given query
 func SearchGitHub(query string, options SearchOptions, client *http.Client, results *[]RepoSearchResult, resultSet map[string]bool) (err error) {
-	// TODO: A lot of this code is shared between GitHub and Gist searches,
-	// so we should rework the logic
 	base := ""
 	if GetFlags().GithubRepo {
 		base = "https://github.com/" + query + "/search"
@@ -117,8 +116,11 @@ func SearchGitHub(query string, options SearchOptions, client *http.Client, resu
 				str := ConstructSearchURL(base, query, options)
 				// fmt.Println(str)
 				response, err := client.Get(str)
+				// fmt.Println(response.StatusCode)
+				// fmt.Println(err)
 				if err != nil {
 					if response != nil {
+						// fmt.Println(response.StatusCode)
 						if response.StatusCode == 403 {
 							response.Body.Close()
 							delay += 5
@@ -137,6 +139,8 @@ func SearchGitHub(query string, options SearchOptions, client *http.Client, resu
 				}
 				responseData, err := ioutil.ReadAll(response.Body)
 				responseStr := string(responseData)
+				// fmt.Println(responseStr)
+
 				if err != nil {
 					log.Fatal(err)
 				}
@@ -214,15 +218,19 @@ func SearchGitHub(query string, options SearchOptions, client *http.Client, resu
 							}
 							resultSet[(result.RepoName + result.Path)] = true
 							SearchWaitGroup.Add(1)
+							if !GetFlags().AllResults {
+								go ScanAndPrintResult(client, RepoSearchResult{
+									Repo:   result.RepoName,
+									File:   result.Path,
+									Raw:    result.RepoName + "/" + result.CommitSha + "/" + result.Path,
+									Source: "repo",
+									Query:  query,
+									URL:    "https://github.com/" + result.RepoName + "/blob/" + result.CommitSha + "/" + result.Path,
+								})	
+							} else {
+
+							}
 							// fmt.Println(result.RepoName + "/" + result.DefaultBranch + "/" + result.Path)
-							go ScanAndPrintResult(client, RepoSearchResult{
-								Repo:   result.RepoName,
-								File:   result.Path,
-								Raw:    result.RepoName + "/" + result.CommitSha + "/" + result.Path,
-								Source: "repo",
-								Query:  query,
-								URL:    "https://github.com/" + result.RepoName + "/blob/" + result.CommitSha + "/" + result.Path,
-							})
 						}	
 					}
 				} else {