diff --git a/cmd/root.go b/cmd/root.go index 037700d..4af8ff1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,7 +34,9 @@ func InitializeFlags() { rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoKeywords, "no-keywords", false, "Don't search for built-in keywords") rootCmd.PersistentFlags().BoolVar(&app.GetFlags().ManyResults, "many-results", false, "Search >100 pages with filtering hack") rootCmd.PersistentFlags().BoolVar(&app.GetFlags().OnlyFiltered, "filtered-only", false, "Only print filtered results (language files)") + rootCmd.PersistentFlags().BoolVar(&app.GetFlags().AllResults, "all-results", false, "Print all results, even if they do not contain secrets") rootCmd.PersistentFlags().BoolVar(&app.GetFlags().JsonOutput, "json", false, "Print results in JSON format") + rootCmd.PersistentFlags().BoolVar(&app.GetFlags().FastMode, "fast", false, "Skip file grepping and only return search preview") rootCmd.PersistentFlags().IntVar(&app.GetFlags().Threads, "threads", 20, "Threads to dig with") rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoGists, "no-gists", false, "Don't search Gists") rootCmd.PersistentFlags().BoolVar(&app.GetFlags().NoRepos, "no-repos", false, "Don't search repos") diff --git a/internal/app/keyword_scan.go b/internal/app/keyword_scan.go index b4b54b8..8897f61 100644 --- a/internal/app/keyword_scan.go +++ b/internal/app/keyword_scan.go @@ -48,13 +48,31 @@ func ScanAndPrintResult(client *http.Client, repo RepoSearchResult) { if scannedRepos[repo.Repo] { return } - base := GetRawURLForSearchResult(repo) - defer SearchWaitGroup.Done() - data, err := DownloadRawFile(client, base, repo) - if err != nil { - log.Fatal(err) + var resultString string + if !GetFlags().FastMode { + base := GetRawURLForSearchResult(repo) + defer SearchWaitGroup.Done() + data, err := DownloadRawFile(client, base, repo) + if err != nil { + log.Fatal(err) + } + repo.Contents = string(data) } - resultString := string(data) + if GetFlags().AllResults { + if GetFlags().JsonOutput { + a, _ := json.Marshal(map[string]string{ + "repo": repo.Repo, + "file": repo.File, + "content": repo.Contents, + }) + fmt.Println(string(a)) + } else { + color.New(color.Faint).Println("[" + repo.Repo + "]") + color.New(color.Faint).Println("[" + repo.File + "]") + color.New(color.Faint).Println(repo.Contents) + } + } else { + // fmt.Println(resultString) matches, score := GetMatchesForString(resultString, repo) if repo.Source == "repo" && (GetFlags().DigCommits || GetFlags().DigRepo) && RepoIsUnpopular(client, repo) && score > -1 { scannedRepos[repo.Repo] = true @@ -96,6 +114,7 @@ func ScanAndPrintResult(client *http.Client, repo RepoSearchResult) { } } } + } } // MatchKeywords takes a string and checks if it contains sensitive information using pattern matching. @@ -117,10 +136,12 @@ func MatchKeywords(source string) (matches []Match) { } } } + // fmt.Println(source) // loop over regexes from database for _, regex := range GetFlags().TextRegexes.Rules { regexp := regex.Regex.RegExp matchStrings := regexp.FindAllString(source, -1) + // fmt.Println(matchStrings) for _, match := range matchStrings { shouldMatch := !regex.SmartFiltering if regex.SmartFiltering { diff --git a/internal/app/options.go b/internal/app/options.go index f5fc595..42db5f1 100644 --- a/internal/app/options.go +++ b/internal/app/options.go @@ -16,6 +16,8 @@ type Flags struct { NoFiles bool NoKeywords bool OnlyFiltered bool + AllResults bool + FastMode bool Threads int Debug bool LegacySearch bool diff --git a/internal/app/search.go b/internal/app/search.go index 45c6850..3ffd584 100644 --- a/internal/app/search.go +++ b/internal/app/search.go @@ -21,6 +21,7 @@ type RepoSearchResult struct { File string Raw string Source string + Contents string Query string URL string searchOptions *SearchOptions @@ -95,8 +96,6 @@ func Search(query string, client *http.Client) (results []RepoSearchResult, err // SearchGitHub searches GitHub code results for the given query func SearchGitHub(query string, options SearchOptions, client *http.Client, results *[]RepoSearchResult, resultSet map[string]bool) (err error) { - // TODO: A lot of this code is shared between GitHub and Gist searches, - // so we should rework the logic base := "" if GetFlags().GithubRepo { base = "https://github.com/" + query + "/search" @@ -117,8 +116,11 @@ func SearchGitHub(query string, options SearchOptions, client *http.Client, resu str := ConstructSearchURL(base, query, options) // fmt.Println(str) response, err := client.Get(str) + // fmt.Println(response.StatusCode) + // fmt.Println(err) if err != nil { if response != nil { + // fmt.Println(response.StatusCode) if response.StatusCode == 403 { response.Body.Close() delay += 5 @@ -137,6 +139,8 @@ func SearchGitHub(query string, options SearchOptions, client *http.Client, resu } responseData, err := ioutil.ReadAll(response.Body) responseStr := string(responseData) + // fmt.Println(responseStr) + if err != nil { log.Fatal(err) } @@ -214,15 +218,19 @@ func SearchGitHub(query string, options SearchOptions, client *http.Client, resu } resultSet[(result.RepoName + result.Path)] = true SearchWaitGroup.Add(1) + if !GetFlags().AllResults { + go ScanAndPrintResult(client, RepoSearchResult{ + Repo: result.RepoName, + File: result.Path, + Raw: result.RepoName + "/" + result.CommitSha + "/" + result.Path, + Source: "repo", + Query: query, + URL: "https://github.com/" + result.RepoName + "/blob/" + result.CommitSha + "/" + result.Path, + }) + } else { + + } // fmt.Println(result.RepoName + "/" + result.DefaultBranch + "/" + result.Path) - go ScanAndPrintResult(client, RepoSearchResult{ - Repo: result.RepoName, - File: result.Path, - Raw: result.RepoName + "/" + result.CommitSha + "/" + result.Path, - Source: "repo", - Query: query, - URL: "https://github.com/" + result.RepoName + "/blob/" + result.CommitSha + "/" + result.Path, - }) } } } else {