diff --git a/go.mod b/go.mod index a3ceb04fe02e..28d1b1a4921a 100644 --- a/go.mod +++ b/go.mod @@ -31,13 +31,13 @@ require ( github.com/joho/godotenv v1.4.0 github.com/jpillora/overseer v1.1.6 github.com/kylelemons/godebug v1.1.0 - github.com/mattn/go-colorable v0.1.12 github.com/paulbellamy/ratecounter v0.2.0 github.com/pkg/errors v0.9.1 github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2 github.com/rs/zerolog v1.26.1 github.com/sergi/go-diff v1.2.0 github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.0 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 github.com/xanzy/go-gitlab v0.64.0 github.com/zricethezav/gitleaks/v8 v8.5.2 @@ -48,6 +48,7 @@ require ( google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf google.golang.org/protobuf v1.28.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 + gopkg.in/h2non/gock.v1 v1.1.2 ) require ( @@ -71,6 +72,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect github.com/aws/smithy-go v1.11.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect @@ -84,6 +86,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect github.com/googleapis/gax-go/v2 v2.2.0 // indirect + github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -91,11 +94,13 @@ require ( github.com/jpillora/s3 v1.1.4 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 // indirect @@ -110,4 +115,5 @@ require ( google.golang.org/grpc v1.45.0 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 3e6fa7ee4db1..88770615ca63 100644 --- a/go.sum +++ b/go.sum @@ -296,6 +296,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -391,6 +393,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -930,6 +934,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 8c20e8a63241..6e70559471da 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -11,13 +11,13 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/go-errors/errors" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v42/github" - "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "golang.org/x/oauth2" "golang.org/x/sync/semaphore" @@ -37,8 +37,8 @@ import ( type Source struct { name string - sourceId int64 - jobId int64 + sourceID int64 + jobID int64 verify bool repos []string orgs []string @@ -64,11 +64,11 @@ func (s *Source) Type() sourcespb.SourceType { } func (s *Source) SourceID() int64 { - return s.sourceId + return s.sourceID } func (s *Source) JobID() int64 { - return s.jobId + return s.jobID } func (s *Source) Token(ctx context.Context, installationClient *github.Client) (string, error) { @@ -94,13 +94,13 @@ func (s *Source) Token(ctx context.Context, installationClient *github.Client) ( } // Init returns an initialized GitHub source. -func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error { +func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error { s.log = log.WithField("source", s.Type()).WithField("name", name) s.aCtx = aCtx s.name = name - s.sourceId = sourceId - s.jobId = jobId + s.sourceID = sourceID + s.jobID = jobID s.verify = verify s.jobSem = semaphore.NewWeighted(int64(concurrency)) @@ -333,16 +333,27 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err } func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error { - scanned := 0 + var scanned uint64 log.Debugf("Found %v total repos to scan", len(s.repos)) wg := sync.WaitGroup{} - errs := []error{} - var errsMut sync.Mutex + errs := make(chan error, 1) + reportErr := func(err error) { + // save the error if there's room, otherwise log and drop it + select { + case errs <- err: + default: + log.WithError(err).Warn("dropping error") + } + } + for i, repoURL := range s.repos { if err := s.jobSem.Acquire(ctx, 1); err != nil { + // Acquire blocks until it can acquire the semaphore or returns an + // error if the context is finished log.WithError(err).Debug("could not acquire semaphore") - continue + reportErr(err) + break } wg.Add(1) go func(ctx context.Context, repoURL string, i int) { @@ -367,10 +378,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch var token string token, err = s.Token(ctx, installationClient) if err != nil { - // TODO: maybe we can use a channel here - errsMut.Lock() - errs = append(errs, err) - errsMut.Unlock() + reportErr(err) return } path, repo, err = git.CloneRepoUsingToken(token, repoURL, "clone") @@ -391,20 +399,20 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch if err != nil { log.WithError(err).Errorf("unable to scan repo, continuing") } - // TODO: use atomic library - scanned++ - logrus.Debugf("scanned %d/%d repos", scanned, len(s.repos)) + atomic.AddUint64(&scanned, 1) + log.Debugf("scanned %d/%d repos", scanned, len(s.repos)) }(ctx, repoURL, i) } wg.Wait() // This only returns first error which is what we did prior to concurrency - if len(errs) > 0 { - return errs[0] + select { + case err := <-errs: + return err + default: + return nil } - - return nil } // handleRateLimit returns true if a rate limit was handled @@ -443,7 +451,8 @@ func handleRateLimit(errIn error, res *github.Response) bool { return true } -func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error { +func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, org string) ([]string, error) { + repos := []string{} opts := &github.RepositoryListByOrgOptions{ ListOptions: github.ListOptions{ PerPage: 100, @@ -459,7 +468,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or continue } if err != nil { - return fmt.Errorf("could not list repos for org %s: %w", org, err) + return nil, fmt.Errorf("could not list repos for org %s: %w", org, err) } if len(someRepos) == 0 { break @@ -472,7 +481,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or continue } } - common.AddStringSliceItem(r.GetCloneURL(), &s.repos) + repos = append(repos, r.GetCloneURL()) } if res.NextPage == 0 { break @@ -480,10 +489,23 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or opts.Page = res.NextPage } log.WithField("org", org).Debugf("Found %d repos (%d forks)", numRepos, numForks) + return repos, nil +} + +func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error { + repos, err := s.getReposByOrg(ctx, apiClient, org) + if err != nil { + return err + } + // add the repos to the set of repos + for _, repo := range repos { + common.AddStringSliceItem(repo, &s.repos) + } return nil } -func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error { +func (s *Source) getReposByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) { + repos := []string{} opts := &github.RepositoryListOptions{ ListOptions: github.ListOptions{ PerPage: 50, @@ -498,23 +520,36 @@ func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, u continue } if err != nil { - return fmt.Errorf("could not list repos for user %s: %w", user, err) + return nil, fmt.Errorf("could not list repos for user %s: %w", user, err) } for _, r := range someRepos { if r.GetFork() && !s.conn.IncludeForks { continue } - common.AddStringSliceItem(r.GetCloneURL(), &s.repos) + repos = append(repos, r.GetCloneURL()) } if res.NextPage == 0 { break } opts.Page = res.NextPage } + return repos, nil +} + +func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error { + repos, err := s.getReposByUser(ctx, apiClient, user) + if err != nil { + return err + } + // add the repos to the set of repos + for _, repo := range repos { + common.AddStringSliceItem(repo, &s.repos) + } return nil } -func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) { +func (s *Source) getGistsByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) { + gistURLs := []string{} gistOpts := &github.GistListOptions{} for { gists, resp, err := apiClient.Gists.List(ctx, user, gistOpts) @@ -525,21 +560,33 @@ func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, u continue } if err != nil { - log.WithError(err).Warnf("Could not get gists for user %s", user) + log.WithError(err).Warnf("could not list repos for user %s", user) + return nil, fmt.Errorf("could not list repos for user %s: %w", user, err) } for _, gist := range gists { - common.AddStringSliceItem(gist.GetGitPullURL(), &s.repos) + gistURLs = append(gistURLs, gist.GetGitPullURL()) } if resp == nil || resp.NextPage == 0 { break } gistOpts.Page = resp.NextPage } - return + return gistURLs, nil } -func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error { +func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) error { + gists, err := s.getGistsByUser(ctx, apiClient, user) + if err != nil { + return err + } + // add the gists to the set of repos + for _, gist := range gists { + common.AddStringSliceItem(gist, &s.repos) + } + return nil +} +func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error { opts := &github.ListOptions{ PerPage: 500, } @@ -650,24 +697,34 @@ func (s *Source) addOrgsByUser(ctx context.Context, apiClient *github.Client, us func (s *Source) normalizeRepos(ctx context.Context, apiClient *github.Client) { // TODO: Add check/fix for repos that are missing scheme - repoIter := make([]string, len(s.repos)) - copy(repoIter, s.repos) - for _, repo := range repoIter { - if parts := strings.Split(repo, "/"); len(parts) == 1 { - origSources := len(s.repos) - s.addGistsByUser(ctx, apiClient, repo) - if err := s.addReposByUser(ctx, apiClient, repo); err != nil { - log.WithError(err).Error("error fetching repos by user") - } - if origSources != len(s.repos) { - common.RemoveStringSliceItem(repo, &s.repos) + normalizedRepos := map[string]struct{}{} + for _, repo := range s.repos { + // if there's a '/', assume it's a URL and try to normalize it + if strings.ContainsRune(repo, '/') { + repoNormalized, err := giturl.NormalizeGithubRepo(repo) + if err != nil { + log.WithError(err).Warnf("Repo not in expected format: %s", repo) continue } + normalizedRepos[repoNormalized] = struct{}{} + continue } - repoNormalized, err := giturl.NormalizeGithubRepo(repo) - if err != nil { - log.WithError(err).Warnf("Repo not in expected format: %s", repo) + // otherwise, assume it's a user and enumerate repositories and gists + if repos, err := s.getReposByUser(ctx, apiClient, repo); err == nil { + for _, repo := range repos { + normalizedRepos[repo] = struct{}{} + } + } + if gists, err := s.getGistsByUser(ctx, apiClient, repo); err == nil { + for _, gist := range gists { + normalizedRepos[gist] = struct{}{} + } } - common.AddStringSliceItem(repoNormalized, &s.repos) + } + + // replace s.repos + s.repos = s.repos[:0] + for key := range normalizedRepos { + s.repos = append(s.repos, key) } } diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go new file mode 100644 index 000000000000..78477c627326 --- /dev/null +++ b/pkg/sources/github/github_integration_test.go @@ -0,0 +1,587 @@ +//go:build integration +// +build integration + +package github + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-github/v42/github" + + "github.com/kylelemons/godebug/pretty" + "github.com/mattn/go-colorable" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +func TestSource_Scan(t *testing.T) { + os.Setenv("DO_NOT_RANDOMIZE", "true") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + + secret, err := common.GetTestSecret(ctx) + if err != nil { + t.Fatal(fmt.Errorf("failed to access secret: %v", err)) + } + + // For the personal access token test + githubToken := secret.MustGetField("GITHUB_TOKEN") + + //For the NEW github app test (+Member enum) + githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW") + githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New) + if err != nil { + t.Fatal(err) + } + githubPrivateKeyNew := string(githubPrivateKeyBytesNew) + githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") + githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") + + //OLD app for breaking app change tests + // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY") + // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64) + // if err != nil { + // t.Fatal(err) + // } + // githubPrivateKey := string(githubPrivateKeyBytes) + // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID") + // githubAppID := secret.MustGetField("GITHUB_APP_ID") + + type init struct { + name string + verify bool + connection *sourcespb.GitHub + } + tests := []struct { + name string + init init + wantChunk *sources.Chunk + wantErr bool + minRepo int + minOrg int + }{ + { + name: "token authenticated, single repo", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://github.com/dustin-decker/secretsandstuff.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + }, + { + name: "token authenticated, single repo, no .git", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://github.com/dustin-decker/secretsandstuff.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + }, + { + name: "token authenticated, single org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Organizations: []string{"trufflesecurity"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 0, + }, + { + name: "token authenticated, username in org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Organizations: []string{"dustin-decker"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 0, + }, + { + name: "token authenticated, username in repo", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"dustin-decker"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 0, + }, + { + name: "token authenticated, org in repo", + // I do not think that this is a supported case, but adding the test to specify there is no requirement. + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"trufflesecurity"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 0, + minOrg: 0, + }, + /* + { + name: "token authenticated, no org or user (enum)", + // This configuration currently will only find gists from the user. No repos or orgs will be scanned. + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 0, + minOrg: 0, + }, + { + name: "app authenticated (old), no repo or org (enum)", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + ScanUsers: false, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKey, + InstallationId: githubInstallationID, + AppId: githubAppID, + }, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 0, + }, + */ + { + name: "unauthenticated, single org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Organizations: []string{"trufflesecurity"}, + Credential: &sourcespb.GitHub_Unauthenticated{}, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 1, + }, + { + name: "unauthenticated, single repo", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, + Credential: &sourcespb.GitHub_Unauthenticated{}, + }, + }, + wantChunk: &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://github.com/trufflesecurity/driftwood.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + }, + /* + { + name: "app authenticated, no repo or org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + ScanUsers: true, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 0, + }, + */ + { + name: "app authenticated, single repo", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + }, + }, + wantChunk: &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://github.com/trufflesecurity/driftwood.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + minRepo: 1, + minOrg: 0, + }, + { + name: "app authenticated, single org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Organizations: []string{"trufflesecurity"}, + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 3, + minOrg: 1, + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + log.Debugf("Beginning test %d: %s", i, tt.name) + s := Source{} + + log.SetLevel(log.DebugLevel) + //uncomment for windows Testing + log.SetFormatter(&log.TextFormatter{ForceColors: true}) + log.SetOutput(colorable.NewColorableStdout()) + + conn, err := anypb.New(tt.init.connection) + if err != nil { + t.Fatal(err) + } + + err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4) + if (err != nil) != tt.wantErr { + t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr) + return + } + chunksCh := make(chan *sources.Chunk, 5) + go func() { + err = s.Chunks(ctx, chunksCh) + if (err != nil) != tt.wantErr { + if ctx.Err() != nil { + return + } + t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr) + return + } + }() + if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil { + t.Error(err) + } + }) + } +} + +func TestSource_paginateGists(t *testing.T) { + + os.Setenv("DO_NOT_RANDOMIZE", "true") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + secret, err := common.GetTestSecret(ctx) + if err != nil { + t.Fatal(fmt.Errorf("failed to access secret: %v", err)) + } + //For the NEW github app test (+Member enum) + githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW") + githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New) + if err != nil { + t.Fatal(err) + } + githubPrivateKeyNew := string(githubPrivateKeyBytesNew) + githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") + githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") + type init struct { + name string + verify bool + connection *sourcespb.GitHub + } + tests := []struct { + name string + init init + wantChunk *sources.Chunk + wantErr bool + user string + minRepos int + }{ + { + name: "get gist secret", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + }, + }, + wantChunk: &sources.Chunk{ + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + user: "dustin-decker", + minRepos: 1, + }, + { + name: "get multiple pages of gists", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + }, + }, + wantChunk: nil, + wantErr: false, + user: "andrew", + minRepos: 101, + }, + /* { + name: "get multiple pages of gists", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: githubPrivateKeyNew, + InstallationId: githubInstallationIDNew, + AppId: githubAppIDNew, + }, + }, + }, + }, + wantChunk: &sources.Chunk{ + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + user: "andrew", + }, + */ + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Source{} + + log.SetLevel(log.DebugLevel) + //uncomment for windows Testing + log.SetFormatter(&log.TextFormatter{ForceColors: true}) + log.SetOutput(colorable.NewColorableStdout()) + + conn, err := anypb.New(tt.init.connection) + if err != nil { + t.Fatal(err) + } + + err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4) + if (err != nil) != tt.wantErr { + t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr) + return + } + chunksCh := make(chan *sources.Chunk, 5) + go func() { + s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user) + chunksCh <- &sources.Chunk{} + }() + var wantedRepo string + if tt.wantChunk != nil { + wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository + } + if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil { + t.Error(err) + } + }) + } +} + +func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc { + return func(chunk *sources.Chunk) error { + if minRepos != 0 && minRepos > len(s.repos) { + return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos)) + } + if expected != "" { + for _, repo := range s.repos { + if repo == expected { + return nil + } + } + return fmt.Errorf("expected repo not included: %s", expected) + } + return nil + } +} + +func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc { + return func(chunk *sources.Chunk) error { + if minOrg != 0 && minOrg > len(s.orgs) { + return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs)) + } + if minRepo != 0 && minRepo > len(s.repos) { + return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos)) + } + if wantChunk != nil { + if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" { + return nil + } + return common.MatchError + } + return nil + } +} + +// func TestSource_paginateRepos(t *testing.T) { +// type args struct { +// ctx context.Context +// apiClient *github.Client +// } +// tests := []struct { +// name string +// org string +// args args +// }{ +// { +// org: "fakeNetflix", +// args: args{ +// ctx: context.Background(), +// apiClient: github.NewClient(common.SaneHttpClient()), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// s := &Source{httpClient: common.SaneHttpClient()} +// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org) +// if len(s.repos) < 101 { +// t.Errorf("expected > 100 repos, got %d", len(s.repos)) +// } +// }) +// } +// } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 45207d90e3d5..d4899cf57817 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -1,584 +1,339 @@ package github import ( + "bytes" "context" - "encoding/base64" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "fmt" - "os" + "net/http" + "sort" + "strconv" "testing" "time" "github.com/google/go-github/v42/github" - - "github.com/kylelemons/godebug/pretty" - "github.com/mattn/go-colorable" - log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/anypb" + "gopkg.in/h2non/gock.v1" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" - "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) -func TestSource_Scan(t *testing.T) { - os.Setenv("DO_NOT_RANDOMIZE", "true") - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) - defer cancel() - - secret, err := common.GetTestSecret(ctx) +func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) { + s := &Source{} + conn, err := anypb.New(src) if err != nil { - t.Fatal(fmt.Errorf("failed to access secret: %v", err)) + panic(err) } + return s, conn +} - // For the personal access token test - githubToken := secret.MustGetField("GITHUB_TOKEN") - - //For the NEW github app test (+Member enum) - githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW") - githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New) - if err != nil { - t.Fatal(err) - } - githubPrivateKeyNew := string(githubPrivateKeyBytesNew) - githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") - githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") - - //OLD app for breaking app change tests - // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY") - // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64) - // if err != nil { - // t.Fatal(err) - // } - // githubPrivateKey := string(githubPrivateKeyBytes) - // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID") - // githubAppID := secret.MustGetField("GITHUB_APP_ID") - - type init struct { - name string - verify bool - connection *sourcespb.GitHub +func initTestSource(src *sourcespb.GitHub) *Source { + s, conn := createTestSource(src) + if err := s.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1); err != nil { + panic(err) } - tests := []struct { - name string - init init - wantChunk *sources.Chunk - wantErr bool - minRepo int - minOrg int - }{ - { - name: "token authenticated, single repo", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"}, - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: &sources.Chunk{ - SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://github.com/dustin-decker/secretsandstuff.git", - }, - }, - }, - Verify: false, - }, - wantErr: false, - }, - { - name: "token authenticated, single repo, no .git", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"}, - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: &sources.Chunk{ - SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://github.com/dustin-decker/secretsandstuff.git", - }, - }, - }, - Verify: false, - }, - wantErr: false, - }, - { - name: "token authenticated, single org", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Organizations: []string{"trufflesecurity"}, - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 0, - }, - { - name: "token authenticated, username in org", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Organizations: []string{"dustin-decker"}, - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 0, - }, - { - name: "token authenticated, username in repo", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Repositories: []string{"dustin-decker"}, - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 0, - }, - { - name: "token authenticated, org in repo", - // I do not think that this is a supported case, but adding the test to specify there is no requirement. - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Repositories: []string{"trufflesecurity"}, - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 0, - minOrg: 0, + return s +} + +func TestInit(t *testing.T) { + source, conn := createTestSource(&sourcespb.GitHub{ + Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"}, + Credential: &sourcespb.GitHub_Token{ + Token: "super secret token", }, - /* - { - name: "token authenticated, no org or user (enum)", - // This configuration currently will only find gists from the user. No repos or orgs will be scanned. - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 0, - minOrg: 0, - }, - { - name: "app authenticated (old), no repo or org (enum)", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - ScanUsers: false, - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKey, - InstallationId: githubInstallationID, - AppId: githubAppID, - }, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 0, + }) + + err := source.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1) + assert.Nil(t, err) + + // TODO: test error case +} + +func TestAddReposByOrg(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/orgs/super-secret-org/repos"). + Reply(200). + JSON([]map[string]string{{"clone_url": "super-secret-repo"}}) + + s := initTestSource(nil) + // gock works here because github.NewClient is using the default HTTP Transport + err := s.addReposByOrg(context.TODO(), github.NewClient(nil), "super-secret-org") + assert.Nil(t, err) + assert.Equal(t, 1, len(s.repos)) + assert.Equal(t, []string{"super-secret-repo"}, s.repos) + assert.True(t, gock.IsDone()) +} + +func TestAddReposByUser(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/users/super-secret-user/repos"). + Reply(200). + JSON([]map[string]string{{"clone_url": "super-secret-repo"}}) + + s := initTestSource(nil) + err := s.addReposByUser(context.TODO(), github.NewClient(nil), "super-secret-user") + assert.Nil(t, err) + assert.Equal(t, 1, len(s.repos)) + assert.Equal(t, []string{"super-secret-repo"}, s.repos) + assert.True(t, gock.IsDone()) +} + +func TestAddGistsByUser(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/users/super-secret-user/gists"). + Reply(200). + JSON([]map[string]string{{"git_pull_url": "super-secret-gist"}}) + + s := initTestSource(nil) + err := s.addGistsByUser(context.TODO(), github.NewClient(nil), "super-secret-user") + assert.Nil(t, err) + assert.Equal(t, 1, len(s.repos)) + assert.Equal(t, []string{"super-secret-gist"}, s.repos) + assert.True(t, gock.IsDone()) +} + +func TestAddMembersByApp(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/app/installations"). + Reply(200). + JSON([]map[string]interface{}{ + {"account": map[string]string{"login": "super-secret-org"}}, + }) + gock.New("https://api.github.com"). + Get("/orgs/super-secret-org/members"). + Reply(200). + JSON([]map[string]interface{}{ + {"login": "ssm1"}, + {"login": "ssm2"}, + {"login": "ssm3"}, + }) + + s := initTestSource(nil) + err := s.addMembersByApp(context.TODO(), github.NewClient(nil), github.NewClient(nil)) + assert.Nil(t, err) + assert.Equal(t, 3, len(s.members)) + assert.Equal(t, []string{"ssm1", "ssm2", "ssm3"}, s.members) + assert.True(t, gock.IsDone()) +} + +func TestAddReposByApp(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/installation/repositories"). + Reply(200). + JSON(map[string]interface{}{ + "repositories": []map[string]string{ + {"clone_url": "ssr1"}, + {"clone_url": "ssr2"}, }, - */ + }) + + s := initTestSource(nil) + err := s.addReposByApp(context.TODO(), github.NewClient(nil)) + assert.Nil(t, err) + assert.Equal(t, 2, len(s.repos)) + assert.Equal(t, []string{"ssr1", "ssr2"}, s.repos) + assert.True(t, gock.IsDone()) +} + +func TestAddOrgsByUser(t *testing.T) { + defer gock.Off() + + // NOTE: addOrgsByUser calls /user/orgs to get the orgs of the + // authenticated user + gock.New("https://api.github.com"). + Get("/user/orgs"). + Reply(200). + JSON([]map[string]interface{}{ + {"name": "sso1"}, + {"login": "sso2"}, + }) + + s := initTestSource(nil) + s.addOrgsByUser(context.TODO(), github.NewClient(nil), "super-secret-user") + assert.Equal(t, 2, len(s.orgs)) + assert.Equal(t, []string{"sso1", "sso2"}, s.orgs) + assert.True(t, gock.IsDone()) +} + +func TestNormalizeRepos(t *testing.T) { + defer gock.Off() + + tests := []struct { + name string + setup func() + repos []string + expected []string + }{ { - name: "unauthenticated, single org", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Organizations: []string{"trufflesecurity"}, - Credential: &sourcespb.GitHub_Unauthenticated{}, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 1, + name: "repo url", + setup: func() {}, + repos: []string{"https://github.com/super-secret-user/super-secret-repo"}, + expected: []string{"https://github.com/super-secret-user/super-secret-repo.git"}, }, { - name: "unauthenticated, single repo", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, - Credential: &sourcespb.GitHub_Unauthenticated{}, - }, + name: "username with gists", + setup: func() { + gock.New("https://api.github.com"). + Get("/users/super-secret-user/gists"). + Reply(200). + JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-user/super-secret-gist.git"}}) + gock.New("https://api.github.com"). + Get("/users/super-secret-user/repos"). + Reply(200). + JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git"}}) }, - wantChunk: &sources.Chunk{ - SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://github.com/trufflesecurity/driftwood.git", - }, - }, - }, - Verify: false, + repos: []string{"super-secret-user"}, + expected: []string{ + "https://github.com/super-secret-user/super-secret-repo.git", + "https://github.com/super-secret-user/super-secret-gist.git", }, - wantErr: false, }, - /* - { - name: "app authenticated, no repo or org", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - ScanUsers: true, - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 0, - }, - */ { - name: "app authenticated, single repo", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: &sources.Chunk{ - SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://github.com/trufflesecurity/driftwood.git", - }, - }, - }, - Verify: false, + name: "not found", + setup: func() { + gock.New("https://api.github.com"). + Get("/users/not-found/gists"). + Reply(404) + gock.New("https://api.github.com"). + Get("/users/not-found/repos"). + Reply(404) }, - wantErr: false, - minRepo: 1, - minOrg: 0, + repos: []string{"not-found"}, + expected: []string{}, }, { - name: "app authenticated, single org", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Organizations: []string{"trufflesecurity"}, - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 1, + name: "unexpected format", + setup: func() {}, + repos: []string{"/foo/"}, + expected: []string{}, }, } - for i, tt := range tests { + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - log.Debugf("Beginning test %d: %s", i, tt.name) - s := Source{} - - log.SetLevel(log.DebugLevel) - //uncomment for windows Testing - log.SetFormatter(&log.TextFormatter{ForceColors: true}) - log.SetOutput(colorable.NewColorableStdout()) - - conn, err := anypb.New(tt.init.connection) - if err != nil { - t.Fatal(err) - } - - err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4) - if (err != nil) != tt.wantErr { - t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr) - return - } - chunksCh := make(chan *sources.Chunk, 5) - go func() { - err = s.Chunks(ctx, chunksCh) - if (err != nil) != tt.wantErr { - if ctx.Err() != nil { - return - } - t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr) - return - } - }() - if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil { - t.Error(err) - } + defer gock.Off() + tt.setup() + s := initTestSource(nil) + s.repos = tt.repos + + s.normalizeRepos(context.TODO(), github.NewClient(nil)) + assert.Equal(t, len(tt.expected), len(s.repos)) + // sort and compare + sort.Slice(tt.expected, func(i, j int) bool { return tt.expected[i] < tt.expected[j] }) + sort.Slice(s.repos, func(i, j int) bool { return s.repos[i] < s.repos[j] }) + assert.Equal(t, tt.expected, s.repos) + + assert.True(t, gock.IsDone()) }) } } -func TestSource_paginateGists(t *testing.T) { +func TestHandleRateLimit(t *testing.T) { + assert.False(t, handleRateLimit(nil, nil)) - os.Setenv("DO_NOT_RANDOMIZE", "true") + err := &github.RateLimitError{} + res := &github.Response{Response: &http.Response{Header: make(http.Header)}} + res.Header.Set("x-ratelimit-remaining", "0") + res.Header.Set("x-ratelimit-reset", strconv.FormatInt(time.Now().Unix()+1, 10)) + assert.True(t, handleRateLimit(err, res)) +} - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() +func TestEnumerateUnauthenticated(t *testing.T) { + defer gock.Off() - secret, err := common.GetTestSecret(ctx) - if err != nil { - t.Fatal(fmt.Errorf("failed to access secret: %v", err)) - } - //For the NEW github app test (+Member enum) - githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW") - githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New) - if err != nil { - t.Fatal(err) - } - githubPrivateKeyNew := string(githubPrivateKeyBytesNew) - githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") - githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") - type init struct { - name string - verify bool - connection *sourcespb.GitHub - } - tests := []struct { - name string - init init - wantChunk *sources.Chunk - wantErr bool - user string - minRepos int - }{ - { - name: "get gist secret", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: &sources.Chunk{ - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git", - }, - }, - }, - Verify: false, - }, - wantErr: false, - user: "dustin-decker", - minRepos: 1, - }, - { - name: "get multiple pages of gists", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: nil, - wantErr: false, - user: "andrew", - minRepos: 101, - }, - /* { - name: "get multiple pages of gists", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: &sources.Chunk{ - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git", - }, - }, - }, - Verify: false, - }, - wantErr: false, - user: "andrew", - }, - */ - } + gock.New("https://api.github.com"). + Get("/orgs/super-secret-org/repos"). + Reply(200). + JSON([]map[string]string{{"clone_url": "super-secret-repo"}}) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := Source{} - - log.SetLevel(log.DebugLevel) - //uncomment for windows Testing - log.SetFormatter(&log.TextFormatter{ForceColors: true}) - log.SetOutput(colorable.NewColorableStdout()) - - conn, err := anypb.New(tt.init.connection) - if err != nil { - t.Fatal(err) - } - - err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4) - if (err != nil) != tt.wantErr { - t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr) - return - } - chunksCh := make(chan *sources.Chunk, 5) - go func() { - s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user) - chunksCh <- &sources.Chunk{} - }() - var wantedRepo string - if tt.wantChunk != nil { - wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository - } - if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil { - t.Error(err) - } - }) - } + s := initTestSource(nil) + s.orgs = []string{"super-secret-org"} + _ = s.enumerateUnauthenticated(context.TODO()) + assert.Equal(t, 1, len(s.repos)) + assert.Equal(t, []string{"super-secret-repo"}, s.repos) + assert.True(t, gock.IsDone()) } -func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc { - return func(chunk *sources.Chunk) error { - if minRepos != 0 && minRepos > len(s.repos) { - return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos)) - } - if expected != "" { - for _, repo := range s.repos { - if repo == expected { - return nil - } - } - return fmt.Errorf("expected repo not included: %s", expected) - } - return nil - } +func TestEnumerateWithToken(t *testing.T) { + defer gock.Off() + + gock.New("https://api.github.com"). + Get("/user"). + Reply(200). + JSON(map[string]string{"login": "super-secret-user"}) + + gock.New("https://api.github.com"). + Get("/users/super-secret-user/repos"). + Reply(200). + JSON([]map[string]string{{"clone_url": "super-secret-repo"}}) + + s := initTestSource(nil) + _, err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token") + assert.Nil(t, err) + assert.Equal(t, 1, len(s.repos)) + assert.Equal(t, []string{"super-secret-repo"}, s.repos) + assert.True(t, gock.IsDone()) } -func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc { - return func(chunk *sources.Chunk) error { - if minOrg != 0 && minOrg > len(s.orgs) { - return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs)) - } - if minRepo != 0 && minRepo > len(s.repos) { - return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos)) +func TestEnumerateWithApp(t *testing.T) { + defer gock.Off() + + // generate a private key (it just needs to be in the right format) + privateKey := func() string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(err) } - if wantChunk != nil { - if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" { - return nil - } - return common.MatchError + + data := x509.MarshalPKCS1PrivateKey(key) + var pemKey bytes.Buffer + if err := pem.Encode(&pemKey, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: data, + }); err != nil { + panic(err) } - return nil - } -} + return pemKey.String() + }() + + gock.New("https://api.github.com"). + Post("/app/installations/1337/access_tokens"). + Reply(200). + JSON(map[string]string{"token": "dontlook"}) + + gock.New("https://api.github.com"). + Get("/installation/repositories"). + Reply(200). + JSON(map[string]string{}) -// func TestSource_paginateRepos(t *testing.T) { -// type args struct { -// ctx context.Context -// apiClient *github.Client -// } -// tests := []struct { -// name string -// org string -// args args -// }{ -// { -// org: "fakeNetflix", -// args: args{ -// ctx: context.Background(), -// apiClient: github.NewClient(common.SaneHttpClient()), -// }, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// s := &Source{httpClient: common.SaneHttpClient()} -// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org) -// if len(s.repos) < 101 { -// t.Errorf("expected > 100 repos, got %d", len(s.repos)) -// } -// }) -// } -// } + s := initTestSource(nil) + _, _, err := s.enumerateWithApp( + context.TODO(), + "https://api.github.com", + &credentialspb.GitHubApp{ + InstallationId: "1337", + AppId: "4141", + PrivateKey: privateKey, + }, + ) + fmt.Println(err) + assert.Nil(t, err) + assert.Equal(t, 0, len(s.repos)) + + assert.True(t, gock.IsDone()) +}