Skip to content

Commit

Permalink
Feature/synapse bitbucket (#80)
Browse files Browse the repository at this point in the history 
* added refresh token api

* added unit tests

* added unit test for secret parser expired function

* disable golangci for false positive case

* added comments

* added token type

* added gitprovider condition in refresh oauth

* added basic auth supprt in git init

* fixed spelling

* fixed unit tests

* fixed unit test

* Changes get oauth secret test
  • Loading branch information
@utkarsh-lambdatest
utkarsh-lambdatest committed Mar 24, 2022
1 parent f3cfea0 commit cf6756a
Show file tree
Hide file tree
Showing 14 changed files with 120 additions and 71 deletions.
3 changes: 2 additions & 1 deletion .sample.synapse.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
"SecretKey": "SecretKeyForLambdaTest"
},
"Git": {
"Token": ""
"Token": "",
"TokenType": "Bearer"
},
"ContainerRegistry": {
"PullPolicy": "always",
Expand Down
3 changes: 2 additions & 1 deletion config/synapsemodel.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ type LambdatestConfig struct {

// GitConfig contains git token
type GitConfig struct {
Token string
Token string
TokenType string
}

// PullPolicyType defines when to pull docker image
Expand Down
4 changes: 2 additions & 2 deletions pkg/core/interfaces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ type TASConfigManager interface {
// GitManager manages the cloning of git repositories
type GitManager interface {
// Clone repository from TAS config
Clone(ctx context.Context, payload *Payload, cloneToken string) error
Clone(ctx context.Context, payload *Payload, oauth *Oauth) error
}

// DiffManager manages the diff findings for the given payload
type DiffManager interface {
GetChangedFiles(ctx context.Context, payload *Payload, cloneToken string) (map[string]int, error)
GetChangedFiles(ctx context.Context, payload *Payload, oauth *Oauth) (map[string]int, error)
}

// TestDiscoveryService services discovery of tests
Expand Down
13 changes: 7 additions & 6 deletions pkg/core/lifecycle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,15 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
os.Exit(0)
}

oauth, err := pl.getOauthSecret(payload.RepoID)
oauth, err := pl.getOauthSecret(payload.RepoID, payload.GitProvider)
if err != nil {
pl.Logger.Fatalf("failed to get oauth secret %v", err)
}

// set payload on pipeline object
pl.Payload = payload
if pl.Cfg.ParseMode {
err = pl.GitManager.Clone(ctx, payload, oauth.Data.AccessToken)
err = pl.GitManager.Clone(ctx, payload, oauth)
if err != nil {
pl.Logger.Fatalf("failed to clone YML for build ID: %s, error: %v", payload.BuildID, err)
}
Expand Down Expand Up @@ -134,7 +134,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {

if pl.Cfg.DiscoverMode {
pl.Logger.Infof("Cloning repo ...")
err = pl.GitManager.Clone(ctx, pl.Payload, oauth.Data.AccessToken)
err = pl.GitManager.Clone(ctx, pl.Payload, oauth)
if err != nil {
pl.Logger.Errorf("Unable to clone repo '%s': %s", payload.RepoLink, err)
errRemark = fmt.Sprintf("Unable to clone repo: %s", payload.RepoLink)
Expand Down Expand Up @@ -249,7 +249,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {

pl.Logger.Infof("Identifying changed files ...")
diffExists := true
diff, err := pl.DiffManager.GetChangedFiles(ctx, payload, oauth.Data.AccessToken)
diff, err := pl.DiffManager.GetChangedFiles(ctx, payload, oauth)
if err != nil {
if errors.Is(err, errs.ErrGitDiffNotFound) {
diffExists = false
Expand Down Expand Up @@ -363,13 +363,13 @@ func (pl *Pipeline) sendStats(payload ExecutionResults) error {
}

// getOauthSecret returns a valid oauth token
func (pl *Pipeline) getOauthSecret(repoID string) (*Oauth, error) {
func (pl *Pipeline) getOauthSecret(repoID, gitProvider string) (*Oauth, error) {
oauth, err := pl.SecretParser.GetOauthSecret(global.OauthSecretPath)
if err != nil {
return nil, err
}

if !pl.SecretParser.Expired(oauth) {
if gitProvider != Bitbucket || !pl.SecretParser.Expired(oauth) {
return oauth, nil
}

Expand Down Expand Up @@ -397,5 +397,6 @@ func (pl *Pipeline) getOauthSecret(repoID string) (*Oauth, error) {
pl.Logger.Errorf("error while unmarshaling json to oauth for RepoID %s : %s", repoID, err)
}

refreshedOauth.Data.Type = Bearer
return refreshedOauth, nil
}
10 changes: 10 additions & 0 deletions pkg/core/models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,12 +268,22 @@ const (
Bitbucket string = "bitbucket"
)

type TokenType string

const (
// Bearer as token type
Bearer TokenType = "Bearer"
// Basic as token type
Basic TokenType = "Basic"
)

// Oauth represents the sructure of Oauth
type Oauth struct {
Data struct {
AccessToken string `json:"access_token"`
Expiry time.Time `json:"expiry"`
RefreshToken string `json:"refresh_token"`
Type TokenType `json:"token_type,omitempty"`
} `json:"data"`
}

Expand Down
16 changes: 8 additions & 8 deletions pkg/diffmanager/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func (dm *diffManager) updateWithOr(m map[string]int, key string, value int) {
m[key] = m[key] | value
}

func (dm *diffManager) getCommitDiff(gitprovider, repoURL string, cloneToken string, baseCommit, targetCommit string) ([]byte, error) {
func (dm *diffManager) getCommitDiff(gitprovider, repoURL string, oauth *core.Oauth, baseCommit, targetCommit string) ([]byte, error) {
if baseCommit == "" {
dm.logger.Debugf("basecommit is empty for gitprovider %v error %v", gitprovider, errs.ErrGitDiffNotFound)
return nil, errs.ErrGitDiffNotFound
Expand All @@ -85,8 +85,8 @@ func (dm *diffManager) getCommitDiff(gitprovider, repoURL string, cloneToken str
if err != nil {
return nil, err
}
if cloneToken != "" {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cloneToken))
if oauth.Data.AccessToken != "" {
req.Header.Add("Authorization", fmt.Sprintf("%s %s", oauth.Data.Type, oauth.Data.AccessToken))
}
req.Header.Add("Accept", "application/vnd.github.v3.diff")
resp, err := dm.client.Do(req)
Expand All @@ -102,7 +102,7 @@ func (dm *diffManager) getCommitDiff(gitprovider, repoURL string, cloneToken str
return ioutil.ReadAll(resp.Body)
}

func (dm *diffManager) getPRDiff(gitprovider, repoURL string, prNumber int, cloneToken string) ([]byte, error) {
func (dm *diffManager) getPRDiff(gitprovider, repoURL string, prNumber int, oauth *core.Oauth) ([]byte, error) {
parsedUrl, err := url.Parse(repoURL)
if err != nil {
return nil, err
Expand All @@ -123,7 +123,7 @@ func (dm *diffManager) getPRDiff(gitprovider, repoURL string, prNumber int, clon
dm.logger.Errorf("failed to create http request for changelist url error: %v", err)
return nil, err
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cloneToken))
req.Header.Add("Authorization", fmt.Sprintf("%s %s", oauth.Data.Type, oauth.Data.AccessToken))
req.Header.Set("Accept", "application/vnd.github.v3.diff")

resp, err := dm.client.Do(req)
Expand Down Expand Up @@ -197,20 +197,20 @@ func (dm *diffManager) parseGitDiff(gitprovider string, eventType core.EventType
}

// GetChangedFiles Figure out changed files
func (dm *diffManager) GetChangedFiles(ctx context.Context, payload *core.Payload, cloneToken string) (map[string]int, error) {
func (dm *diffManager) GetChangedFiles(ctx context.Context, payload *core.Payload, oauth *core.Oauth) (map[string]int, error) {
// map to store file and type of change (added, removed, modified)
var m map[string]int

var diff []byte
var err error
if payload.EventType == core.EventPullRequest {
diff, err = dm.getPRDiff(payload.GitProvider, payload.RepoLink, payload.PullRequestNumber, cloneToken)
diff, err = dm.getPRDiff(payload.GitProvider, payload.RepoLink, payload.PullRequestNumber, oauth)
if err != nil {
dm.logger.Errorf("failed to parse pr diff for gitprovider: %s error: %v", payload.GitProvider, err)
return nil, err
}
} else {
diff, err = dm.getCommitDiff(payload.GitProvider, payload.RepoLink, cloneToken, payload.BuildBaseCommit, payload.BuildTargetCommit)
diff, err = dm.getCommitDiff(payload.GitProvider, payload.RepoLink, oauth, payload.BuildBaseCommit, payload.BuildTargetCommit)
if err != nil {
dm.logger.Errorf("failed to get commit diff for gitprovider: %s error: %v", payload.GitProvider, err)
return nil, err
Expand Down
51 changes: 30 additions & 21 deletions pkg/diffmanager/setup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,21 @@ import (
"net/http/httptest"
"reflect"
"testing"
"time"

"github.com/LambdaTest/synapse/pkg/core"
"github.com/LambdaTest/synapse/pkg/global"
"github.com/LambdaTest/synapse/testutils"
)

//nolint unused
type oauthData struct {
AccessToken string `json:"access_token"`
Expiry time.Time `json:"expiry"`
RefreshToken string `json:"refresh_token"`
Type core.TokenType `json:"token_type,omitempty"`
}

func Test_updateWithOr(t *testing.T) {
check := func(t *testing.T) {
dm := &diffManager{}
Expand Down Expand Up @@ -52,9 +61,9 @@ func Test_diffManager_GetChangedFiles_PRDiff(t *testing.T) {

dm := NewDiffManager(config, logger)
type args struct {
ctx context.Context
payload *core.Payload
cloneToken string
ctx context.Context
payload *core.Payload
oauth *core.Oauth
}
tests := []struct {
name string
Expand All @@ -63,19 +72,19 @@ func Test_diffManager_GetChangedFiles_PRDiff(t *testing.T) {
wantErr bool
}{
// expects to hit Server.URL/testdata/pulls/2
{"Test GetChangedFile for PRdiff for github gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", GitProvider: "github", PrivateRepo: false, EventType: "pull-request", Diff: "xyz", PullRequestNumber: 2}, cloneToken: ""}, map[string]int{}, false},
{"Test GetChangedFile for PRdiff for github gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", GitProvider: "github", PrivateRepo: false, EventType: "pull-request", Diff: "xyz", PullRequestNumber: 2}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, false},

// expects to hit Server.URL/testdata/merge_requests/2/changes
{"Test GetChangedFile for PRdiff for gitlab gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", GitProvider: "gitlab", PrivateRepo: false, EventType: "pull-request", Diff: "xyz", PullRequestNumber: 2}, cloneToken: ""}, map[string]int{}, false},
{"Test GetChangedFile for PRdiff for gitlab gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", GitProvider: "gitlab", PrivateRepo: false, EventType: "pull-request", Diff: "xyz", PullRequestNumber: 2}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, false},

{"Test GetChangedFile for Commitdiff for unsupported gitprovider", args{ctx: context.TODO(), payload: &core.Payload{GitProvider: "unsupported"}}, map[string]int{}, true},
{"Test GetChangedFile for Commitdiff for unsupported gitprovider", args{ctx: context.TODO(), payload: &core.Payload{GitProvider: "unsupported"}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, true},

{"Test GetChangedFile for PRdiff for unsupported gitprovider", args{ctx: context.TODO(), payload: &core.Payload{GitProvider: "unsupported", EventType: "pull-request"}}, map[string]int{}, true},
{"Test GetChangedFile for PRdiff for unsupported gitprovider", args{ctx: context.TODO(), payload: &core.Payload{GitProvider: "unsupported", EventType: "pull-request"}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
global.APIHostURLMap[tt.args.payload.GitProvider] = server.URL
resp, err := dm.GetChangedFiles(tt.args.ctx, tt.args.payload, tt.args.cloneToken)
resp, err := dm.GetChangedFiles(tt.args.ctx, tt.args.payload, tt.args.oauth)

if tt.wantErr {
if err == nil {
Expand Down Expand Up @@ -113,9 +122,9 @@ func Test_diffManager_GetChangedFiles_CommitDiff_Github(t *testing.T) {

dm := NewDiffManager(config, logger)
type args struct {
ctx context.Context
payload *core.Payload
cloneToken string
ctx context.Context
payload *core.Payload
oauth *core.Oauth
}
tests := []struct {
name string
Expand All @@ -124,18 +133,18 @@ func Test_diffManager_GetChangedFiles_CommitDiff_Github(t *testing.T) {
wantErr bool
}{
// expects to hit serverURL/testdata/compare/abc...xyz
{"Test GetChangedFile for CommitDiff for github gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", BuildTargetCommit: "xyz", BuildBaseCommit: "abc", GitProvider: "github", EventType: "push", Diff: "xyz", PullRequestNumber: 2}}, map[string]int{}, false},
{"Test GetChangedFile for CommitDiff for github gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", BuildTargetCommit: "xyz", BuildBaseCommit: "abc", GitProvider: "github", EventType: "push", Diff: "xyz", PullRequestNumber: 2}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, false},

{"Test GetChangedFile for CommitDiff for github provider and empty base commit", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", BuildBaseCommit: "", GitProvider: "gitlab", EventType: "push"}}, map[string]int{}, true},
{"Test GetChangedFile for CommitDiff for github provider and empty base commit", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", BuildBaseCommit: "", GitProvider: "gitlab", EventType: "push"}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, true},

{"Test GetChangedFile for CommitDiff for github provider for non 200 response", args{ctx: context.TODO(), payload: &core.Payload{RepoLink: server.URL + "/notfound/", BuildTargetCommit: "xyz", BuildBaseCommit: "abc", GitProvider: "gitlab", EventType: "push"}}, map[string]int{}, true},
{"Test GetChangedFile for CommitDiff for github provider for non 200 response", args{ctx: context.TODO(), payload: &core.Payload{RepoLink: server.URL + "/notfound/", BuildTargetCommit: "xyz", BuildBaseCommit: "abc", GitProvider: "gitlab", EventType: "push"}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, true},

{"Test GetChangedFile for CommitDiff for non supported git provider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/notfound/", RepoLink: server.URL + "/notfound/", BuildTargetCommit: "xyz", BuildBaseCommit: "abc", GitProvider: "gittest", EventType: "push"}}, map[string]int{}, false},
{"Test GetChangedFile for CommitDiff for non supported git provider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/notfound/", RepoLink: server.URL + "/notfound/", BuildTargetCommit: "xyz", BuildBaseCommit: "abc", GitProvider: "gittest", EventType: "push"}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
global.APIHostURLMap[tt.args.payload.GitProvider] = server.URL
resp, err := dm.GetChangedFiles(tt.args.ctx, tt.args.payload, tt.args.cloneToken)
resp, err := dm.GetChangedFiles(tt.args.ctx, tt.args.payload, tt.args.oauth)
// t.Errorf("")
if tt.args.payload.GitProvider == "gittest" {
if resp != nil || err == nil {
Expand Down Expand Up @@ -188,22 +197,22 @@ func Test_diffManager_GetChangedFiles_CommitDiff_Gitlab(t *testing.T) {

dm := NewDiffManager(config, logger)
type args struct {
ctx context.Context
payload *core.Payload
cloneToken string
ctx context.Context
payload *core.Payload
oauth *core.Oauth
}
tests := []struct {
name string
args args
want map[string]int
}{
// expects to hit serverURL/testdata/repository/compare?from=abc&to=abcd
{"Test GetChangedFile for CommitDiff for gitlab gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", BuildTargetCommit: "abcd", BuildBaseCommit: "abc", TaskID: "taskid", BranchName: "branchname", BuildID: "buildid", RepoID: "repoid", OrgID: "orgid", GitProvider: "gitlab", PrivateRepo: false, EventType: "push", Diff: "xyz", PullRequestNumber: 2}, cloneToken: ""}, map[string]int{}},
{"Test GetChangedFile for CommitDiff for gitlab gitprovider", args{ctx: context.TODO(), payload: &core.Payload{RepoSlug: "/testdata", RepoLink: server.URL + "/testdata", BuildTargetCommit: "abcd", BuildBaseCommit: "abc", TaskID: "taskid", BranchName: "branchname", BuildID: "buildid", RepoID: "repoid", OrgID: "orgid", GitProvider: "gitlab", PrivateRepo: false, EventType: "push", Diff: "xyz", PullRequestNumber: 2}, oauth: &core.Oauth{Data: oauthData{}}}, map[string]int{}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
global.APIHostURLMap[tt.args.payload.GitProvider] = server.URL
resp, err := dm.GetChangedFiles(tt.args.ctx, tt.args.payload, tt.args.cloneToken)
resp, err := dm.GetChangedFiles(tt.args.ctx, tt.args.payload, tt.args.oauth)

if err != nil {
t.Errorf("error in getting changed files, error %v", err.Error())
Expand Down
30 changes: 22 additions & 8 deletions pkg/gitmanager/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package gitmanager

import (
"context"
"encoding/base64"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -36,7 +37,7 @@ func NewGitManager(logger lumber.Logger, execManager core.ExecutionManager) core
}
}

func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, cloneToken string) error {
func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, oauth *core.Oauth) error {
repoLink := payload.RepoLink
repoItems := strings.Split(repoLink, "/")
repoName := repoItems[len(repoItems)-1]
Expand All @@ -48,7 +49,7 @@ func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, cloneTok
return err
}
gm.logger.Debugf("cloning from %s", archiveURL)
err = gm.downloadFile(ctx, archiveURL, commitID+".zip", cloneToken)
err = gm.downloadFile(ctx, archiveURL, commitID+".zip", oauth)
if err != nil {
gm.logger.Errorf("failed to download file %v", err)
return err
Expand All @@ -65,7 +66,7 @@ func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, cloneTok
return err
}

if err = gm.initGit(ctx, payload, cloneToken); err != nil {
if err = gm.initGit(ctx, payload, oauth); err != nil {
gm.logger.Errorf("failed to initialize git, error %v", err)
return err
}
Expand All @@ -74,13 +75,13 @@ func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, cloneTok
}

// downloadFile clones the archive from github and extracts the file if it is a zip file.
func (gm *gitManager) downloadFile(ctx context.Context, archiveURL, fileName, cloneToken string) error {
func (gm *gitManager) downloadFile(ctx context.Context, archiveURL, fileName string, oauth *core.Oauth) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, archiveURL, nil)
if err != nil {
return err
}
if cloneToken != "" {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cloneToken))
if oauth.Data.AccessToken != "" {
req.Header.Add("Authorization", fmt.Sprintf("%s %s", oauth.Data.Type, oauth.Data.AccessToken))
}
resp, err := gm.httpClient.Do(req)
if err != nil {
Expand Down Expand Up @@ -130,13 +131,26 @@ func (gm *gitManager) copyAndExtractFile(resp *http.Response, path string) error
return err
}

func (gm *gitManager) initGit(ctx context.Context, payload *core.Payload, cloneToken string) error {
func (gm *gitManager) initGit(ctx context.Context, payload *core.Payload, oauth *core.Oauth) error {
branch := payload.BranchName
repoURL, perr := url.Parse(payload.RepoLink)
if perr != nil {
return perr
}
repoURL.User = url.UserPassword("x-token-auth", cloneToken)

if oauth.Data.Type == core.Basic {
decodedToken, err := base64.StdEncoding.DecodeString(oauth.Data.AccessToken)
if err != nil {
gm.logger.Errorf("Failed to decode basic oauth token for RepoID %s: %s", payload.RepoID, err)
return err
}

creds := strings.Split(string(decodedToken), ":")
repoURL.User = url.UserPassword(creds[0], creds[1])
} else {
repoURL.User = url.UserPassword("x-token-auth", oauth.Data.AccessToken)
}

urlWithToken := repoURL.String()
commands := []string{
"git init",
Expand Down
Loading

0 comments on commit cf6756a

Please sign in to comment.