From 93101a89aae2ecb3241ae9c9656f484b02cdce75 Mon Sep 17 00:00:00 2001 From: Dan Molik Date: Mon, 4 Apr 2022 12:02:43 -0400 Subject: [PATCH 1/4] feat: New SCM and pull request ApplicationSet generators for Gitea An initial implementation of Pull Request and SCM generators for Gitea. API paging, and repo labels have not been implemented. Signed-off-by: Dan Molik --- applicationset/generators/pull_request.go | 8 + applicationset/generators/scm_provider.go | 9 ++ applicationset/services/pull_request/gitea.go | 67 ++++++++ applicationset/services/scm_provider/gitea.go | 133 ++++++++++++++++ .../applicationset/Generators-Pull-Request.md | 37 +++++ .../applicationset/Generators-SCM-Provider.md | 39 ++++- go.mod | 2 + go.sum | 6 + manifests/core-install.yaml | 147 ++++++++++++++++++ manifests/crds/applicationset-crd.yaml | 147 ++++++++++++++++++ manifests/ha/install.yaml | 147 ++++++++++++++++++ manifests/install.yaml | 147 ++++++++++++++++++ .../v1alpha1/applicationset_types.go | 30 ++++ .../v1alpha1/zz_generated.deepcopy.go | 50 ++++++ 14 files changed, 968 insertions(+), 1 deletion(-) create mode 100644 applicationset/services/pull_request/gitea.go create mode 100644 applicationset/services/scm_provider/gitea.go diff --git a/applicationset/generators/pull_request.go b/applicationset/generators/pull_request.go index 7aa25ea1c5716..8303134212a76 100644 --- a/applicationset/generators/pull_request.go +++ b/applicationset/generators/pull_request.go @@ -86,6 +86,14 @@ func (g *PullRequestGenerator) selectServiceProvider(ctx context.Context, genera } return pullrequest.NewGithubService(ctx, token, providerConfig.API, providerConfig.Owner, providerConfig.Repo, providerConfig.Labels) } + if generatorConfig.Gitea != nil { + providerConfig := generatorConfig.Gitea + token, err := g.getSecretRef(ctx, providerConfig.TokenRef, applicationSetInfo.Namespace) + if err != nil { + return nil, fmt.Errorf("error fetching Secret token: %v", err) + } + return pullrequest.NewGiteaService(ctx, token, providerConfig.API, providerConfig.Owner, providerConfig.Repo, providerConfig.Insecure) + } return nil, fmt.Errorf("no Pull Request provider implementation configured") } diff --git a/applicationset/generators/scm_provider.go b/applicationset/generators/scm_provider.go index b04a425245062..7d0d2283512a3 100644 --- a/applicationset/generators/scm_provider.go +++ b/applicationset/generators/scm_provider.go @@ -77,6 +77,15 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha if err != nil { return nil, fmt.Errorf("error initializing Gitlab service: %v", err) } + } else if providerConfig.Gitea != nil { + token, err := g.getSecretRef(ctx, providerConfig.Gitea.TokenRef, applicationSetInfo.Namespace) + if err != nil { + return nil, fmt.Errorf("error fetching Gitea token: %v", err) + } + provider, err = scm_provider.NewGiteaProvider(ctx, providerConfig.Gitea.Owner, token, providerConfig.Gitea.API, providerConfig.Gitea.AllBranches, providerConfig.Gitea.Insecure) + if err != nil { + return nil, fmt.Errorf("error initializing Gitea service: %v", err) + } } else { return nil, fmt.Errorf("no SCM provider implementation configured") } diff --git a/applicationset/services/pull_request/gitea.go b/applicationset/services/pull_request/gitea.go new file mode 100644 index 0000000000000..3520e60261c47 --- /dev/null +++ b/applicationset/services/pull_request/gitea.go @@ -0,0 +1,67 @@ +package pull_request + +import ( + "context" + "crypto/tls" + "net/http" + "net/http/cookiejar" + "os" + + "code.gitea.io/sdk/gitea" +) + +type GiteaService struct { + client *gitea.Client + owner string + repo string +} + +var _ PullRequestService = (*GiteaService)(nil) + +func NewGiteaService(ctx context.Context, token, url, owner, repo string, insecure bool) (PullRequestService, error) { + if token == "" { + token = os.Getenv("GITEA_TOKEN") + } + httpClient := &http.Client{} + if insecure { + cookieJar, _ := cookiejar.New(nil) + + httpClient = &http.Client{ + Jar: cookieJar, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + } + client, err := gitea.NewClient(url, gitea.SetToken(token), gitea.SetHTTPClient(httpClient)) + if err != nil { + return nil, err + } + return &GiteaService{ + client: client, + owner: owner, + repo: repo, + }, nil +} + +func (g *GiteaService) List(ctx context.Context) ([]*PullRequest, error) { + opts := gitea.ListPullRequestsOptions{ + State: gitea.StateOpen, + /* ListOptions: gitea.ListOptions{ + Page: 1, + PageSize: 100, + },*/ + } + prs, _, err := g.client.ListRepoPullRequests(g.owner, g.repo, opts) + if err != nil { + return nil, err + } + list := []*PullRequest{} + for _, pr := range prs { + list = append(list, &PullRequest{ + Number: int(pr.ID), + Branch: pr.Base.Ref, + HeadSHA: pr.Head.Sha, + }) + } + return list, nil +} diff --git a/applicationset/services/scm_provider/gitea.go b/applicationset/services/scm_provider/gitea.go new file mode 100644 index 0000000000000..4641ea6151cf0 --- /dev/null +++ b/applicationset/services/scm_provider/gitea.go @@ -0,0 +1,133 @@ +package scm_provider + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "net/http/cookiejar" + "os" + + "code.gitea.io/sdk/gitea" +) + +type GiteaProvider struct { + client *gitea.Client + owner string + allBranches bool +} + +var _ SCMProviderService = &GiteaProvider{} + +func NewGiteaProvider(ctx context.Context, owner, token, url string, allBranches, insecure bool) (*GiteaProvider, error) { + if token == "" { + token = os.Getenv("GITEA_TOKEN") + } + httpClient := &http.Client{} + if insecure { + cookieJar, _ := cookiejar.New(nil) + + httpClient = &http.Client{ + Jar: cookieJar, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + } + client, err := gitea.NewClient(url, gitea.SetToken(token), gitea.SetHTTPClient(httpClient)) + if err != nil { + return nil, err + } + return &GiteaProvider{ + client: client, + owner: owner, + allBranches: allBranches, + }, nil +} + +func (g *GiteaProvider) GetBranches(ctx context.Context, repo *Repository) ([]*Repository, error) { + if !g.allBranches { + branch, _, err := g.client.GetRepoBranch(g.owner, repo.Repository, repo.Branch) + if err != nil { + return nil, err + } + return []*Repository{ + { + Organization: repo.Organization, + Repository: repo.Repository, + Branch: repo.Branch, + URL: repo.URL, + SHA: branch.Commit.ID, + Labels: repo.Labels, + RepositoryId: repo.RepositoryId, + }, + }, nil + } + repos := []*Repository{} + opts := gitea.ListRepoBranchesOptions{} + branches, _, err := g.client.ListRepoBranches(g.owner, repo.Repository, opts) + if err != nil { + return nil, err + } + for _, branch := range branches { + repos = append(repos, &Repository{ + Organization: repo.Organization, + Repository: repo.Repository, + Branch: branch.Name, + URL: repo.URL, + SHA: branch.Commit.ID, + Labels: repo.Labels, + RepositoryId: repo.RepositoryId, + }) + } + return repos, nil +} + +func (g *GiteaProvider) ListRepos(ctx context.Context, cloneProtocol string) ([]*Repository, error) { + repos := []*Repository{} + repoOpts := gitea.ListOrgReposOptions{} + giteaRepos, _, err := g.client.ListOrgRepos(g.owner, repoOpts) + if err != nil { + return nil, err + } + for _, repo := range giteaRepos { + var url string + switch cloneProtocol { + // Default to SSH if unspecified (i.e. if ""). + case "", "ssh": + url = repo.SSHURL + case "https": + url = repo.HTMLURL + default: + return nil, fmt.Errorf("unknown clone protocol for GitHub %v", cloneProtocol) + } + labelOpts := gitea.ListLabelsOptions{} + giteaLabels, _, err := g.client.ListRepoLabels(g.owner, repo.Name, labelOpts) + if err != nil { + return nil, err + } + labels := []string{} + for _, label := range giteaLabels { + labels = append(labels, label.Name) + } + repos = append(repos, &Repository{ + Organization: g.owner, + Repository: repo.Name, + Branch: repo.DefaultBranch, + URL: url, + Labels: labels, + RepositoryId: int(repo.ID), + }) + } + return repos, nil +} + +func (g *GiteaProvider) RepoHasPath(ctx context.Context, repo *Repository, path string) (bool, error) { + _, resp, err := g.client.GetContents(repo.Organization, repo.Repository, repo.Branch, path) + if resp != nil && resp.StatusCode == 404 { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} diff --git a/docs/operator-manual/applicationset/Generators-Pull-Request.md b/docs/operator-manual/applicationset/Generators-Pull-Request.md index 5336f3d2b20c2..480179012c3cc 100644 --- a/docs/operator-manual/applicationset/Generators-Pull-Request.md +++ b/docs/operator-manual/applicationset/Generators-Pull-Request.md @@ -53,6 +53,43 @@ spec: * `tokenRef`: A `Secret` name and key containing the GitHub access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. (Optional) * `labels`: Labels is used to filter the PRs that you want to target. (Optional) +## Gitea + +Specify the repository from which to fetch the Gitea Pull requests. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - pullRequest: + gitea: + # The Gitea organization or user. + owner: myorg + # The Gitea repository + repo: myrepository + # The Gitea url to use + api: https://gitea.mydomain.com/ + # Reference to a Secret containing an access token. (optional) + tokenRef: + secretName: gitea-token + key: token + # many gitea deployments use TLS, but many are self-hosted and self-signed certificates + insecure: true + requeueAfterSeconds: 1800 + template: + # ... +``` + +* `owner`: Required name of the Gitea organization or user. +* `repo`: Required name of the Gitea repositry. +* `api`: The url of the Gitea instance. +* `tokenRef`: A `Secret` name and key containing the Gitea access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. (Optional) +* `insecure`: `Allow for self-signed certificates, primarily for testing.` + + ## Template As with all generators, several keys are available for replacement in the generated application. diff --git a/docs/operator-manual/applicationset/Generators-SCM-Provider.md b/docs/operator-manual/applicationset/Generators-SCM-Provider.md index 4107e4ac6563a..684d17b536daf 100644 --- a/docs/operator-manual/applicationset/Generators-SCM-Provider.md +++ b/docs/operator-manual/applicationset/Generators-SCM-Provider.md @@ -94,6 +94,43 @@ For label filtering, the repository tags are used. Available clone protocols are `ssh` and `https`. +## Gitea + +The Gitea mode uses the Gitea API to scan organizations in your instance + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: myapps +spec: + generators: + - scmProvider: + gitea: + # The Gitea owner to scan. + owner: myorg + # The Gitea instance url + api: https://gitea.mydomain.com/ + # If true, scan every branch of every repository. If false, scan only the default branch. Defaults to false. + allBranches: true + # Reference to a Secret containing an access token. (optional) + tokenRef: + secretName: gitea-token + key: token + template: + # ... +``` + +* `owner`: Required name of the Gitea organization to scan. If you have multiple orgs, use multiple generators. +* `api`: The URL of the Gitea instance you are using. +* `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter. +* `tokenRef`: A `Secret` name and key containing the Gitea access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. +* `insecure`: Allow for self-signed TLS certificates. + +This SCM provider does not yet support label filtering + +Available clone protocols are `ssh` and `https`. + ## Filters Filters allow selecting which repositories to generate for. Each filter can declare one or more conditions, all of which must pass. If multiple filters are present, any can match for a repository to be included. If no filters are specified, all repositories will be processed. @@ -155,4 +192,4 @@ spec: * `url`: The clone URL for the repository. * `branch`: The default branch of the repository. * `sha`: The Git commit SHA for the branch -* `labels`: A comma-separated list of repository labels \ No newline at end of file +* `labels`: A comma-separated list of repository labels diff --git a/go.mod b/go.mod index fafcf5aabab89..f1121a85bde4d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/argoproj/argo-cd/v2 go 1.17 require ( + code.gitea.io/sdk/gitea v0.15.1 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/Masterminds/semver/v3 v3.1.1 github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d @@ -154,6 +155,7 @@ require ( github.com/gorilla/websocket v1.4.2 // indirect github.com/gregdel/pushover v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-version v1.2.1 // indirect github.com/huandu/xstrings v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/itchyny/timefmt-go v0.1.2 // indirect diff --git a/go.sum b/go.sum index ec20421e98233..e61c20c0d4fa6 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,9 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= +code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M= +code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -610,6 +613,8 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -1509,6 +1514,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml index dae4436ee8656..cc127b512b160 100644 --- a/manifests/core-install.yaml +++ b/manifests/core-install.yaml @@ -4456,6 +4456,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -4783,6 +4808,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -6488,6 +6537,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -6815,6 +6889,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -7385,6 +7483,31 @@ spec: type: object pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -7712,6 +7835,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: diff --git a/manifests/crds/applicationset-crd.yaml b/manifests/crds/applicationset-crd.yaml index 065976751dfa5..95fc8de02ed42 100644 --- a/manifests/crds/applicationset-crd.yaml +++ b/manifests/crds/applicationset-crd.yaml @@ -2304,6 +2304,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -2631,6 +2656,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -4336,6 +4385,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -4663,6 +4737,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -5233,6 +5331,31 @@ spec: type: object pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -5560,6 +5683,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index 183e81075769a..33d6e8d144b89 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -4456,6 +4456,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -4783,6 +4808,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -6488,6 +6537,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -6815,6 +6889,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -7385,6 +7483,31 @@ spec: type: object pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -7712,6 +7835,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: diff --git a/manifests/install.yaml b/manifests/install.yaml index f1271a024758c..40a5cddc05726 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -4456,6 +4456,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -4783,6 +4808,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -6488,6 +6537,31 @@ spec: x-kubernetes-preserve-unknown-fields: true pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -6815,6 +6889,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: @@ -7385,6 +7483,31 @@ spec: type: object pullRequest: properties: + gitea: + properties: + api: + type: string + insecure: + type: boolean + owner: + type: string + repo: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + - repo + type: object github: properties: api: @@ -7712,6 +7835,30 @@ spec: type: string type: object type: array + gitea: + properties: + allBranches: + type: boolean + api: + type: string + insecure: + type: boolean + owner: + type: string + tokenRef: + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - api + - owner + type: object github: properties: allBranches: diff --git a/pkg/apis/applicationset/v1alpha1/applicationset_types.go b/pkg/apis/applicationset/v1alpha1/applicationset_types.go index 81ecde174ba99..54ba70c1228e2 100644 --- a/pkg/apis/applicationset/v1alpha1/applicationset_types.go +++ b/pkg/apis/applicationset/v1alpha1/applicationset_types.go @@ -296,6 +296,7 @@ type SCMProviderGenerator struct { Github *SCMProviderGeneratorGithub `json:"github,omitempty"` Gitlab *SCMProviderGeneratorGitlab `json:"gitlab,omitempty"` Bitbucket *SCMProviderGeneratorBitbucket `json:"bitbucket,omitempty"` + Gitea *SCMProviderGeneratorGitea `json:"gitea,omitempty"` // Filters for which repos should be considered. Filters []SCMProviderGeneratorFilter `json:"filters,omitempty"` // Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers @@ -306,6 +307,20 @@ type SCMProviderGenerator struct { Template ApplicationSetTemplate `json:"template,omitempty"` } +// SCMProviderGeneratorGitea defines a connection info specific to Gitea. +type SCMProviderGeneratorGitea struct { + // Gitea organization or user to scan. Required. + Owner string `json:"owner"` + // The Gitea URL to talk to. For example https://gitea.mydomain.com/. + API string `json:"api"` + // Authentication token reference. + TokenRef *SecretRef `json:"tokenRef,omitempty"` + // Scan all branches instead of just the default branch. + AllBranches bool `json:"allBranches,omitempty"` + // Allow self-signed TLS / Certificates; default: false + Insecure bool `json:"insecure,omitempty"` +} + // SCMProviderGeneratorGithub defines a connection info specific to GitHub. type SCMProviderGeneratorGithub struct { // GitHub org to scan. Required. @@ -362,11 +377,26 @@ type SCMProviderGeneratorFilter struct { type PullRequestGenerator struct { // Which provider to use and config for it. Github *PullRequestGeneratorGithub `json:"github,omitempty"` + Gitea *PullRequestGeneratorGitea `json:"gitea,omitempty"` // Standard parameters. RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"` Template ApplicationSetTemplate `json:"template,omitempty"` } +// PullRequestGenerator defines a connection info specific to Gitea. +type PullRequestGeneratorGitea struct { + // Gitea org or user to scan. Required. + Owner string `json:"owner"` + // Gitea repo name to scan. Required. + Repo string `json:"repo"` + // The Gitea API URL to talk to. Required + API string `json:"api"` + // Authentication token reference. + TokenRef *SecretRef `json:"tokenRef,omitempty"` + // Allow insecure tls, for self-signed certificates; default: false. + Insecure bool `json:"insecure,omitempty"` +} + // PullRequestGenerator defines a connection info specific to GitHub. type PullRequestGeneratorGithub struct { // GitHub org or user to scan. Required. diff --git a/pkg/apis/applicationset/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/applicationset/v1alpha1/zz_generated.deepcopy.go index e90291cefe0c8..613e154e98be7 100644 --- a/pkg/apis/applicationset/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/applicationset/v1alpha1/zz_generated.deepcopy.go @@ -660,6 +660,11 @@ func (in *PullRequestGenerator) DeepCopyInto(out *PullRequestGenerator) { *out = new(PullRequestGeneratorGithub) (*in).DeepCopyInto(*out) } + if in.Gitea != nil { + in, out := &in.Gitea, &out.Gitea + *out = new(PullRequestGeneratorGitea) + (*in).DeepCopyInto(*out) + } if in.RequeueAfterSeconds != nil { in, out := &in.RequeueAfterSeconds, &out.RequeueAfterSeconds *out = new(int64) @@ -678,6 +683,26 @@ func (in *PullRequestGenerator) DeepCopy() *PullRequestGenerator { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PullRequestGeneratorGitea) DeepCopyInto(out *PullRequestGeneratorGitea) { + *out = *in + if in.TokenRef != nil { + in, out := &in.TokenRef, &out.TokenRef + *out = new(SecretRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullRequestGeneratorGitea. +func (in *PullRequestGeneratorGitea) DeepCopy() *PullRequestGeneratorGitea { + if in == nil { + return nil + } + out := new(PullRequestGeneratorGitea) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PullRequestGeneratorGithub) DeepCopyInto(out *PullRequestGeneratorGithub) { *out = *in @@ -721,6 +746,11 @@ func (in *SCMProviderGenerator) DeepCopyInto(out *SCMProviderGenerator) { *out = new(SCMProviderGeneratorBitbucket) (*in).DeepCopyInto(*out) } + if in.Gitea != nil { + in, out := &in.Gitea, &out.Gitea + *out = new(SCMProviderGeneratorGitea) + (*in).DeepCopyInto(*out) + } if in.Filters != nil { in, out := &in.Filters, &out.Filters *out = make([]SCMProviderGeneratorFilter, len(*in)) @@ -801,6 +831,26 @@ func (in *SCMProviderGeneratorFilter) DeepCopy() *SCMProviderGeneratorFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SCMProviderGeneratorGitea) DeepCopyInto(out *SCMProviderGeneratorGitea) { + *out = *in + if in.TokenRef != nil { + in, out := &in.TokenRef, &out.TokenRef + *out = new(SecretRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SCMProviderGeneratorGitea. +func (in *SCMProviderGeneratorGitea) DeepCopy() *SCMProviderGeneratorGitea { + if in == nil { + return nil + } + out := new(SCMProviderGeneratorGitea) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SCMProviderGeneratorGithub) DeepCopyInto(out *SCMProviderGeneratorGithub) { *out = *in From 1787ab902929c41f9de6ecfb73d91c5b61de6824 Mon Sep 17 00:00:00 2001 From: Dan Molik Date: Mon, 4 Apr 2022 16:13:12 -0400 Subject: [PATCH 2/4] chore: white space in hack/test.sh re-trigger linting check Signed-off-by: Dan Molik --- hack/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/test.sh b/hack/test.sh index c4487ae5e8d3f..1ca29343be2b2 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -16,9 +16,9 @@ fi mkdir -p $TEST_RESULTS report() { - set -eux -o pipefail + set -eux -o pipefail - go-junit-report < $TEST_RESULTS/test.out > $TEST_RESULTS/junit.xml + go-junit-report < $TEST_RESULTS/test.out > $TEST_RESULTS/junit.xml } trap 'report' EXIT From 745ba224ac488457c0cb80c0b7fe7e2d9158b72e Mon Sep 17 00:00:00 2001 From: Dan Molik Date: Tue, 5 Apr 2022 14:10:20 -0400 Subject: [PATCH 3/4] chore: add gitea_scm and gitea_pr tests Signed-off-by: Dan Molik --- applicationset/services/pull_request/gitea.go | 8 +- .../services/pull_request/gitea_test.go | 19 ++++ .../services/scm_provider/gitea_test.go | 93 +++++++++++++++++++ 3 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 applicationset/services/pull_request/gitea_test.go create mode 100644 applicationset/services/scm_provider/gitea_test.go diff --git a/applicationset/services/pull_request/gitea.go b/applicationset/services/pull_request/gitea.go index 3520e60261c47..44d6bb04bc7c9 100644 --- a/applicationset/services/pull_request/gitea.go +++ b/applicationset/services/pull_request/gitea.go @@ -46,10 +46,6 @@ func NewGiteaService(ctx context.Context, token, url, owner, repo string, insecu func (g *GiteaService) List(ctx context.Context) ([]*PullRequest, error) { opts := gitea.ListPullRequestsOptions{ State: gitea.StateOpen, - /* ListOptions: gitea.ListOptions{ - Page: 1, - PageSize: 100, - },*/ } prs, _, err := g.client.ListRepoPullRequests(g.owner, g.repo, opts) if err != nil { @@ -58,8 +54,8 @@ func (g *GiteaService) List(ctx context.Context) ([]*PullRequest, error) { list := []*PullRequest{} for _, pr := range prs { list = append(list, &PullRequest{ - Number: int(pr.ID), - Branch: pr.Base.Ref, + Number: int(pr.Index), + Branch: pr.Head.Ref, HeadSHA: pr.Head.Sha, }) } diff --git a/applicationset/services/pull_request/gitea_test.go b/applicationset/services/pull_request/gitea_test.go new file mode 100644 index 0000000000000..f3647aca4e094 --- /dev/null +++ b/applicationset/services/pull_request/gitea_test.go @@ -0,0 +1,19 @@ +package pull_request + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGiteaList(t *testing.T) { + host, err := NewGiteaService(context.Background(), "", "https://gitea.com", "test-argocd", "pr-test", false) + assert.Nil(t, err) + prs, err := host.List(context.Background()) + assert.Nil(t, err) + assert.Equal(t, len(prs), 1) + assert.Equal(t, prs[0].Number, 1) + assert.Equal(t, prs[0].Branch, "test") + assert.Equal(t, prs[0].HeadSHA, "7bbaf62d92ddfafd9cc8b340c619abaec32bc09f") +} diff --git a/applicationset/services/scm_provider/gitea_test.go b/applicationset/services/scm_provider/gitea_test.go new file mode 100644 index 0000000000000..8c9d5c9aa3752 --- /dev/null +++ b/applicationset/services/scm_provider/gitea_test.go @@ -0,0 +1,93 @@ +package scm_provider + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" +) + +func TestGiteaListRepos(t *testing.T) { + cases := []struct { + name, proto, url string + hasError, allBranches, includeSubgroups bool + branches []string + filters []v1alpha1.SCMProviderGeneratorFilter + }{ + { + name: "blank protocol", + allBranches: false, + url: "git@gitea.com:gitea/go-sdk.git", + branches: []string{"master"}, + }, + { + name: "ssh protocol", + allBranches: false, + proto: "ssh", + url: "git@gitea.com:gitea/go-sdk.git", + }, + { + name: "https protocol", + allBranches: false, + proto: "https", + url: "https://gitea.com/gitea/go-sdk", + }, + { + name: "other protocol", + allBranches: false, + proto: "other", + hasError: true, + }, + { + name: "all branches", + allBranches: true, + url: "git@gitea.com:gitea/go-sdk.git", + branches: []string{"master", "release/v0.11", "release/v0.12", "release/v0.13", "release/v0.14", "release/v0.15"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + provider, _ := NewGiteaProvider(context.Background(), "gitea", "", "https://gitea.com/", c.allBranches, false) + rawRepos, err := ListRepos(context.Background(), provider, c.filters, c.proto) + if c.hasError { + assert.NotNil(t, err) + } else { + checkRateLimit(t, err) + assert.Nil(t, err) + // Just check that this one project shows up. Not a great test but better thing nothing? + repos := []*Repository{} + branches := []string{} + for _, r := range rawRepos { + if r.Repository == "go-sdk" { + repos = append(repos, r) + branches = append(branches, r.Branch) + } + } + assert.NotEmpty(t, repos) + assert.Equal(t, c.url, repos[0].URL) + for _, b := range c.branches { + assert.Contains(t, branches, b) + } + } + }) + } +} + +func TestGiteaHasPath(t *testing.T) { + host, _ := NewGiteaProvider(context.Background(), "gitea", "", "https://gitea.com/", false, false) + repo := &Repository{ + Organization: "gitea", + Repository: "go-sdk", + Branch: "master", + } + ok, err := host.RepoHasPath(context.Background(), repo, "README.md") + assert.Nil(t, err) + assert.True(t, ok) + + ok, err = host.RepoHasPath(context.Background(), repo, "notathing") + assert.Nil(t, err) + assert.False(t, ok) +} From 872b1b4a2bd82f4eced1fe59607b9150167093d3 Mon Sep 17 00:00:00 2001 From: Dan Molik Date: Wed, 6 Apr 2022 06:31:34 -0400 Subject: [PATCH 4/4] bug: ensure gitea scm haspath detects directories correctly Signed-off-by: Dan Molik --- applicationset/services/scm_provider/gitea.go | 3 +++ applicationset/services/scm_provider/gitea_test.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/applicationset/services/scm_provider/gitea.go b/applicationset/services/scm_provider/gitea.go index 4641ea6151cf0..15e8f1a9adbc1 100644 --- a/applicationset/services/scm_provider/gitea.go +++ b/applicationset/services/scm_provider/gitea.go @@ -126,6 +126,9 @@ func (g *GiteaProvider) RepoHasPath(ctx context.Context, repo *Repository, path if resp != nil && resp.StatusCode == 404 { return false, nil } + if fmt.Sprint(err) == "expect file, got directory" { + return true, nil + } if err != nil { return false, err } diff --git a/applicationset/services/scm_provider/gitea_test.go b/applicationset/services/scm_provider/gitea_test.go index 8c9d5c9aa3752..e3992b3987371 100644 --- a/applicationset/services/scm_provider/gitea_test.go +++ b/applicationset/services/scm_provider/gitea_test.go @@ -87,6 +87,11 @@ func TestGiteaHasPath(t *testing.T) { assert.Nil(t, err) assert.True(t, ok) + // directory + ok, err = host.RepoHasPath(context.Background(), repo, "gitea") + assert.Nil(t, err) + assert.True(t, ok) + ok, err = host.RepoHasPath(context.Background(), repo, "notathing") assert.Nil(t, err) assert.False(t, ok)