Add support for generating based on scraping the Github API.

api/v1alpha1/applicationset_types.go

@@ -21,6 +21,12 @@ import (
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// Utility struct for a reference to a secret key.
+type SecretRef struct {
+ SecretName string `json:"secretName"`
+ Key string `json:"key"`
+}
+
 // ApplicationSet is a set of Application resources
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:path=applicationsets,shortName=appset;appsets
@@ -65,9 +71,10 @@ type ApplicationSetTemplateMeta struct {
 
 // ApplicationSetGenerator include list item info
 type ApplicationSetGenerator struct {
- List *ListGenerator `json:"list,omitempty"`
- Clusters *ClusterGenerator `json:"clusters,omitempty"`
- Git *GitGenerator `json:"git,omitempty"`
+ List *ListGenerator `json:"list,omitempty"`
+ Clusters *ClusterGenerator `json:"clusters,omitempty"`
+ Git *GitGenerator `json:"git,omitempty"`
+ SCMProvider *SCMProviderGenerator `json:"scmProvider,omitempty"`
 }
 
 // ListGenerator include items info
@@ -114,6 +121,47 @@ type GitFileGeneratorItem struct {
  Path string `json:"path"`
 }
 
+// SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos.
+type SCMProviderGenerator struct {
+ // Which provider to use and config for it.
+ Github *SCMProviderGeneratorGithub `json:"github,omitempty"`
+ // TODO other providers.
+ // Filters for which repos should be considered.
+ Filters []SCMProviderGeneratorFilter `json:"filters,omitempty"`
+ // Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers
+ // necessarily support all protocols.
+ CloneProtocol string `json:"cloneProtocol,omitempty"`
+ // Standard parameters.
+ RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"`
+ Template ApplicationSetTemplate `json:"template,omitempty"`
+}
+
+// SCMProviderGeneratorGithub defines a connection info specific to GitHub.
+type SCMProviderGeneratorGithub struct {
+ // GitHub org to scan. Required.
+ Organization string `json:"organization"`
+ // The GitHub API URL to talk to. If blank, use https://api.github.com/.
+ API string `json:"api,omitempty"`
+ // Authentication token reference.
+ TokenRef *SecretRef `json:"tokenRef,omitempty"`
+ // Scan all branches instead of just the default branch.
+ AllBranches bool `json:"allBranches,omitempty"`
+}
+
+// SCMProviderGeneratorFilter is a single repository filter.
+// If multiple filter types are set on a single struct, they will be AND'd together. All filters must
+// pass for a repo to be included.
+type SCMProviderGeneratorFilter struct {
+ // A regex for repo names.
+ RepositoryMatch *string `json:"repositoryMatch,omitempty"`
+ // An array of paths, all of which must exist.
+ PathsExist []string `json:"pathsExist,omitempty"`
+ // A regex which must match at least one label.
+ LabelMatch *string `json:"labelMatch,omitempty"`
+ // A regex which must match the branch name.
+ BranchMatch *string `json:"branchMatch,omitempty"`
+}
+
 // ApplicationSetStatus defines the observed state of ApplicationSet
 type ApplicationSetStatus struct {
  // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster

docs/Generators.md

@@ -367,3 +367,124 @@ spec:
 Any `config.json` files found under the `cluster-config` directory will be parameterized based on the `path` wildcard pattern specified. Within each file JSON fields are flattened into key/value pairs, with this ApplicationSet example using the `cluster.address` as `cluster.name` parameters in the template.
 
 As with other generators, clusters *must* already be defined within Argo CD, in order to generate Applications for them.
+
+## SCM Provider Generator
+
+The SCMProvider generator uses the API of an SCMaaS provider to discover repositories. This fits well with many repos following the same GitOps layout patterns such as microservices.
+
+Support is currently limited to GitHub, PRs are welcome to add more SCM providers.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+ name: myapps
+spec:
+ generators:
+ - scmProvider:
+ # Which protocol to clone using.
+ cloneProtocol: ssh
+ # See below for provider specific options.
+ github:
+ # ...
+```
+
+* `cloneProtocol`: Which protocol to use for the SCM URL. Default is provider-specific but ssh if possible. Not all providers necessarily support all protocols, see provider documentation below for available options.
+
+### GitHub
+
+The GitHub mode uses the GitHub API to scan and organization in either github.com or GitHub Enterprise.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+ name: myapps
+spec:
+ generators:
+ - scmProvider:
+ github:
+ # The GitHub organization to scan.
+ organization: myorg
+ # For GitHub Enterprise:
+ api: https://git.example.com/
+ # If true, scan every branch of every repository. If false, scan only the default branch. Defaults to false.
+ allBranches: true
+ # Reference to a Secret containing an access token. (optional)
+ tokenRef:
+ secretName: github-token
+ key: token
+ template:
+ # ...
+```
+
+* `organization`: Required name of the GitHub organization to scan. If you have multiple orgs, use multiple generators.
+* `api`: If using GitHub Enterprise, the URL to access it.
+* `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter.
+* `tokenRef`: A Secret name and key containing the GitHub access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories.
+
+For label filtering, the repository topics are used.
+
+Available clone protocols are `ssh` and `https`.
+
+### Filters
+
+Filters allow selecting which repositories to generate for. Each filter can declare one or more conditions, all of which must pass. If multiple filters are present, any can match for a repository to be included. If no filters are specified, all repositories will be processed.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+ name: myapps
+spec:
+ generators:
+ - scmProvider:
+ filters:
+ # Include any repository starting with "myapp" AND including a Kustomize config AND labeled with "deploy-ok" ...
+ - repositoryMatch: ^myapp
+ pathsExist: [kubernetes/kustomization.yaml]
+ labelMatch: deploy-ok
+ # ... OR any repository starting with "otherapp" AND a Helm folder.
+ - repositoryMatch: ^otherapp
+ pathsExist: [helm]
+ template:
+ # ...
+```
+
+* `repositoryMatch`: A regexp matched against the repository name.
+* `pathsExist`: An array of paths within the repository that must exist. Can be a file or directory, but do not include the trailing `/` for directories.
+* `labelMatch`: A regexp matched against repository labels. If any label matches, the repository is included.
+* `branchMatch`: A regexp matched against branch names.
+
+### Template
+
+As with all generators, several keys are available for replacement in the generated application.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+ name: myapps
+spec:
+ generators:
+ - scmProvider:
+ # ...
+ template:
+ metadata:
+ name: '{{ repository }}'
+ spec:
+ source:
+ repoURL: '{{ url }}'
+ targetRevision: '{{ branch }}'
+ path: kubernetes/
+ project: default
+ destination:
+ server: https://kubernetes.default.svc
+ namespace: default
+```
+
+* `organization`: The name of the organization the repository is in.
+* `repository`: The name of the repository.
+* `url`: The clone URL for the repository.
+* `branch`: The default branch of the repository.
+

examples/scm-provider-generator/scm-provider-example.yaml

@@ -0,0 +1,24 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+ name: guestbook
+spec:
+ generators:
+ - scmProvider:
+ github:
+ organization: argoproj
+ cloneProtocol: https
+ filters:
+ - repositoryMatch: example-apps
+ template:
+ metadata:
+ name: '{{ repository }}-guestbook'
+ spec:
+ project: "default"
+ source:
+ repoURL: '{{ url }}'
+ targetRevision: '{{ branch }}'
+ path: guestbook
+ destination:
+ server: https://kubernetes.default.svc
+ namespace: guestbook

go.mod

@@ -7,12 +7,14 @@ require (
  github.com/argoproj/gitops-engine v0.2.1
  github.com/argoproj/pkg v0.2.0
  github.com/go-logr/logr v0.3.0
+ github.com/google/go-github/v35 v35.0.0
  github.com/imdario/mergo v0.3.10
  github.com/jeremywohl/flatten v1.0.1
  github.com/pkg/errors v0.9.1
  github.com/sirupsen/logrus v1.6.0
  github.com/stretchr/testify v1.6.1
  github.com/valyala/fasttemplate v1.2.1
+ golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78
  k8s.io/api v0.19.2
  k8s.io/apimachinery v0.19.2
  k8s.io/client-go v11.0.1-0.20190816222228-6d55c1b1f1ca+incompatible