feat: support refreshing applicationset using webhook

Currently, the repository used in git-generator will be polled every 3 mins. This commit adds support to refresh the applicationset using a git webhook. It exposes a service listening for incoming webhook payloads and triggers the applicationset controller.

api/v1alpha1/applicationset_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+ "github.com/argoproj-labs/applicationset/common"
  "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
  apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -220,3 +221,9 @@ type ApplicationSetList struct {
 func init() {
  SchemeBuilder.Register(&ApplicationSet{}, &ApplicationSetList{})
 }
+
+// RefreshRequired checks if the ApplicationSet needs to be refreshed
+func (a *ApplicationSet) RefreshRequired() bool {
+ _, found := a.Annotations[common.AnnotationGitGeneratorRefresh]
+ return found
+}

common/annotations.go

@@ -0,0 +1,6 @@
+package common
+
+const (
+ // AnnotationGitGeneratorRefresh is an annotation that is added when an ApplicationSet with the git generator is requested to be refreshed by a webhook. The ApplicationSet controller will remove this annotation at the end of reconcilation.
+ AnnotationGitGeneratorRefresh = "argocd.argoproj.io/git-gen-refresh"
+)

go.mod

@@ -16,6 +16,7 @@ require (
  github.com/valyala/fasttemplate v1.2.1
  github.com/xanzy/go-gitlab v0.50.0
  golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
+ gopkg.in/go-playground/webhooks.v5 v5.11.0
  k8s.io/api v0.21.1
  k8s.io/apiextensions-apiserver v0.21.1
  k8s.io/apimachinery v0.21.1

go.sum

@@ -1179,6 +1179,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/go-playground/webhooks.v5 v5.11.0 h1:V3vej+ZXrVvO2EmBTKlhClEbpTqXH44K5OyLUMOkHMg=
 gopkg.in/go-playground/webhooks.v5 v5.11.0/go.mod h1:LZbya/qLVdbqDR1aKrGuWV6qbia2zCYSR5dpom2SInQ=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=

main.go

@@ -20,6 +20,7 @@ import (
  "context"
  "flag"
  "fmt"
+ "net/http"
  "os"
  "time"
 
@@ -139,6 +140,10 @@ func main() {
 
  argoCDDB := db.NewDB(namespace, argoSettingsMgr, k8s)
 
+ // start a webhook server that listens to incoming webhook payloads
+ webhookHandler := utils.NewWebhookHandler(namespace, argoSettingsMgr, mgr.GetClient())
+ startWebhookServer(webhookHandler)
+
  baseGenerators := map[string]generators.Generator{
  "List": generators.NewListGenerator(),
  "Clusters": generators.NewClusterGenerator(mgr.GetClient(), context.Background(), k8s, namespace),
@@ -197,3 +202,16 @@ func setLoggingLevel(debug bool, logLevel string) {
  log.SetLevel(level)
  }
 }
+
+func startWebhookServer(webhookHandler *utils.WebhookHandler) {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/webhook", webhookHandler.Handler)
+ go func() {
+ setupLog.Info("Starting webhook server")
+ err := http.ListenAndServe(":7000", mux)
+ if err != nil {
+ setupLog.Error(err, "failed to start webhook server")
+ os.Exit(1)
+ }
+ }()
+}

manifests/base/deployment.yaml

@@ -21,6 +21,8 @@ spec:
  image: quay.io/argoproj/argocd-applicationset:latest
  imagePullPolicy: Always
  name: argocd-applicationset-controller
+ ports:
+ - containerPort: 7000
  env:
  - name: NAMESPACE
  valueFrom:

manifests/base/kustomization.yaml

@@ -11,3 +11,4 @@ namespace: argocd
 resources:
 - deployment.yaml
 - rbac.yaml
+- service.yaml

manifests/base/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/component: controller
+ app.kubernetes.io/name: argocd-applicationset-controller
+ app.kubernetes.io/part-of: argocd-applicationset
+ name: argocd-applicationset-controller
+ namespace: argocd
+spec:
+ ports:
+ - name: webhook
+ port: 7000
+ protocol: TCP
+ targetPort: 7000
+ selector:
+ app.kubernetes.io/name: argocd-applicationset-controller

manifests/install-with-argo-cd.yaml

@@ -7043,6 +7043,24 @@ type: Opaque
 ---
 apiVersion: v1
 kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/component: controller
+ app.kubernetes.io/name: argocd-applicationset-controller
+ app.kubernetes.io/part-of: argocd-applicationset
+ name: argocd-applicationset-controller
+ namespace: argocd
+spec:
+ ports:
+ - name: webhook
+ port: 7000
+ protocol: TCP
+ targetPort: 7000
+ selector:
+ app.kubernetes.io/name: argocd-applicationset-controller
+---
+apiVersion: v1
+kind: Service
 metadata:
  labels:
  app.kubernetes.io/component: dex-server
@@ -7193,6 +7211,8 @@ spec:
  image: quay.io/argoproj/argocd-applicationset:latest
  imagePullPolicy: Always
  name: argocd-applicationset-controller
+ ports:
+ - containerPort: 7000
  volumeMounts:
  - mountPath: /app/config/ssh
  name: ssh-known-hosts

manifests/install.yaml

@@ -4637,6 +4637,24 @@ subjects:
  name: argocd-applicationset-controller
  namespace: argocd
 ---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/component: controller
+ app.kubernetes.io/name: argocd-applicationset-controller
+ app.kubernetes.io/part-of: argocd-applicationset
+ name: argocd-applicationset-controller
+ namespace: argocd
+spec:
+ ports:
+ - name: webhook
+ port: 7000
+ protocol: TCP
+ targetPort: 7000
+ selector:
+ app.kubernetes.io/name: argocd-applicationset-controller
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -4666,6 +4684,8 @@ spec:
  image: quay.io/argoproj/argocd-applicationset:latest
  imagePullPolicy: Always
  name: argocd-applicationset-controller
+ ports:
+ - containerPort: 7000
  volumeMounts:
  - mountPath: /app/config/ssh
  name: ssh-known-hosts

pkg/controllers/applicationset_controller.go

@@ -19,6 +19,7 @@ import (
  "fmt"
  "time"
 
+ "github.com/argoproj-labs/applicationset/common"
  "github.com/argoproj-labs/applicationset/pkg/generators"
  "github.com/argoproj-labs/applicationset/pkg/utils"
  argov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
@@ -126,6 +127,15 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque
  }
  }
 
+ if applicationSetInfo.RefreshRequired() {
+ delete(applicationSetInfo.Annotations, common.AnnotationGitGeneratorRefresh)
+ err := r.Client.Update(ctx, &applicationSetInfo)
+ if err != nil {
+ log.Warnf("error occurred while updating ApplicationSet: %v", err)
+ return ctrl.Result{}, err
+ }
+ }
+
  requeueAfter := r.getMinRequeueAfter(&applicationSetInfo)
  log.WithField("requeueAfter", requeueAfter).Info("end reconcile")
 

pkg/utils/testdata/github-commit-event.json

@@ -0,0 +1,186 @@
+{
+ "ref": "refs/heads/master",
+ "before": "d5c1ffa8e294bc18c639bfb4e0df499251034414",
+ "after": "63738bb582c8b540af7bcfc18f87c575c3ed66e0",
+ "created": false,
+ "deleted": false,
+ "forced": true,
+ "base_ref": null,
+ "compare": "https://github.com/org/repo/compare/d5c1ffa8e294...63738bb582c8",
+ "commits": [
+ {
+ "id": "63738bb582c8b540af7bcfc18f87c575c3ed66e0",
+ "tree_id": "64897da445207e409ad05af93b1f349ad0a4ee19",
+ "distinct": true,
+ "message": "Add staging-argocd-demo environment",
+ "timestamp": "2018-05-04T15:40:02-07:00",
+ "url": "https://github.com/org/repo/commit/63738bb582c8b540af7bcfc18f87c575c3ed66e0",
+ "author": {
+ "name": "Jesse Suen",
+ "email": "Jesse_Suen@example.com",
+ "username": "org"
+ },
+ "committer": {
+ "name": "Jesse Suen",
+ "email": "Jesse_Suen@example.com",
+ "username": "org"
+ },
+ "added": [
+ "ksapps/test-app/environments/staging-argocd-demo/main.jsonnet",
+ "ksapps/test-app/environments/staging-argocd-demo/params.libsonnet"
+ ],
+ "removed": [
+
+ ],
+ "modified": [
+ "ksapps/test-app/app.yaml"
+ ]
+ }
+ ],
+ "head_commit": {
+ "id": "63738bb582c8b540af7bcfc18f87c575c3ed66e0",
+ "tree_id": "64897da445207e409ad05af93b1f349ad0a4ee19",
+ "distinct": true,
+ "message": "Add staging-argocd-demo environment",
+ "timestamp": "2018-05-04T15:40:02-07:00",
+ "url": "https://github.com/org/repo/commit/63738bb582c8b540af7bcfc18f87c575c3ed66e0",
+ "author": {
+ "name": "Jesse Suen",
+ "email": "Jesse_Suen@example.com",
+ "username": "org"
+ },
+ "committer": {
+ "name": "Jesse Suen",
+ "email": "Jesse_Suen@example.com",
+ "username": "org"
+ },
+ "added": [
+ "ksapps/test-app/environments/staging-argocd-demo/main.jsonnet",
+ "ksapps/test-app/environments/staging-argocd-demo/params.libsonnet"
+ ],
+ "removed": [
+
+ ],
+ "modified": [
+ "ksapps/test-app/app.yaml"
+ ]
+ },
+ "repository": {
+ "id": 123060978,
+ "name": "repo",
+ "full_name": "org/repo",
+ "owner": {
+ "name": "org",
+ "email": "org@users.noreply.github.com",
+ "login": "org",
+ "id": 12677113,
+ "avatar_url": "https://avatars0.githubusercontent.com/u/12677113?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/org",
+ "html_url": "https://github.com/org",
+ "followers_url": "https://api.github.com/users/org/followers",
+ "following_url": "https://api.github.com/users/org/following{/other_user}",
+ "gists_url": "https://api.github.com/users/org/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/org/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/org/subscriptions",
+ "organizations_url": "https://api.github.com/users/org/orgs",
+ "repos_url": "https://api.github.com/users/org/repos",
+ "events_url": "https://api.github.com/users/org/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/org/received_events",
+ "type": "User",
+ "site_admin": false
+ },
+ "private": false,
+ "html_url": "https://github.com/org/repo",
+ "description": "Test Repository",
+ "fork": false,
+ "url": "https://github.com/org/repo",
+ "forks_url": "https://api.github.com/repos/org/repo/forks",
+ "keys_url": "https://api.github.com/repos/org/repo/keys{/key_id}",
+ "collaborators_url": "https://api.github.com/repos/org/repo/collaborators{/collaborator}",
+ "teams_url": "https://api.github.com/repos/org/repo/teams",
+ "hooks_url": "https://api.github.com/repos/org/repo/hooks",
+ "issue_events_url": "https://api.github.com/repos/org/repo/issues/events{/number}",
+ "events_url": "https://api.github.com/repos/org/repo/events",
+ "assignees_url": "https://api.github.com/repos/org/repo/assignees{/user}",
+ "branches_url": "https://api.github.com/repos/org/repo/branches{/branch}",
+ "tags_url": "https://api.github.com/repos/org/repo/tags",
+ "blobs_url": "https://api.github.com/repos/org/repo/git/blobs{/sha}",
+ "git_tags_url": "https://api.github.com/repos/org/repo/git/tags{/sha}",
+ "git_refs_url": "https://api.github.com/repos/org/repo/git/refs{/sha}",
+ "trees_url": "https://api.github.com/repos/org/repo/git/trees{/sha}",
+ "statuses_url": "https://api.github.com/repos/org/repo/statuses/{sha}",
+ "languages_url": "https://api.github.com/repos/org/repo/languages",
+ "stargazers_url": "https://api.github.com/repos/org/repo/stargazers",
+ "contributors_url": "https://api.github.com/repos/org/repo/contributors",
+ "subscribers_url": "https://api.github.com/repos/org/repo/subscribers",
+ "subscription_url": "https://api.github.com/repos/org/repo/subscription",
+ "commits_url": "https://api.github.com/repos/org/repo/commits{/sha}",
+ "git_commits_url": "https://api.github.com/repos/org/repo/git/commits{/sha}",
+ "comments_url": "https://api.github.com/repos/org/repo/comments{/number}",
+ "issue_comment_url": "https://api.github.com/repos/org/repo/issues/comments{/number}",
+ "contents_url": "https://api.github.com/repos/org/repo/contents/{+path}",
+ "compare_url": "https://api.github.com/repos/org/repo/compare/{base}...{head}",
+ "merges_url": "https://api.github.com/repos/org/repo/merges",
+ "archive_url": "https://api.github.com/repos/org/repo/{archive_format}{/ref}",
+ "downloads_url": "https://api.github.com/repos/org/repo/downloads",
+ "issues_url": "https://api.github.com/repos/org/repo/issues{/number}",
+ "pulls_url": "https://api.github.com/repos/org/repo/pulls{/number}",
+ "milestones_url": "https://api.github.com/repos/org/repo/milestones{/number}",
+ "notifications_url": "https://api.github.com/repos/org/repo/notifications{?since,all,participating}",
+ "labels_url": "https://api.github.com/repos/org/repo/labels{/name}",
+ "releases_url": "https://api.github.com/repos/org/repo/releases{/id}",
+ "deployments_url": "https://api.github.com/repos/org/repo/deployments",
+ "created_at": 1519698615,
+ "updated_at": "2018-05-04T22:37:55Z",
+ "pushed_at": 1525473610,
+ "git_url": "git://github.com/org/repo.git",
+ "ssh_url": "git@github.com:org/repo.git",
+ "clone_url": "https://github.com/org/repo.git",
+ "svn_url": "https://github.com/org/repo",
+ "homepage": null,
+ "size": 538,
+ "stargazers_count": 0,
+ "watchers_count": 0,
+ "language": null,
+ "has_issues": true,
+ "has_projects": true,
+ "has_downloads": true,
+ "has_wiki": true,
+ "has_pages": false,
+ "forks_count": 1,
+ "mirror_url": null,
+ "archived": false,
+ "open_issues_count": 0,
+ "license": null,
+ "forks": 1,
+ "open_issues": 0,
+ "watchers": 0,
+ "default_branch": "master",
+ "stargazers": 0,
+ "master_branch": "master"
+ },
+ "pusher": {
+ "name": "org",
+ "email": "org@users.noreply.github.com"
+ },
+ "sender": {
+ "login": "org",
+ "id": 12677113,
+ "avatar_url": "https://avatars0.githubusercontent.com/u/12677113?v=4",
+ "gravatar_id": "",
+ "url": "https://api.github.com/users/org",
+ "html_url": "https://github.com/org",
+ "followers_url": "https://api.github.com/users/org/followers",
+ "following_url": "https://api.github.com/users/org/following{/other_user}",
+ "gists_url": "https://api.github.com/users/org/gists{/gist_id}",
+ "starred_url": "https://api.github.com/users/org/starred{/owner}{/repo}",
+ "subscriptions_url": "https://api.github.com/users/org/subscriptions",
+ "organizations_url": "https://api.github.com/users/org/orgs",
+ "repos_url": "https://api.github.com/users/org/repos",
+ "events_url": "https://api.github.com/users/org/events{/privacy}",
+ "received_events_url": "https://api.github.com/users/org/received_events",
+ "type": "User",
+ "site_admin": false
+ }
+ }