feat: refresh ApplicationSet resource with git generator using webhook

api/v1alpha1/applicationset_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+ "github.com/argoproj-labs/applicationset/common"
  "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
  apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -220,3 +221,9 @@ type ApplicationSetList struct {
 func init() {
  SchemeBuilder.Register(&ApplicationSet{}, &ApplicationSetList{})
 }
+
+// RefreshRequired checks if the ApplicationSet needs to be refreshed
+func (a *ApplicationSet) RefreshRequired() bool {
+ _, found := a.Annotations[common.AnnotationGitGeneratorRefresh]
+ return found
+}

common/annotations.go

@@ -0,0 +1,6 @@
+package common
+
+const (
+ // AnnotationGitGeneratorRefresh is an annotation that is added when an ApplicationSet with the git generator is requested to be refreshed by a webhook. The ApplicationSet controller will remove this annotation at the end of reconcilation.
+ AnnotationGitGeneratorRefresh = "argocd.argoproj.io/git-gen-refresh"
+)

docs/Generators-Git.md

@@ -225,3 +225,65 @@ In addition to the flattened key/value pairs from the configuration file, the fo
 
 - `{{path}}`: The path to the matching configuration file within the Git repository. Example: `/cluster-config/staging/config.json`
 - `{{path.basename}}`: The filename of the configuration file. Example: `config.json`
+
+## Webhook Configuration
+
+When using a Git generator, ApplicationSet polls Git repositories every three minutes to detect changes. To eliminate
+this delay from polling, the ApplicationSet webhook server can be configured to receive webhook events. ApplicationSet supports
+Git webhook notifications from GitHub and GitLab. The following explains how to configure a Git webhook for GitHub, but the same process should be applicable to other providers.
+
+!!! note
+ ApplicationSet exposes the webhook server as a service of type ClusterIP. An Ingress resource needs to be created to expose this service to the webhook source.
+
+### 1. Create The WebHook In The Git Provider
+
+In your Git provider, navigate to the settings page where webhooks can be configured. The payload
+URL configured in the Git provider should use the `/api/webhook` endpoint of your ApplicationSet instance
+(e.g. `https://applicationset.example.com/api/webhook`). If you wish to use a shared secret, input an
+arbitrary value in the secret. This value will be used when configuring the webhook in the next step.
+
+![Add Webhook](./assets/webhook-config.png "Add Webhook")
+
+!!! note
+ When creating the webhook in GitHub, the "Content type" needs to be set to "application/json". The default value "application/x-www-form-urlencoded" is not supported by the library used to handle the hooks
+
+### 2. Configure ApplicationSet With The WebHook Secret (Optional)
+
+Configuring a webhook shared secret is optional, since ApplicationSet will still refresh applications
+generated by Git generators, even with unauthenticated webhook events. This is safe to do since
+the contents of webhook payloads are considered untrusted, and will only result in a refresh of the
+application (a process which already occurs at three-minute intervals). If ApplicationSet is publicly
+accessible, then configuring a webhook secret is recommended to prevent a DDoS attack.
+
+In the `argocd-secret` kubernetes secret, include the Git provider's webhook secret configured in step 1.
+
+Edit the Argo CD kubernetes secret:
+
+```bash
+kubectl edit secret argocd-secret -n argocd
+```
+
+TIP: for ease of entering secrets, kubernetes supports inputting secrets in the `stringData` field,
+which saves you the trouble of base64 encoding the values and copying it to the `data` field.
+Simply copy the shared webhook secret created in step 1, to the corresponding
+GitHub/GitLab/BitBucket key under the `stringData` field:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+ name: argocd-secret
+ namespace: argocd
+type: Opaque
+data:
+...
+
+stringData:
+ # github webhook secret
+ webhook.github.secret: shhhh! it's a github secret
+
+ # gitlab webhook secret
+ webhook.gitlab.secret: shhhh! it's a gitlab secret
+```
+
+After saving, please restart the ApplicationSet pod for the changes to take effect.

go.mod

@@ -16,6 +16,7 @@ require (
  github.com/valyala/fasttemplate v1.2.1
  github.com/xanzy/go-gitlab v0.50.0
  golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
+ gopkg.in/go-playground/webhooks.v5 v5.11.0
  k8s.io/api v0.21.1
  k8s.io/apiextensions-apiserver v0.21.1
  k8s.io/apimachinery v0.21.1

go.sum

@@ -1179,6 +1179,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
+gopkg.in/go-playground/webhooks.v5 v5.11.0 h1:V3vej+ZXrVvO2EmBTKlhClEbpTqXH44K5OyLUMOkHMg=
 gopkg.in/go-playground/webhooks.v5 v5.11.0/go.mod h1:LZbya/qLVdbqDR1aKrGuWV6qbia2zCYSR5dpom2SInQ=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=

main.go

@@ -20,6 +20,7 @@ import (
  "context"
  "flag"
  "fmt"
+ "net/http"
  "os"
  "time"
 
@@ -139,6 +140,14 @@ func main() {
 
  argoCDDB := db.NewDB(namespace, argoSettingsMgr, k8s)
 
+ // start a webhook server that listens to incoming webhook payloads
+ webhookHandler, err := utils.NewWebhookHandler(namespace, argoSettingsMgr, mgr.GetClient())
+ if err != nil {
+ setupLog.Error(err, "failed to create webhook handler")
+ os.Exit(1)
+ }
+ startWebhookServer(webhookHandler)
+
  baseGenerators := map[string]generators.Generator{
  "List": generators.NewListGenerator(),
  "Clusters": generators.NewClusterGenerator(mgr.GetClient(), context.Background(), k8s, namespace),
@@ -197,3 +206,16 @@ func setLoggingLevel(debug bool, logLevel string) {
  log.SetLevel(level)
  }
 }
+
+func startWebhookServer(webhookHandler *utils.WebhookHandler) {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/webhook", webhookHandler.Handler)
+ go func() {
+ setupLog.Info("Starting webhook server")
+ err := http.ListenAndServe(":7000", mux)
+ if err != nil {
+ setupLog.Error(err, "failed to start webhook server")
+ os.Exit(1)
+ }
+ }()
+}

manifests/base/deployment.yaml

@@ -21,6 +21,8 @@ spec:
  image: quay.io/argoproj/argocd-applicationset:latest
  imagePullPolicy: Always
  name: argocd-applicationset-controller
+ ports:
+ - containerPort: 7000
  env:
  - name: NAMESPACE
  valueFrom:

manifests/base/kustomization.yaml

@@ -11,3 +11,4 @@ namespace: argocd
 resources:
 - deployment.yaml
 - rbac.yaml
+- service.yaml

manifests/base/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/component: controller
+ app.kubernetes.io/name: argocd-applicationset-controller
+ app.kubernetes.io/part-of: argocd-applicationset
+ name: argocd-applicationset-controller
+ namespace: argocd
+spec:
+ ports:
+ - name: webhook
+ port: 7000
+ protocol: TCP
+ targetPort: 7000
+ selector:
+ app.kubernetes.io/name: argocd-applicationset-controller

manifests/install-with-argo-cd.yaml

@@ -7072,6 +7072,24 @@ type: Opaque
 ---
 apiVersion: v1
 kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/component: controller
+ app.kubernetes.io/name: argocd-applicationset-controller
+ app.kubernetes.io/part-of: argocd-applicationset
+ name: argocd-applicationset-controller
+ namespace: argocd
+spec:
+ ports:
+ - name: webhook
+ port: 7000
+ protocol: TCP
+ targetPort: 7000
+ selector:
+ app.kubernetes.io/name: argocd-applicationset-controller
+---
+apiVersion: v1
+kind: Service
 metadata:
  labels:
  app.kubernetes.io/component: dex-server
@@ -7222,6 +7240,8 @@ spec:
  image: quay.io/argoproj/argocd-applicationset:latest
  imagePullPolicy: Always
  name: argocd-applicationset-controller
+ ports:
+ - containerPort: 7000
  volumeMounts:
  - mountPath: /app/config/ssh
  name: ssh-known-hosts

manifests/install.yaml

@@ -4637,6 +4637,24 @@ subjects:
  name: argocd-applicationset-controller
  namespace: argocd
 ---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/component: controller
+ app.kubernetes.io/name: argocd-applicationset-controller
+ app.kubernetes.io/part-of: argocd-applicationset
+ name: argocd-applicationset-controller
+ namespace: argocd
+spec:
+ ports:
+ - name: webhook
+ port: 7000
+ protocol: TCP
+ targetPort: 7000
+ selector:
+ app.kubernetes.io/name: argocd-applicationset-controller
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -4666,6 +4684,8 @@ spec:
  image: quay.io/argoproj/argocd-applicationset:latest
  imagePullPolicy: Always
  name: argocd-applicationset-controller
+ ports:
+ - containerPort: 7000
  volumeMounts:
  - mountPath: /app/config/ssh
  name: ssh-known-hosts

pkg/controllers/applicationset_controller.go

@@ -19,6 +19,7 @@ import (
  "fmt"
  "time"
 
+ "github.com/argoproj-labs/applicationset/common"
  "github.com/argoproj-labs/applicationset/pkg/generators"
  "github.com/argoproj-labs/applicationset/pkg/utils"
  argov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
@@ -126,6 +127,15 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque
  }
  }
 
+ if applicationSetInfo.RefreshRequired() {
+ delete(applicationSetInfo.Annotations, common.AnnotationGitGeneratorRefresh)
+ err := r.Client.Update(ctx, &applicationSetInfo)
+ if err != nil {
+ log.Warnf("error occurred while updating ApplicationSet: %v", err)
+ return ctrl.Result{}, err
+ }
+ }
+
  requeueAfter := r.getMinRequeueAfter(&applicationSetInfo)
  log.WithField("requeueAfter", requeueAfter).Info("end reconcile")