feat: add the ability to exclude files when using the git file genera…

…tor (#468)

Signed-off-by: Adam Johnson <adamjohnson01@gmail.com>

api/v1alpha1/applicationset_types.go

@@ -272,23 +272,19 @@ type DuckTypeGenerator struct {
 }
 
 type GitGenerator struct {
-	RepoURL             string                      `json:"repoURL"`
-	Directories         []GitDirectoryGeneratorItem `json:"directories,omitempty"`
-	Files               []GitFileGeneratorItem      `json:"files,omitempty"`
-	Revision            string                      `json:"revision"`
-	RequeueAfterSeconds *int64                      `json:"requeueAfterSeconds,omitempty"`
-	Template            ApplicationSetTemplate      `json:"template,omitempty"`
+	RepoURL             string                 `json:"repoURL"`
+	Directories         []GitGeneratorItem     `json:"directories,omitempty"`
+	Files               []GitGeneratorItem     `json:"files,omitempty"`
+	Revision            string                 `json:"revision"`
+	RequeueAfterSeconds *int64                 `json:"requeueAfterSeconds,omitempty"`
+	Template            ApplicationSetTemplate `json:"template,omitempty"`
 }
 
-type GitDirectoryGeneratorItem struct {
+type GitGeneratorItem struct {
 	Path    string `json:"path"`
 	Exclude bool   `json:"exclude,omitempty"`
 }
 
-type GitFileGeneratorItem struct {
-	Path string `json:"path"`
-}
-
 // SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos.
 type SCMProviderGenerator struct {
 	// Which provider to use and config for it.

docs/Generators-Git.md

@@ -58,6 +58,28 @@ The generator parameters are:
 - `{{path.basename}}`: For any directory path within the Git repository that matches the `path` wildcard, the right-most path name is extracted (e.g. `/directory/directory2` would produce `directory2`).
 - `{{path.basenameNormalized}}`: This field is the same as `path.basename` with unsupported characters replaced with `-` (e.g. a `path` of `/directory/directory_2`, and `path.basename` of `directory_2` would produce `directory-2` here).
 
+'**' can also be used to include any level of directories in the match
+```yaml
+- path: '**/apps/*'
+```
+
+Would include all the app directories in this tree:
+
+```
+.
+└── path1
+    └── subDir1
+        └──apps
+            ├── app1
+            └── app2
+└── path2
+    └── subDir2
+        └──apps
+            ├── app3
+            └── app4
+```
+
+
 Whenever a new Helm chart/Kustomize YAML/Application/plain subfolder is added to the Git repository, the ApplicationSet controller will detect this change and automatically deploy the resulting manifests within new `Application` resources.
 
 As with other generators, clusters *must* already be defined within Argo CD, in order to generate Applications for them.
@@ -230,6 +252,43 @@ In addition to the flattened key/value pairs from the configuration file, the fo
 - `{{path.basename}}`: Basename of the path to the folder containing the configuration file (e.g. `clusterA`, with the above example.)
 - `{{path.basenameNormalized}}`: This field is the same as `path.basename` with unsupported characters replaced with `-` (e.g. a `path` of `/directory/directory_2`, and `path.basename` of `directory_2` would produce `directory-2` here).
 
+### Exclude files
+
+The Git file generator also supports an `exclude` option in order to exclude files in the repository from being scanned by the ApplicationSet controller:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: cluster-addons
+spec:
+  generators:
+  - git:
+      repoURL: https://github.com/argoproj/applicationset.git
+      revision: HEAD
+      directories:
+      - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+      - path: "examples/git-generator-files-discovery/cluster-config/*/test/config.json"
+        exclude: true
+  template:
+    metadata:
+      name: '{{path.basename}}'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/applicationset.git
+        targetRevision: HEAD
+        path: '{{path}}'
+      destination:
+        server: https://kubernetes.default.svc
+        namespace: '{{path.basename}}'
+```
+(*The full example can be found [here](https://github.com/argoproj/applicationset/tree/master/examples/git-generator-files-discovery/excludes).*)
+
+This example excludes the config.json file in the `test` directory from the list of files scanned for this `ApplictionSet` resource.
+
+!!! note "Exclude rules are the same as for the Directory generator"
+
 ## Webhook Configuration
 
 When using a Git generator, ApplicationSet polls Git repositories every three minutes to detect changes. To eliminate

examples/git-generator-files-discovery/cluster-config/engineering/test/config.json

@@ -0,0 +1,9 @@
+{
+  "aws_account": "123456",
+  "asset_id": "11223344",
+  "cluster": {
+    "owner": "cluster-admin@company.com",
+    "name": "engineering-test",
+    "address": "http://1.2.3.4"
+  }
+}

examples/git-generator-files-discovery/excludes/git-generator-files.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: guestbook
+spec:
+  generators:
+    - git:
+        repoURL: https://github.com/argoproj/applicationset.git
+        revision: HEAD
+        files:
+        - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+        - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+          exclude: true
+  template:
+    metadata:
+      name: '{{cluster.name}}-guestbook'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/applicationset.git
+        targetRevision: HEAD
+        path: "examples/git-generator-files-discovery/apps/guestbook"
+      destination:
+        server: https://kubernetes.default.svc
+        #server: '{{cluster.address}}'
+        namespace: guestbook

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/argoproj/gitops-engine v0.5.1
 	github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0
 	github.com/go-logr/logr v0.4.0
+	github.com/gobwas/glob v0.2.3
 	github.com/google/go-github/v35 v35.0.0
 	github.com/imdario/mergo v0.3.12
 	github.com/jeremywohl/flatten v1.0.1

manifests/crds/argoproj.io_applicationsets.yaml

@@ -651,6 +651,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -1831,6 +1833,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -3917,6 +3921,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

manifests/install-with-argo-cd.yaml

@@ -2468,6 +2468,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -3648,6 +3650,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -5734,6 +5738,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

manifests/install.yaml

@@ -650,6 +650,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -1830,6 +1832,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -3916,6 +3920,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

pkg/generators/git.go

@@ -11,6 +11,7 @@ import (
 
 	argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1"
 	"github.com/argoproj/applicationset/pkg/services"
+	"github.com/gobwas/glob"
 	"github.com/jeremywohl/flatten"
 	log "github.com/sirupsen/logrus"
 	"sigs.k8s.io/yaml"
@@ -85,7 +86,7 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
 		"revision": appSetGenerator.Git.Revision,
 	}).Info("applications result from the repo service")
 
-	requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths)
+	requestedApps := g.filterPaths(appSetGenerator.Git.Directories, allPaths)
 
 	res := g.generateParamsFromApps(requestedApps, appSetGenerator)
 
@@ -114,9 +115,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	}
 	sort.Strings(allPaths)
 
+	filteredPaths := g.filterPaths(appSetGenerator.Git.Files, allPaths)
+
 	// Generate params from each path, and return
 	res := []map[string]string{}
-	for _, path := range allPaths {
+	for _, path := range filteredPaths {
 
 		// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
 		paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path])
@@ -174,29 +177,36 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 
 }
 
-func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
+func (g *GitGenerator) filterPaths(items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
 	res := []string{}
-	for _, appPath := range allPaths {
-		appInclude := false
-		appExclude := false
-		// Iterating over each appPath and check whether directories object has requestedPath that matches the appPath
-		for _, requestedPath := range Directories {
-			match, err := path.Match(requestedPath.Path, appPath)
-			if err != nil {
-				log.WithError(err).WithField("requestedPath", requestedPath).
-					WithField("appPath", appPath).Error("error while matching appPath to requestedPath")
-				continue
+	for _, itemPath := range allPaths {
+		include := false
+		exclude := false
+		for _, requestedPath := range items {
+			var match bool
+			var err error
+			if strings.Contains(requestedPath.Path, "**") {
+				pathGlob := glob.MustCompile(requestedPath.Path)
+				match = pathGlob.Match(itemPath)
+			} else {
+				match, err = path.Match(requestedPath.Path, itemPath)
+				if err != nil {
+					log.WithError(err).WithField("requestedPath", requestedPath).
+						WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
+					continue
+				}
 			}
+
 			if match && !requestedPath.Exclude {
-				appInclude = true
+				include = true
 			}
 			if match && requestedPath.Exclude {
-				appExclude = true
+				exclude = true
 			}
 		}
 		// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
-		if appInclude && !appExclude {
-			res = append(res, appPath)
+		if include && !exclude {
+			res = append(res, itemPath)
 		}
 	}
 	return res