feat: add exclude files when using the git file generator (#468)

api/v1alpha1/applicationset_types.go

@@ -272,23 +272,19 @@ type DuckTypeGenerator struct {
 }
 
 type GitGenerator struct {
-	RepoURL             string                      `json:"repoURL"`
-	Directories         []GitDirectoryGeneratorItem `json:"directories,omitempty"`
-	Files               []GitFileGeneratorItem      `json:"files,omitempty"`
-	Revision            string                      `json:"revision"`
-	RequeueAfterSeconds *int64                      `json:"requeueAfterSeconds,omitempty"`
-	Template            ApplicationSetTemplate      `json:"template,omitempty"`
+	RepoURL             string                 `json:"repoURL"`
+	Directories         []GitGeneratorItem     `json:"directories,omitempty"`
+	Files               []GitGeneratorItem     `json:"files,omitempty"`
+	Revision            string                 `json:"revision"`
+	RequeueAfterSeconds *int64                 `json:"requeueAfterSeconds,omitempty"`
+	Template            ApplicationSetTemplate `json:"template,omitempty"`
 }
 
-type GitDirectoryGeneratorItem struct {
+type GitGeneratorItem struct {
 	Path    string `json:"path"`
 	Exclude bool   `json:"exclude,omitempty"`
 }
 
-type GitFileGeneratorItem struct {
-	Path string `json:"path"`
-}
-
 // SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos.
 type SCMProviderGenerator struct {
 	// Which provider to use and config for it.

docs/Generators-Git.md

@@ -233,6 +233,43 @@ In addition to the flattened key/value pairs from the configuration file, the fo
 - `{{path.basename}}`: Basename of the path to the folder containing the configuration file (e.g. `clusterA`, with the above example.)
 - `{{path.basenameNormalized}}`: This field is the same as `path.basename` with unsupported characters replaced with `-` (e.g. a `path` of `/directory/directory_2`, and `path.basename` of `directory_2` would produce `directory-2` here).
 
+### Exclude files
+
+The Git file generator also supports an `exclude` option in order to exclude files in the repository from being scanned by the ApplicationSet controller:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: cluster-addons
+spec:
+  generators:
+  - git:
+      repoURL: https://github.com/argoproj/applicationset.git
+      revision: HEAD
+      directories:
+      - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+      - path: "examples/git-generator-files-discovery/cluster-config/*/test/config.json"
+        exclude: true
+  template:
+    metadata:
+      name: '{{path.basename}}'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/applicationset.git
+        targetRevision: HEAD
+        path: '{{path}}'
+      destination:
+        server: https://kubernetes.default.svc
+        namespace: '{{path.basename}}'
+```
+(*The full example can be found [here](https://github.com/argoproj/applicationset/tree/master/examples/git-generator-files-discovery/excludes).*)
+
+This example excludes the config.json file in the `test` directory from the list of files scanned for this `ApplictionSet` resource.
+
+File exclude paths are matched using [doublestar.Match](https://github.com/bmatcuk/doublestar/blob/master/match.go#L8)
+
 ## Webhook Configuration
 
 When using a Git generator, ApplicationSet polls Git repositories every three minutes to detect changes. To eliminate

docs/assets/broken-link-ignore-list.txt

@@ -8,3 +8,4 @@ https://github.com/argoproj/applicationset/commit
 https://github.com/infra-team/cluster-deployments.git
 https://gitlab.example.com/
 https://kubernetes.default.svc
+https://github.com/argoproj/applicationset/tree/master/examples/git-generator-files-discovery/excludes

examples/git-generator-files-discovery/cluster-config/engineering/test/config.json

@@ -0,0 +1,9 @@
+{
+  "aws_account": "123456",
+  "asset_id": "11223344",
+  "cluster": {
+    "owner": "cluster-admin@company.com",
+    "name": "engineering-test",
+    "address": "http://1.2.3.4"
+  }
+}

examples/git-generator-files-discovery/excludes/git-generator-files.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: guestbook
+spec:
+  generators:
+    - git:
+        repoURL: https://github.com/argoproj/applicationset.git
+        revision: HEAD
+        files:
+        - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+        - path: "examples/git-generator-files-discovery/cluster-config/*/test/config.json"
+          exclude: true
+  template:
+    metadata:
+      name: '{{cluster.name}}-guestbook'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/applicationset.git
+        targetRevision: HEAD
+        path: "examples/git-generator-files-discovery/apps/guestbook"
+      destination:
+        server: https://kubernetes.default.svc
+        #server: '{{cluster.address}}'
+        namespace: guestbook

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/argoproj/argo-cd/v2 v2.3.0-rc5.0.20220206192056-4b04a3918029
 	github.com/argoproj/gitops-engine v0.5.1-0.20220126184517-b0c5e00ccfa5
 	github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0
+	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/go-logr/logr v1.2.2
 	github.com/google/go-github/v35 v35.0.0
 	github.com/imdario/mergo v0.3.12
@@ -97,6 +98,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect

go.sum

@@ -148,6 +148,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
+github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM=
 github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio=

manifests/crds/argoproj.io_applicationsets.yaml

@@ -667,6 +667,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -1879,6 +1881,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -4021,6 +4025,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

manifests/install.yaml

@@ -666,6 +666,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -1878,6 +1880,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -4020,6 +4024,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

pkg/generators/git.go

@@ -11,6 +11,7 @@ import (
 
 	argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1"
 	"github.com/argoproj/applicationset/pkg/services"
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/jeremywohl/flatten"
 	log "github.com/sirupsen/logrus"
 	"sigs.k8s.io/yaml"
@@ -114,9 +115,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	}
 	sort.Strings(allPaths)
 
+	filteredPaths := g.filterPaths(appSetGenerator.Git.Files, allPaths)
+
 	// Generate params from each path, and return
 	res := []map[string]string{}
-	for _, path := range allPaths {
+	for _, path := range filteredPaths {
 
 		// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
 		paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path])
@@ -174,7 +177,7 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 
 }
 
-func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
+func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
 	res := []string{}
 	for _, appPath := range allPaths {
 		appInclude := false
@@ -202,6 +205,36 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG
 	return res
 }
 
+func (g *GitGenerator) filterPaths(items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
+	res := []string{}
+	for _, itemPath := range allPaths {
+		include := false
+		exclude := false
+		for _, requestedPath := range items {
+			var match bool
+			var err error
+			match, err = doublestar.Match(requestedPath.Path, itemPath)
+			if err != nil {
+				log.WithError(err).WithField("requestedPath", requestedPath).
+					WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
+				continue
+			}
+
+			if match && !requestedPath.Exclude {
+				include = true
+			}
+			if match && requestedPath.Exclude {
+				exclude = true
+			}
+		}
+		// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
+		if include && !exclude {
+			res = append(res, itemPath)
+		}
+	}
+	return res
+}
+
 func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator) []map[string]string {
 	// TODO: At some point, the appicationSetGenerator param should be used