Skip to content
This repository has been archived by the owner on Jul 15, 2024. It is now read-only.

Commit

Permalink
feat: add the ability to exclude files when using the git file genera…
Browse files Browse the repository at this point in the history 
…tor (#468)

Signed-off-by: Adam Johnson <adamjohnson01gmail.com>
  • Loading branch information
Adam Johnson committed Feb 20, 2022
1 parent 8220c8a commit dcfbe54
Show file tree
Hide file tree
Showing 12 changed files with 126 additions and 83 deletions.
18 changes: 7 additions & 11 deletions api/v1alpha1/applicationset_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,23 +272,19 @@ type DuckTypeGenerator struct {
}

type GitGenerator struct {
RepoURL string `json:"repoURL"`
Directories []GitDirectoryGeneratorItem `json:"directories,omitempty"`
Files []GitFileGeneratorItem `json:"files,omitempty"`
Revision string `json:"revision"`
RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"`
Template ApplicationSetTemplate `json:"template,omitempty"`
RepoURL string `json:"repoURL"`
Directories []GitGeneratorItem `json:"directories,omitempty"`
Files []GitGeneratorItem `json:"files,omitempty"`
Revision string `json:"revision"`
RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"`
Template ApplicationSetTemplate `json:"template,omitempty"`
}

type GitDirectoryGeneratorItem struct {
type GitGeneratorItem struct {
Path string `json:"path"`
Exclude bool `json:"exclude,omitempty"`
}

type GitFileGeneratorItem struct {
Path string `json:"path"`
}

// SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos.
type SCMProviderGenerator struct {
// Which provider to use and config for it.
Expand Down
50 changes: 18 additions & 32 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/argoproj/gitops-engine v0.5.1
github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0
github.com/go-logr/logr v0.4.0
github.com/gobwas/glob v0.2.3
github.com/google/go-github/v35 v35.0.0
github.com/imdario/mergo v0.3.12
github.com/jeremywohl/flatten v1.0.1
Expand Down
6 changes: 6 additions & 0 deletions manifests/crds/argoproj.io_applicationsets.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down Expand Up @@ -1831,6 +1833,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down Expand Up @@ -3917,6 +3921,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down
6 changes: 6 additions & 0 deletions manifests/install-with-argo-cd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2468,6 +2468,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down Expand Up @@ -3648,6 +3650,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down Expand Up @@ -5734,6 +5738,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down
6 changes: 6 additions & 0 deletions manifests/install.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -650,6 +650,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down Expand Up @@ -1830,6 +1832,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down Expand Up @@ -3916,6 +3920,8 @@ spec:
files:
items:
properties:
exclude:
type: boolean
path:
type: string
required:
Expand Down
44 changes: 27 additions & 17 deletions pkg/generators/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1"
"github.com/argoproj/applicationset/pkg/services"
"github.com/gobwas/glob"
"github.com/jeremywohl/flatten"
log "github.com/sirupsen/logrus"
"sigs.k8s.io/yaml"
Expand Down Expand Up @@ -85,7 +86,7 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
"revision": appSetGenerator.Git.Revision,
}).Info("applications result from the repo service")

requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths)
requestedApps := g.filterPaths("directories", appSetGenerator.Git.Directories, allPaths)

res := g.generateParamsFromApps(requestedApps, appSetGenerator)

Expand Down Expand Up @@ -114,9 +115,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
}
sort.Strings(allPaths)

filteredPaths := g.filterPaths("files", appSetGenerator.Git.Files, allPaths)

// Generate params from each path, and return
res := []map[string]string{}
for _, path := range allPaths {
for _, path := range filteredPaths {

// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path])
Expand Down Expand Up @@ -174,29 +177,36 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []

}

func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
func (g *GitGenerator) filterPaths(pathType string, items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
res := []string{}
for _, appPath := range allPaths {
appInclude := false
appExclude := false
// Iterating over each appPath and check whether directories object has requestedPath that matches the appPath
for _, requestedPath := range Directories {
match, err := path.Match(requestedPath.Path, appPath)
if err != nil {
log.WithError(err).WithField("requestedPath", requestedPath).
WithField("appPath", appPath).Error("error while matching appPath to requestedPath")
continue
for _, itemPath := range allPaths {
include := false
exclude := false
for _, requestedPath := range items {
var match bool
var err error
if pathType == "files" {
pathGlob := glob.MustCompile(requestedPath.Path)
match = pathGlob.Match(itemPath)
} else {
match, err = path.Match(requestedPath.Path, itemPath)
if err != nil {
log.WithError(err).WithField("requestedPath", requestedPath).
WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
continue
}
}

if match && !requestedPath.Exclude {
appInclude = true
include = true
}
if match && requestedPath.Exclude {
appExclude = true
exclude = true
}
}
// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
if appInclude && !appExclude {
res = append(res, appPath)
if include && !exclude {
res = append(res, itemPath)
}
}
return res
Expand Down
Loading

0 comments on commit dcfbe54

Please sign in to comment.