Skip to content

Commit

Permalink
feat: Allow Matrix generator to have two Git child generators without…
Browse files Browse the repository at this point in the history 
… conflict (argoproj#10522) (argoproj#10523)

* misc: NoError instead of Error in repository test
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* misc: Extend lint timeout
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* feat: Add GitGenerator.PathParamPrefix
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* auto: Results of codegen
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* test: Add tests for PathParamPrefix
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* docs: Add notes to Matrix/Git generator docs about PathParamPrefix
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* misc: Undo unrelated test change
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* auto: Results of codegen
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

* docs: Add detailed example
Signed-off-by: Lobstrosity <matthew.t.bennett@gmail.com>

Co-authored-by: Michael Crenshaw <350466+crenshaw-dev@users.noreply.github.com>
Signed-off-by: emirot <emirot.nolan@gmail.com>
  • Loading branch information
@Lobstrosity @crenshaw-dev @emirot
2 people authored and emirot committed Jan 27, 2023
1 parent 80e3545 commit 529bec7
Show file tree
Hide file tree
Showing 13 changed files with 940 additions and 610 deletions.
54 changes: 34 additions & 20 deletions applicationset/generators/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
"total": len(allPaths),
"repoURL": appSetGenerator.Git.RepoURL,
"revision": appSetGenerator.Git.Revision,
"pathParamPrefix": appSetGenerator.Git.PathParamPrefix,
}).Info("applications result from the repo service")

requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths)
Expand Down Expand Up @@ -121,7 +122,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
for _, path := range allPaths {

// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], useGoTemplate)
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], useGoTemplate, appSetGenerator.Git.PathParamPrefix)
if err != nil {
return nil, fmt.Errorf("unable to process file '%s': %v", path, err)
}
Expand All @@ -133,7 +134,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
return res, nil
}

func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, useGoTemplate bool) ([]map[string]interface{}, error) {
func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, useGoTemplate bool, pathParamPrefix string) ([]map[string]interface{}, error) {
objectsFound := []map[string]interface{}{}

// First, we attempt to parse as an array
Expand Down Expand Up @@ -167,7 +168,11 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(paramPath["path"].(string)))
paramPath["filenameNormalized"] = utils.SanitizeName(path.Base(paramPath["filename"].(string)))
paramPath["segments"] = strings.Split(paramPath["path"].(string), "/")
params["path"] = paramPath
if pathParamPrefix != "" {
params[pathParamPrefix] = map[string]interface{}{"path": paramPath}
} else {
params["path"] = paramPath
}
} else {
flat, err := flatten.Flatten(objectFound, "", flatten.DotStyle)
if err != nil {
Expand All @@ -176,14 +181,18 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
for k, v := range flat {
params[k] = fmt.Sprintf("%v", v)
}
params["path"] = path.Dir(filePath)
params["path.basename"] = path.Base(params["path"].(string))
params["path.filename"] = path.Base(filePath)
params["path.basenameNormalized"] = utils.SanitizeName(path.Base(params["path"].(string)))
params["path.filenameNormalized"] = utils.SanitizeName(path.Base(params["path.filename"].(string)))
for k, v := range strings.Split(params["path"].(string), "/") {
pathParamName := "path"
if pathParamPrefix != "" {
pathParamName = pathParamPrefix+"."+pathParamName
}
params[pathParamName] = path.Dir(filePath)
params[pathParamName+".basename"] = path.Base(params[pathParamName].(string))
params[pathParamName+".filename"] = path.Base(filePath)
params[pathParamName+".basenameNormalized"] = utils.SanitizeName(path.Base(params[pathParamName].(string)))
params[pathParamName+".filenameNormalized"] = utils.SanitizeName(path.Base(params[pathParamName+".filename"].(string)))
for k, v := range strings.Split(params[pathParamName].(string), "/") {
if len(v) > 0 {
params["path["+strconv.Itoa(k)+"]"] = v
params[pathParamName+"["+strconv.Itoa(k)+"]"] = v
}
}
}
Expand All @@ -192,7 +201,6 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
}

return res, nil

}

func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
Expand Down Expand Up @@ -223,9 +231,7 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG
return res
}

func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) []map[string]interface{} {
// TODO: At some point, the applicationSetGenerator param should be used

func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool) []map[string]interface{} {
res := make([]map[string]interface{}, len(requestedApps))
for i, a := range requestedApps {

Expand All @@ -237,14 +243,22 @@ func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argopro
paramPath["basename"] = path.Base(a)
paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(a))
paramPath["segments"] = strings.Split(paramPath["path"].(string), "/")
params["path"] = paramPath
if appSetGenerator.Git.PathParamPrefix != "" {
params[appSetGenerator.Git.PathParamPrefix] = map[string]interface{}{"path": paramPath}
} else {
params["path"] = paramPath
}
} else {
params["path"] = a
params["path.basename"] = path.Base(a)
params["path.basenameNormalized"] = utils.SanitizeName(path.Base(a))
for k, v := range strings.Split(params["path"].(string), "/") {
pathParamName := "path"
if appSetGenerator.Git.PathParamPrefix != "" {
pathParamName = appSetGenerator.Git.PathParamPrefix+"."+pathParamName
}
params[pathParamName] = a
params[pathParamName+".basename"] = path.Base(a)
params[pathParamName+".basenameNormalized"] = utils.SanitizeName(path.Base(a))
for k, v := range strings.Split(params[pathParamName].(string), "/") {
if len(v) > 0 {
params["path["+strconv.Itoa(k)+"]"] = v
params[pathParamName+"["+strconv.Itoa(k)+"]"] = v
}
}
}
Expand Down
165 changes: 145 additions & 20 deletions applicationset/generators/git_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func Test_generateParamsFromGitFile(t *testing.T) {
params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
foo:
bar: baz
`), false)
`), false, "")
if err != nil {
t.Fatal(err)
}
Expand All @@ -69,11 +69,33 @@ foo:
}, params)
}

func Test_generatePrefixedParamsFromGitFile(t *testing.T) {
params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
foo:
bar: baz
`), false, "myRepo")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, []map[string]interface{}{
{
"foo.bar": "baz",
"myRepo.path": "path/dir",
"myRepo.path.basename": "dir",
"myRepo.path.filename": "file_name.yaml",
"myRepo.path.basenameNormalized": "dir",
"myRepo.path.filenameNormalized": "file-name.yaml",
"myRepo.path[0]": "path",
"myRepo.path[1]": "dir",
},
}, params)
}

func Test_generateParamsFromGitFileGoTemplate(t *testing.T) {
params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
foo:
bar: baz
`), true)
`), true, "")
if err != nil {
t.Fatal(err)
}
Expand All @@ -97,15 +119,46 @@ foo:
}, params)
}

func Test_generatePrefixedParamsFromGitFileGoTemplate(t *testing.T) {
params, err := (*GitGenerator)(nil).generateParamsFromGitFile("path/dir/file_name.yaml", []byte(`
foo:
bar: baz
`), true, "myRepo")
if err != nil {
t.Fatal(err)
}
assert.Equal(t, []map[string]interface{}{
{
"foo": map[string]interface{}{
"bar": "baz",
},
"myRepo": map[string]interface{}{
"path": map[string]interface{}{
"path": "path/dir",
"basename": "dir",
"filename": "file_name.yaml",
"basenameNormalized": "dir",
"filenameNormalized": "file-name.yaml",
"segments": []string{
"path",
"dir",
},
},
},
},
}, params)
}

func TestGitGenerateParamsFromDirectories(t *testing.T) {

cases := []struct {
name string
directories []argoprojiov1alpha1.GitDirectoryGeneratorItem
repoApps []string
repoError error
expected []map[string]interface{}
expectedError error
name string
directories []argoprojiov1alpha1.GitDirectoryGeneratorItem
pathParamPrefix string
repoApps []string
repoError error
expected []map[string]interface{}
expectedError error
}{
{
name: "happy flow - created apps",
Expand All @@ -124,6 +177,24 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
},
expectedError: nil,
},
{
name: "It prefixes path parameters with PathParamPrefix",
directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}},
pathParamPrefix: "myRepo",
repoApps: []string{
"app1",
"app2",
"app_3",
"p1/app4",
},
repoError: nil,
expected: []map[string]interface{}{
{"myRepo.path": "app1", "myRepo.path.basename": "app1", "myRepo.path.basenameNormalized": "app1", "myRepo.path[0]": "app1"},
{"myRepo.path": "app2", "myRepo.path.basename": "app2", "myRepo.path.basenameNormalized": "app2", "myRepo.path[0]": "app2"},
{"myRepo.path": "app_3", "myRepo.path.basename": "app_3", "myRepo.path.basenameNormalized": "app-3", "myRepo.path[0]": "app_3"},
},
expectedError: nil,
},
{
name: "It filters application according to the paths",
directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}},
Expand Down Expand Up @@ -212,9 +283,10 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
Spec: argoprojiov1alpha1.ApplicationSetSpec{
Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{
Git: &argoprojiov1alpha1.GitGenerator{
RepoURL: "RepoURL",
Revision: "Revision",
Directories: testCaseCopy.directories,
RepoURL: "RepoURL",
Revision: "Revision",
Directories: testCaseCopy.directories,
PathParamPrefix: testCaseCopy.pathParamPrefix,
},
}},
},
Expand All @@ -237,12 +309,13 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {

cases := []struct {
name string
directories []argoprojiov1alpha1.GitDirectoryGeneratorItem
repoApps []string
repoError error
expected []map[string]interface{}
expectedError error
name string
directories []argoprojiov1alpha1.GitDirectoryGeneratorItem
pathParamPrefix string
repoApps []string
repoError error
expected []map[string]interface{}
expectedError error
}{
{
name: "happy flow - created apps",
Expand Down Expand Up @@ -288,6 +361,57 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
},
expectedError: nil,
},
{
name: "It prefixes path parameters with PathParamPrefix",
directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}},
pathParamPrefix: "myRepo",
repoApps: []string{
"app1",
"app2",
"app_3",
"p1/app4",
},
repoError: nil,
expected: []map[string]interface{}{
{
"myRepo": map[string]interface{}{
"path": map[string]interface{}{
"path": "app1",
"basename": "app1",
"basenameNormalized": "app1",
"segments": []string{
"app1",
},
},
},
},
{
"myRepo": map[string]interface{}{
"path": map[string]interface{}{
"path": "app2",
"basename": "app2",
"basenameNormalized": "app2",
"segments": []string{
"app2",
},
},
},
},
{
"myRepo": map[string]interface{}{
"path": map[string]interface{}{
"path": "app_3",
"basename": "app_3",
"basenameNormalized": "app-3",
"segments": []string{
"app_3",
},
},
},
},
},
expectedError: nil,
},
{
name: "It filters application according to the paths",
directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}},
Expand Down Expand Up @@ -455,9 +579,10 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
GoTemplate: true,
Generators: []argoprojiov1alpha1.ApplicationSetGenerator{{
Git: &argoprojiov1alpha1.GitGenerator{
RepoURL: "RepoURL",
Revision: "Revision",
Directories: testCaseCopy.directories,
RepoURL: "RepoURL",
Revision: "Revision",
Directories: testCaseCopy.directories,
PathParamPrefix: testCaseCopy.pathParamPrefix,
},
}},
},
Expand Down
3 changes: 3 additions & 0 deletions assets/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6543,6 +6543,9 @@
"$ref": "#/definitions/v1alpha1GitFileGeneratorItem"
}
},
"pathParamPrefix": {
"type": "string"
},
"repoURL": {
"type": "string"
},
Expand Down
3 changes: 3 additions & 0 deletions docs/operator-manual/applicationset/Generators-Git.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ The generator parameters are:

**Note**: The right-most path name always becomes `{{path.basename}}`. For example, for `- path: /one/two/three/four`, `{{path.basename}}` is `four`.

**Note**: If the `pathParamPrefix` option is specified, all `path`-related parameter names above will be prefixed with the specified value and a dot separator. E.g., if `pathParamPrefix` is `myRepo`, then the generated parameter name would be `myRepo.path` instead of `path`. Using this option is necessary in a Matrix generator where both child generators are Git generators (to avoid conflicts when merging the child generators’ items).

Whenever a new Helm chart/Kustomize YAML/Application/plain subdirectory is added to the Git repository, the ApplicationSet controller will detect this change and automatically deploy the resulting manifests within new `Application` resources.

As with other generators, clusters *must* already be defined within Argo CD, in order to generate Applications for them.
Expand Down Expand Up @@ -284,6 +286,7 @@ In addition to the flattened key/value pairs from the configuration file, the fo
**Note**: The right-most *directory* name always becomes `{{path.basename}}`. For example, from `- path: /one/two/three/four/config.json`, `{{path.basename}}` will be `four`.
The filename can always be accessed using `{{path.filename}}`.

**Note**: If the `pathParamPrefix` option is specified, all `path`-related parameter names above will be prefixed with the specified value and a dot separator. E.g., if `pathParamPrefix` is `myRepo`, then the generated parameter name would be `myRepo.path` instead of `path`. Using this option is necessary in a Matrix generator where both child generators are Git generators (to avoid conflicts when merging the child generators’ items).

## Webhook Configuration

Expand Down
Loading

0 comments on commit 529bec7

Please sign in to comment.