Skip to content

Commit

Permalink
Support parameter substitution for resolver params
Browse files Browse the repository at this point in the history 
Part of #4710

Until we do `v1beta1` of `ResolutionRequest`, it will still have parameters of `map[string]string`, so we're only doing limited substitution.

Signed-off-by: Andrew Bayer <andrew.bayer@gmail.com>
  • Loading branch information
@abayer
abayer committed Aug 29, 2022
1 parent 31644db commit 0fb9b6b
Show file tree
Hide file tree
Showing 12 changed files with 645 additions and 45 deletions.
12 changes: 12 additions & 0 deletions pkg/apis/pipeline/v1beta1/pipelineref_validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func (ref *PipelineRef) Validate(ctx context.Context) (errs *apis.FieldError) {
errs = errs.Also(apis.ErrMissingField("resolver"))
}
errs = errs.Also(ValidateParameters(ctx, ref.Params))
errs = errs.Also(validateResolutionParamTypes(ref.Params).ViaField("params"))
}
} else {
if ref.Name == "" {
Expand All @@ -79,3 +80,14 @@ func validateBundleFeatureFlag(ctx context.Context, featureName string, wantValu
}
return nil
}

func validateResolutionParamTypes(params []Param) (errs *apis.FieldError) {
for i, p := range params {
if p.Value.Type == ParamTypeArray || p.Value.Type == ParamTypeObject {
errs = errs.Also(apis.ErrGeneric(fmt.Sprintf("remote resolution parameter type must be %s, not %s",
string(ParamTypeString), string(p.Value.Type))).ViaIndex(i))
}
}

return errs
}
20 changes: 19 additions & 1 deletion pkg/apis/pipeline/v1beta1/pipelineref_validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,24 @@ func TestPipelineRef_Invalid(t *testing.T) {
wantErr: apis.ErrMultipleOneOf("bundle", "params").Also(apis.ErrMissingField("resolver")),
withContext: config.EnableAlphaAPIFields,
}, {
name: "pipelineref param object requires alpha",
name: "pipelineref param array not allowed",
ref: &v1beta1.PipelineRef{
ResolverRef: v1beta1.ResolverRef{
Resolver: "some-resolver",
Params: []v1beta1.Param{{
Name: "foo",
Value: v1beta1.ParamValue{
Type: v1beta1.ParamTypeArray,
ArrayVal: []string{"bar", "baz"},
},
}},
},
},
wantErr: apis.ErrGeneric("remote resolution parameter type must be string, not array").
Also(apis.ErrGeneric("resolver requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")).
Also(apis.ErrGeneric("params requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")),
}, {
name: "pipelineref param object not allowed",
ref: &v1beta1.PipelineRef{
ResolverRef: v1beta1.ResolverRef{
Resolver: "some-resolver",
Expand All @@ -154,6 +171,7 @@ func TestPipelineRef_Invalid(t *testing.T) {
},
},
wantErr: apis.ErrGeneric("object type parameter requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"").
Also(apis.ErrGeneric("remote resolution parameter type must be string, not object")).
Also(apis.ErrGeneric("resolver requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")).
Also(apis.ErrGeneric("params requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")),
}}
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/pipeline/v1beta1/taskref_validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func (ref *TaskRef) Validate(ctx context.Context) (errs *apis.FieldError) {
errs = errs.Also(apis.ErrMissingField("resolver"))
}
errs = errs.Also(ValidateParameters(ctx, ref.Params))
errs = errs.Also(validateResolutionParamTypes(ref.Params))
}
} else {
if ref.Name == "" {
Expand Down
18 changes: 18 additions & 0 deletions pkg/apis/pipeline/v1beta1/taskref_validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,23 @@ func TestTaskRef_Invalid(t *testing.T) {
},
wantErr: apis.ErrMultipleOneOf("bundle", "params").Also(apis.ErrMissingField("resolver")),
wc: config.EnableAlphaAPIFields,
}, {
name: "taskref param array not allowed",
taskRef: &v1beta1.TaskRef{
ResolverRef: v1beta1.ResolverRef{
Resolver: "some-resolver",
Params: []v1beta1.Param{{
Name: "foo",
Value: v1beta1.ParamValue{
Type: v1beta1.ParamTypeArray,
ArrayVal: []string{"bar", "baz"},
},
}},
},
},
wantErr: apis.ErrGeneric("remote resolution parameter type must be string, not array").
Also(apis.ErrGeneric("resolver requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")).
Also(apis.ErrGeneric("params requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")),
}, {
name: "taskref param object requires alpha",
taskRef: &v1beta1.TaskRef{
Expand All @@ -201,6 +218,7 @@ func TestTaskRef_Invalid(t *testing.T) {
},
},
wantErr: apis.ErrGeneric("object type parameter requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"").
Also(apis.ErrGeneric("remote resolution parameter type must be string, not object")).
Also(apis.ErrGeneric("resolver requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")).
Also(apis.ErrGeneric("params requires \"enable-api-fields\" feature gate to be \"alpha\" but it is \"stable\"")),
}}
Expand Down
79 changes: 53 additions & 26 deletions pkg/reconciler/pipelinerun/resources/apply.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,16 @@ const (
// objectElementResultsParseNumber is the value of how many parts we split from
// object attribute result reference. e.g. tasks.<taskName>.results.<objectResultName>.<individualAttribute>
objectElementResultsParseNumber = 5
// objectIndividualVariablePattern is the reference pattern for object individual keys params.<object_param_name>.<key_name>
objectIndividualVariablePattern = "params.%s.%s"
)

var (
paramPatterns = []string{
"params.%s",
"params[%q]",
"params['%s']",
}
)

// ApplyParameters applies the params from a PipelineRun.Params to a PipelineSpec.
Expand All @@ -49,21 +59,12 @@ func ApplyParameters(ctx context.Context, p *v1beta1.PipelineSpec, pr *v1beta1.P
objectReplacements := map[string]map[string]string{}
cfg := config.FromContextOrDefaults(ctx)

patterns := []string{
"params.%s",
"params[%q]",
"params['%s']",
}

// reference pattern for object individual keys params.<object_param_name>.<key_name>
objectIndividualVariablePattern := "params.%s.%s"

// Set all the default stringReplacements
for _, p := range p.Params {
if p.Default != nil {
switch p.Default.Type {
case v1beta1.ParamTypeArray:
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
// array indexing for param is alpha feature
if cfg.FeatureFlags.EnableAPIFields == config.AlphaAPIFields {
for i := 0; i < len(p.Default.ArrayVal); i++ {
Expand All @@ -73,24 +74,47 @@ func ApplyParameters(ctx context.Context, p *v1beta1.PipelineSpec, pr *v1beta1.P
arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ArrayVal
}
case v1beta1.ParamTypeObject:
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.ObjectVal
}
for k, v := range p.Default.ObjectVal {
stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v
}
default:
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Default.StringVal
}
}
}
}
// Set and overwrite params with the ones from the PipelineRun
prStrings, prArrays, prObjects := paramsFromPipelineRun(ctx, pr)

for k, v := range prStrings {
stringReplacements[k] = v
}
for k, v := range prArrays {
arrayReplacements[k] = v
}
for k, v := range prObjects {
objectReplacements[k] = v
}

return ApplyReplacements(ctx, p, stringReplacements, arrayReplacements, objectReplacements)
}

func paramsFromPipelineRun(ctx context.Context, pr *v1beta1.PipelineRun) (map[string]string, map[string][]string, map[string]map[string]string) {
// stringReplacements is used for standard single-string stringReplacements,
// while arrayReplacements/objectReplacements contains arrays/objects that need to be further processed.
stringReplacements := map[string]string{}
arrayReplacements := map[string][]string{}
objectReplacements := map[string]map[string]string{}
cfg := config.FromContextOrDefaults(ctx)

for _, p := range pr.Spec.Params {
switch p.Value.Type {
case v1beta1.ParamTypeArray:
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
// array indexing for param is alpha feature
if cfg.FeatureFlags.EnableAPIFields == config.AlphaAPIFields {
for i := 0; i < len(p.Value.ArrayVal); i++ {
Expand All @@ -100,20 +124,20 @@ func ApplyParameters(ctx context.Context, p *v1beta1.PipelineSpec, pr *v1beta1.P
arrayReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ArrayVal
}
case v1beta1.ParamTypeObject:
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
objectReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.ObjectVal
}
for k, v := range p.Value.ObjectVal {
stringReplacements[fmt.Sprintf(objectIndividualVariablePattern, p.Name, k)] = v
}
default:
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
stringReplacements[fmt.Sprintf(pattern, p.Name)] = p.Value.StringVal
}
}
}

return ApplyReplacements(ctx, p, stringReplacements, arrayReplacements, objectReplacements)
return stringReplacements, arrayReplacements, objectReplacements
}

// ApplyContexts applies the substitution from $(context.(pipelineRun|pipeline).*) with the specified values.
Expand Down Expand Up @@ -151,6 +175,9 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul
pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements, objectReplacements)
pipelineTask.Matrix = replaceParamValues(pipelineTask.Matrix, stringReplacements, nil, nil)
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
if pipelineTask.TaskRef != nil && pipelineTask.TaskRef.Params != nil {
pipelineTask.TaskRef.Params = replaceParamValues(pipelineTask.TaskRef.Params, stringReplacements, arrayReplacements, objectReplacements)
}
resolvedPipelineRunTask.PipelineTask = pipelineTask
}
}
Expand All @@ -163,6 +190,9 @@ func ApplyPipelineTaskStateContext(state PipelineRunState, replacements map[stri
pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
pipelineTask.Params = replaceParamValues(pipelineTask.Params, replacements, nil, nil)
pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(replacements, nil)
if pipelineTask.TaskRef != nil && pipelineTask.TaskRef.Params != nil {
pipelineTask.TaskRef.Params = replaceParamValues(pipelineTask.TaskRef.Params, replacements, nil, nil)
}
resolvedPipelineRunTask.PipelineTask = pipelineTask
}
}
Expand Down Expand Up @@ -195,33 +225,30 @@ func ApplyReplacements(ctx context.Context, p *v1beta1.PipelineSpec, replacement
p.Tasks[i].Workspaces[j].SubPath = substitution.ApplyReplacements(p.Tasks[i].Workspaces[j].SubPath, replacements)
}
p.Tasks[i].WhenExpressions = p.Tasks[i].WhenExpressions.ReplaceWhenExpressionsVariables(replacements, arrayReplacements)
if p.Tasks[i].TaskRef != nil && p.Tasks[i].TaskRef.Params != nil {
p.Tasks[i].TaskRef.Params = replaceParamValues(p.Tasks[i].TaskRef.Params, replacements, arrayReplacements, objectReplacements)
}
p.Tasks[i], replacements, arrayReplacements, objectReplacements = propagateParams(ctx, p.Tasks[i], replacements, arrayReplacements, objectReplacements)
}

for i := range p.Finally {
p.Finally[i].Params = replaceParamValues(p.Finally[i].Params, replacements, arrayReplacements, objectReplacements)
p.Finally[i].Matrix = replaceParamValues(p.Finally[i].Matrix, replacements, arrayReplacements, objectReplacements)
p.Finally[i].WhenExpressions = p.Finally[i].WhenExpressions.ReplaceWhenExpressionsVariables(replacements, arrayReplacements)
if p.Finally[i].TaskRef != nil && p.Finally[i].TaskRef.Params != nil {
p.Finally[i].TaskRef.Params = replaceParamValues(p.Finally[i].TaskRef.Params, replacements, arrayReplacements, objectReplacements)
}
}

return p
}

func propagateParams(ctx context.Context, t v1beta1.PipelineTask, replacements map[string]string, arrayReplacements map[string][]string, objectReplacements map[string]map[string]string) (v1beta1.PipelineTask, map[string]string, map[string][]string, map[string]map[string]string) {
if t.TaskSpec != nil && config.FromContextOrDefaults(ctx).FeatureFlags.EnableAPIFields == "alpha" {
patterns := []string{
"params.%s",
"params[%q]",
"params['%s']",
}

// reference pattern for object individual keys params.<object_param_name>.<key_name>
objectIndividualVariablePattern := "params.%s.%s"

// check if there are task parameters defined that match the params at pipeline level
if len(t.Params) > 0 {
for _, par := range t.Params {
for _, pattern := range patterns {
for _, pattern := range paramPatterns {
checkName := fmt.Sprintf(pattern, par.Name)
// Scoping. Task Params will replace Pipeline Params
if _, ok := replacements[checkName]; ok {
Expand Down
Loading

0 comments on commit 0fb9b6b

Please sign in to comment.