diff --git a/Makefile b/Makefile index ea994d65..d65e0221 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GO111MODULE=on AWS_SERVICE=$(shell echo $(SERVICE) | tr '[:upper:]' '[:lower:]') # Build ldflags -VERSION ?= "v0.9.2" +VERSION ?= "v0.10.0" GITCOMMIT=$(shell git rev-parse HEAD) BUILDDATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') IMPORT_PATH=github.com/aws-controllers-k8s/code-generator diff --git a/go.mod b/go.mod index e058492f..2f425668 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws-controllers-k8s/code-generator go 1.14 require ( - github.com/aws-controllers-k8s/runtime v0.9.2 + github.com/aws-controllers-k8s/runtime v0.10.0 github.com/aws/aws-sdk-go v1.37.10 github.com/dlclark/regexp2 v1.4.0 // pin to v0.1.1 due to release problem with v0.1.2 diff --git a/go.sum b/go.sum index e8ff4c07..4a980a25 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws-controllers-k8s/runtime v0.9.2 h1:53ahm38Cn6DTfQdHNrTgbXmUbCjuKntvhkuWGHkAQ18= -github.com/aws-controllers-k8s/runtime v0.9.2/go.mod h1:kG2WM4JAmLgf67cgZV9IZUkY2DsrUzsaNbmhFMfb05c= +github.com/aws-controllers-k8s/runtime v0.10.0 h1:MPZ4mPeap2mP/EKU6Pk7a7phiBSYaeZ9QJX38OPecXo= +github.com/aws-controllers-k8s/runtime v0.10.0/go.mod h1:kG2WM4JAmLgf67cgZV9IZUkY2DsrUzsaNbmhFMfb05c= github.com/aws/aws-sdk-go v1.37.10 h1:LRwl+97B4D69Z7tz+eRUxJ1C7baBaIYhgrn5eLtua+Q= github.com/aws/aws-sdk-go v1.37.10/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/pkg/generate/ack/controller.go b/pkg/generate/ack/controller.go index 3c600374..d3b5a9f7 100644 --- a/pkg/generate/ack/controller.go +++ b/pkg/generate/ack/controller.go @@ -118,6 +118,15 @@ var ( "GoCodeSetResourceIdentifiers": func(r *ackmodel.CRD, sourceVarName string, targetVarName string, indentLevel int) string { return code.SetResourceIdentifiers(r.Config(), r, sourceVarName, targetVarName, indentLevel) }, + "GoCodeFindLateInitializedFieldNames": func(r *ackmodel.CRD, resVarName string, indentLevel int) string { + return code.FindLateInitializedFieldNames(r.Config(), r, resVarName, indentLevel) + }, + "GoCodeLateInitializeFromReadOne": func(r *ackmodel.CRD, sourceResVarName string, targetResVarName string, indentLevel int) string { + return code.LateInitializeFromReadOne(r.Config(), r, sourceResVarName, targetResVarName, indentLevel) + }, + "GoCodeIncompleteLateInitialization": func(r *ackmodel.CRD, resVarName string, indentLevel int) string { + return code.IncompleteLateInitialization(r.Config(), r, resVarName, indentLevel) + }, } ) diff --git a/pkg/generate/ack/hook.go b/pkg/generate/ack/hook.go index 1810ce2d..99c4a6b0 100644 --- a/pkg/generate/ack/hook.go +++ b/pkg/generate/ack/hook.go @@ -59,6 +59,8 @@ code paths: * sdk_file_end * delta_pre_compare * delta_post_compare +* late_initialize_pre_read_one +* late_initialize_post_read_one The "pre_build_request" hooks are called BEFORE the call to construct the Input shape that is used in the API operation and therefore BEFORE @@ -102,6 +104,12 @@ compares two resources. The "delta_post_compare" hooks are called AFTER the generated code that compares two resources. +The "late_initialize_pre_read_one" hooks are called BEFORE making the +readOne call inside AWSResourceManager.LateInitialize() method + +The "late_initialize_post_read_one" hooks are called AFTER making the +readOne call inside AWSResourceManager.LateInitialize() method + */ // ResourceHookCode returns a string with custom callback code for a resource diff --git a/pkg/generate/ack/runtime_test.go b/pkg/generate/ack/runtime_test.go index 196608c9..aee111ae 100644 --- a/pkg/generate/ack/runtime_test.go +++ b/pkg/generate/ack/runtime_test.go @@ -117,6 +117,10 @@ func (frm *fakeRM) Delete(context.Context, acktypes.AWSResource) (acktypes.AWSRe func (frm *fakeRM) ARNFromName(string) string { return "" } +func (frm *fakeRM) LateInitialize(context.Context, acktypes.AWSResource) (acktypes.AWSResource, error) { + return nil, nil +} + // This test is mostly just a hack to introduce a Go module dependency between // the ACK runtime library and the code generator. The code generator doesn't // actually depend on Go code in the ACK runtime, but it *produces* templated diff --git a/pkg/generate/code/late_initialize.go b/pkg/generate/code/late_initialize.go new file mode 100644 index 00000000..7b3a3ed1 --- /dev/null +++ b/pkg/generate/code/late_initialize.go @@ -0,0 +1,342 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code + +import ( + "fmt" + "sort" + "strings" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config" + "github.com/aws-controllers-k8s/code-generator/pkg/model" +) + +// FindLateInitializedFieldNames outputs the code to create a sorted slice of fieldNames to +// late initialize. This slice helps with short circuiting the AWSResourceManager.LateInitialize() +// method if there are no fields to late initialize. +// +// Sample Output: +// var lateInitializeFieldNames = []string{"Name"} +func FindLateInitializedFieldNames( + cfg *ackgenconfig.Config, + r *model.CRD, + resVarName string, + // Number of levels of indentation to use + indentLevel int, +) string { + out := "" + indent := strings.Repeat("\t", indentLevel) + sortedFieldNames, _ := getSortedLateInitFieldsAndConfig(cfg, r) + if len(sortedFieldNames) > 0 { + out += fmt.Sprintf("%svar %s = []string{", indent, resVarName) + for _, fName := range sortedFieldNames { + out += fmt.Sprintf("%q,", fName) + } + out += "}\n" + } else { + out += fmt.Sprintf("%svar %s = []string{}\n", indent, resVarName) + } + return out +} + +// getSortedLateInitFieldsAndConfig returns the field names in alphabetically sorted order which have LateInitialization +// configuration inside generator config and also a map from fieldName to LateInitializationConfig. +func getSortedLateInitFieldsAndConfig( + cfg *ackgenconfig.Config, + r *model.CRD, +) ([]string, map[string]*ackgenconfig.LateInitializeConfig) { + fieldNameToConfig := cfg.ResourceFields(r.Names.Original) + fieldNameToLateInitConfig := make(map[string]*ackgenconfig.LateInitializeConfig) + sortedLateInitFieldNames := make([]string, 0) + if len(fieldNameToConfig) > 0 { + for fName, fConfig := range fieldNameToConfig { + if fConfig != nil && fConfig.LateInitialize != nil { + fieldNameToLateInitConfig[fName] = fConfig.LateInitialize + sortedLateInitFieldNames = append(sortedLateInitFieldNames, fName) + } + } + sort.Strings(sortedLateInitFieldNames) + } + return sortedLateInitFieldNames, fieldNameToLateInitConfig +} + +// LateInitializeFromReadOne returns the gocode to set LateInitialization fields from the ReadOne output +// Field path separated by '.' indicates members in a struct +// Field path separated by '..' indicates member/key in a map +// Note: Unlike Map, updating individual element of a list is not supported. LateInitializing complete list is supported. +// +// Sample generator config: +// fields: +// Name: +// late_initialize: {} +// ImageScanningConfiguration.ScanOnPush: +// late_initialize: +// min_backoff_seconds: 5 +// max_backoff_seconds: 15 +// map..subfield.x: +// late_initialize: +// min_backoff_seconds: 5 +// another.map..lastfield: +// late_initialize: +// min_backoff_seconds: 5 +// some.list: +// late_initialize: +// min_backoff_seconds: 10 +// structA.mapB..structC.valueD: +// late_initialize: +// min_backoff_seconds: 20 +// +// Sample output: +// observedKo := rm.concreteResource(observed).ko +// latestKo := rm.concreteResource(latest).ko +// if observedKo.Spec.ImageScanningConfiguration != nil && latestKo.Spec.ImageScanningConfiguration != nil { +// if observedKo.Spec.ImageScanningConfiguration.ScanOnPush != nil && latestKo.Spec.ImageScanningConfiguration.ScanOnPush == nil { +// latestKo.Spec.ImageScanningConfiguration.ScanOnPush = observedKo.Spec.ImageScanningConfiguration.ScanOnPush +// } +// } +// if observedKo.Spec.Name != nil && latestKo.Spec.Name == nil { +// latestKo.Spec.Name = observedKo.Spec.Name +// } +// if observedKo.Spec.another != nil && latestKo.Spec.another != nil { +// if observedKo.Spec.another.map != nil && latestKo.Spec.another.map != nil { +// if observedKo.Spec.another.map["lastfield"] != nil && latestKo.Spec.another.map["lastfield"] == nil { +// latestKo.Spec.another.map["lastfield"] = observedKo.Spec.another.map["lastfield"] +// } +// } +// } +// if observedKo.Spec.map != nil && latestKo.Spec.map != nil { +// if observedKo.Spec.map["subfield"] != nil && latestKo.Spec.map["subfield"] != nil { +// if observedKo.Spec.map["subfield"].x != nil && latestKo.Spec.map["subfield"].x == nil { +// latestKo.Spec.map["subfield"].x = observedKo.Spec.map["subfield"].x +// } +// } +// } +// if observedKo.Spec.some != nil && latestKo.Spec.some != nil { +// if observedKo.Spec.some.list != nil && latestKo.Spec.some.list == nil { +// latestKo.Spec.some.list = observedKo.Spec.some.list +// } +// } +// if observedKo.Spec.structA != nil && latestKo.Spec.structA != nil { +// if observedKo.Spec.structA.mapB != nil && latestKo.Spec.structA.mapB != nil { +// if observedKo.Spec.structA.mapB["structC"] != nil && latestKo.Spec.structA.mapB["structC"] != nil { +// if observedKo.Spec.structA.mapB["structC"].valueD != nil && latestKo.Spec.structA.mapB["structC"].valueD == nil { +// latestKo.Spec.structA.mapB["structC"].valueD = observedKo.Spec.structA.mapB["structC"].valueD +// } +// } +// } +// } +// return latest +func LateInitializeFromReadOne( + cfg *ackgenconfig.Config, + r *model.CRD, + sourceResVarName string, + targetResVarName string, + // Number of levels of indentation to use + indentLevel int, +) string { + out := "" + indent := strings.Repeat("\t", indentLevel) + lateInitializedFieldNames, _ := getSortedLateInitFieldsAndConfig(cfg, r) + if len(lateInitializedFieldNames) == 0 { + return fmt.Sprintf("%sreturn %s", indent, targetResVarName) + } + out += fmt.Sprintf("%sobservedKo := rm.concreteResource(%s).ko\n", indent, sourceResVarName) + out += fmt.Sprintf("%slatestKo := rm.concreteResource(%s).ko\n", indent, targetResVarName) + // TODO(vijat@): Add validation for correct field path in lateInitializedFieldNames + for _, fName := range lateInitializedFieldNames { + // split the field name by period + // each substring represents a field. + fNameParts := strings.Split(fName, ".") + // fNameIndentLevel tracks the indentation level for every new line added + // This variable is incremented when building nested if blocks and decremented when closing those if blocks. + fNameIndentLevel := indentLevel + // fParentPath keeps track of parent path for any fNamePart + fParentPath := "" + mapShapedParent := false + // for every part except last, perform the nil check + // entries in both source and target koVarName should not be nil + for i, fNamePart := range fNameParts { + if fNamePart == "" { + mapShapedParent = true + continue + } + indent := strings.Repeat("\t", fNameIndentLevel) + fNamePartAccesor := fmt.Sprintf("Spec%s.%s", fParentPath, fNamePart) + if mapShapedParent { + fNamePartAccesor = fmt.Sprintf("Spec%s[%q]", fParentPath, fNamePart) + } + // Handling for all parts except last one + if i != len(fNameParts)-1 { + out += fmt.Sprintf("%sif observedKo.%s != nil && latestKo.%s != nil {\n", indent, fNamePartAccesor, fNamePartAccesor) + // update fParentPath and fNameIndentLevel for next iteration + if mapShapedParent { + fParentPath = fmt.Sprintf("%s[%q]", fParentPath, fNamePart) + mapShapedParent = false + } else { + fParentPath = fmt.Sprintf("%s.%s", fParentPath, fNamePart) + } + fNameIndentLevel = fNameIndentLevel + 1 + } else { + // handle last part here + // for last part, set the lateInitialized field if user did not specify field value and readOne has server side defaulted value. + // i.e. field is not nil in sourceKoVarName but is nil in targetkoVarName + out += fmt.Sprintf("%sif observedKo.%s != nil && latestKo.%s == nil {\n", indent, fNamePartAccesor, fNamePartAccesor) + fNameIndentLevel = fNameIndentLevel + 1 + indent = strings.Repeat("\t", fNameIndentLevel) + out += fmt.Sprintf("%slatestKo.%s = observedKo.%s\n", indent, fNamePartAccesor, fNamePartAccesor) + } + } + // Close all if blocks with proper indentation + fNameIndentLevel = fNameIndentLevel - 1 + for fNameIndentLevel >= indentLevel { + out += fmt.Sprintf("%s}\n", strings.Repeat("\t", fNameIndentLevel)) + fNameIndentLevel = fNameIndentLevel - 1 + } + } + out += fmt.Sprintf("%sreturn %s", indent, targetResVarName) + return out +} + +// IncompleteLateInitialization returns the go code which checks whether all the fields are late initialized. +// If all the fields are not late initialized, this method also returns the requeue delay needed to attempt +// late initialization again. +// +// Sample GeneratorConfig: +// fields: +// Name: +// late_initialize: {} +// ImageScanningConfiguration.ScanOnPush: +// late_initialize: +// min_backoff_seconds: 5 +// max_backoff_seconds: 15 +// map..subfield.x: +// late_initialize: +// min_backoff_seconds: 5 +// another.map..lastfield: +// late_initialize: +// min_backoff_seconds: 5 +// some.list: +// late_initialize: +// min_backoff_seconds: 10 +// structA.mapB..structC.valueD: +// late_initialize: +// min_backoff_seconds: 20 +// +// +// Sample Output: +// ko := rm.concreteResource(latest).ko +// if ko.Spec.ImageScanningConfiguration != nil { +// if ko.Spec.ImageScanningConfiguration.ScanOnPush == nil { +// return true +// } +// } +// if ko.Spec.Name == nil { +// return true +// } +// if ko.Spec.another != nil { +// if ko.Spec.another.map != nil { +// if ko.Spec.another.map["lastfield"] == nil { +// return true +// } +// } +// } +// if ko.Spec.map != nil { +// if ko.Spec.map["subfield"] != nil { +// if ko.Spec.map["subfield"].x == nil { +// return true +// } +// } +// } +// if ko.Spec.some != nil { +// if ko.Spec.some.list == nil { +// return true +// } +// } +// if ko.Spec.structA != nil { +// if ko.Spec.structA.mapB != nil { +// if ko.Spec.structA.mapB["structC"] != nil { +// if ko.Spec.structA.mapB["structC"].valueD == nil { +// return true +// } +// } +// } +// } +// return false +// +func IncompleteLateInitialization( + cfg *ackgenconfig.Config, + r *model.CRD, + resVarName string, + // Number of levels of indentation to use + indentLevel int, +) string { + out := "" + indent := strings.Repeat("\t", indentLevel) + sortedLateInitFieldNames, _ := getSortedLateInitFieldsAndConfig(cfg, r) + if len(sortedLateInitFieldNames) == 0 { + out += fmt.Sprintf("%sreturn false", indent) + return out + } + out += fmt.Sprintf("%sko := rm.concreteResource(%s).ko\n", indent, resVarName) + for _, fName := range sortedLateInitFieldNames { + // split the field name by period + // each substring represents a field. + fNameParts := strings.Split(fName, ".") + // fNameIndentLevel tracks the indentation level for every new line added + // This variable is incremented when building nested if blocks and decremented when closing those if blocks. + fNameIndentLevel := indentLevel + // fParentPath keeps track of parent path for any fNamePart + fParentPath := "" + mapShapedParent := false + for i, fNamePart := range fNameParts { + if fNamePart == "" { + mapShapedParent = true + continue + } + indent := strings.Repeat("\t", fNameIndentLevel) + fNamePartAccesor := fmt.Sprintf("Spec%s.%s", fParentPath, fNamePart) + if mapShapedParent { + fNamePartAccesor = fmt.Sprintf("Spec%s[%q]", fParentPath, fNamePart) + } + // Handling for all parts except last one + if i != len(fNameParts)-1 { + out += fmt.Sprintf("%sif ko.%s != nil {\n", indent, fNamePartAccesor) + // update fParentPath and fNameIndentLevel for next iteration + if mapShapedParent { + fParentPath = fmt.Sprintf("%s[%q]", fParentPath, fNamePart) + mapShapedParent = false + } else { + fParentPath = fmt.Sprintf("%s.%s", fParentPath, fNamePart) + } + fNameIndentLevel = fNameIndentLevel + 1 + } else { + // handle last part here + // for last part, if the late initialized field is still nil, calculate the retry backoff using + // acktypes.LateInitializationRetryConfig abstraction and set the incompleteInitialization flag to true + out += fmt.Sprintf("%sif ko.%s == nil {\n", indent, fNamePartAccesor) + fNameIndentLevel = fNameIndentLevel + 1 + indent = strings.Repeat("\t", fNameIndentLevel) + out += fmt.Sprintf("%sreturn true\n", indent) + } + } + // Close all if blocks with proper indentation + fNameIndentLevel = fNameIndentLevel - 1 + for fNameIndentLevel >= indentLevel { + out += fmt.Sprintf("%s}\n", strings.Repeat("\t", fNameIndentLevel)) + fNameIndentLevel = fNameIndentLevel - 1 + } + } + out += fmt.Sprintf("%sreturn false", indent) + return out +} diff --git a/pkg/generate/code/late_initialize_test.go b/pkg/generate/code/late_initialize_test.go new file mode 100644 index 00000000..20f7c53f --- /dev/null +++ b/pkg/generate/code/late_initialize_test.go @@ -0,0 +1,235 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code_test + +import ( + "testing" + + "github.com/aws-controllers-k8s/code-generator/pkg/generate/code" + "github.com/aws-controllers-k8s/code-generator/pkg/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_FindLateInitializedFieldNames_EmptyFieldConfig(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForService(t, "ecr") + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + // NO fieldConfig + assert.Empty(crd.Config().ResourceFields(crd.Names.Original)) + expected := + ` var lateInitializeFieldNames = []string{} +` + assert.Equal(expected, code.FindLateInitializedFieldNames(crd.Config(), crd, "lateInitializeFieldNames", 1)) +} + +func Test_FindLateInitializedFieldNames_NoLateInitializations(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{GeneratorConfigFile: "generator-with-field-config.yaml"}) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + // FieldConfig without lateInitialize + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["Name"]) + assert.Nil(crd.Config().ResourceFields(crd.Names.Original)["Name"].LateInitialize) + expected := + ` var lateInitializeFieldNames = []string{} +` + assert.Equal(expected, code.FindLateInitializedFieldNames(crd.Config(), crd, "lateInitializeFieldNames", 1)) +} + +func Test_FindLateInitializedFieldNames(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{GeneratorConfigFile: "generator-with-late-initialize.yaml"}) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["Name"]) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["ImageTagMutability"]) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["Name"].LateInitialize) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["ImageTagMutability"].LateInitialize) + expected := + ` var lateInitializeFieldNames = []string{"ImageTagMutability","Name",} +` + assert.Equal(expected, code.FindLateInitializedFieldNames(crd.Config(), crd, "lateInitializeFieldNames", 1)) +} + +func Test_LateInitializeFromReadOne_NoFieldsToLateInitialize(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForService(t, "ecr") + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + // NO fieldConfig + assert.Empty(crd.Config().ResourceFields(crd.Names.Original)) + expected := " return latest" + assert.Equal(expected, code.LateInitializeFromReadOne(crd.Config(), crd, "observed", "latest", 1)) +} + +func Test_LateInitializeFromReadOne_NonNestedPath(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{GeneratorConfigFile: "generator-with-late-initialize.yaml"}) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["Name"]) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["ImageTagMutability"]) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["Name"].LateInitialize) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["ImageTagMutability"].LateInitialize) + expected := + ` observedKo := rm.concreteResource(observed).ko + latestKo := rm.concreteResource(latest).ko + if observedKo.Spec.ImageTagMutability != nil && latestKo.Spec.ImageTagMutability == nil { + latestKo.Spec.ImageTagMutability = observedKo.Spec.ImageTagMutability + } + if observedKo.Spec.Name != nil && latestKo.Spec.Name == nil { + latestKo.Spec.Name = observedKo.Spec.Name + } + return latest` + assert.Equal(expected, code.LateInitializeFromReadOne(crd.Config(), crd, "observed", "latest", 1)) +} + +func Test_LateInitializeFromReadOne_NestedPath(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{GeneratorConfigFile: "generator-with-nested-path-late-initialize.yaml"}) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["Name"]) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["ImageScanningConfiguration.ScanOnPush"]) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["Name"].LateInitialize) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["ImageScanningConfiguration.ScanOnPush"].LateInitialize) + expected := + ` observedKo := rm.concreteResource(observed).ko + latestKo := rm.concreteResource(latest).ko + if observedKo.Spec.ImageScanningConfiguration != nil && latestKo.Spec.ImageScanningConfiguration != nil { + if observedKo.Spec.ImageScanningConfiguration.ScanOnPush != nil && latestKo.Spec.ImageScanningConfiguration.ScanOnPush == nil { + latestKo.Spec.ImageScanningConfiguration.ScanOnPush = observedKo.Spec.ImageScanningConfiguration.ScanOnPush + } + } + if observedKo.Spec.Name != nil && latestKo.Spec.Name == nil { + latestKo.Spec.Name = observedKo.Spec.Name + } + if observedKo.Spec.another != nil && latestKo.Spec.another != nil { + if observedKo.Spec.another.map != nil && latestKo.Spec.another.map != nil { + if observedKo.Spec.another.map["lastfield"] != nil && latestKo.Spec.another.map["lastfield"] == nil { + latestKo.Spec.another.map["lastfield"] = observedKo.Spec.another.map["lastfield"] + } + } + } + if observedKo.Spec.map != nil && latestKo.Spec.map != nil { + if observedKo.Spec.map["subfield"] != nil && latestKo.Spec.map["subfield"] != nil { + if observedKo.Spec.map["subfield"].x != nil && latestKo.Spec.map["subfield"].x == nil { + latestKo.Spec.map["subfield"].x = observedKo.Spec.map["subfield"].x + } + } + } + if observedKo.Spec.some != nil && latestKo.Spec.some != nil { + if observedKo.Spec.some.list != nil && latestKo.Spec.some.list == nil { + latestKo.Spec.some.list = observedKo.Spec.some.list + } + } + if observedKo.Spec.structA != nil && latestKo.Spec.structA != nil { + if observedKo.Spec.structA.mapB != nil && latestKo.Spec.structA.mapB != nil { + if observedKo.Spec.structA.mapB["structC"] != nil && latestKo.Spec.structA.mapB["structC"] != nil { + if observedKo.Spec.structA.mapB["structC"].valueD != nil && latestKo.Spec.structA.mapB["structC"].valueD == nil { + latestKo.Spec.structA.mapB["structC"].valueD = observedKo.Spec.structA.mapB["structC"].valueD + } + } + } + } + return latest` + assert.Equal(expected, code.LateInitializeFromReadOne(crd.Config(), crd, "observed", "latest", 1)) +} + +func Test_IncompleteLateInitialization_NoFieldsToLateInitialization(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{GeneratorConfigFile: "generator-with-field-config.yaml"}) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + expected := + ` return false` + assert.Equal(expected, code.IncompleteLateInitialization(crd.Config(), crd, "latestWithDefaults", 1)) +} + +func Test_IncompleteLateInitialization(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{GeneratorConfigFile: "generator-with-nested-path-late-initialize.yaml"}) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["Name"]) + assert.NotEmpty(crd.Config().ResourceFields(crd.Names.Original)["ImageScanningConfiguration.ScanOnPush"]) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["Name"].LateInitialize) + assert.NotNil(crd.Config().ResourceFields(crd.Names.Original)["ImageScanningConfiguration.ScanOnPush"].LateInitialize) + expected := + ` ko := rm.concreteResource(latest).ko + if ko.Spec.ImageScanningConfiguration != nil { + if ko.Spec.ImageScanningConfiguration.ScanOnPush == nil { + return true + } + } + if ko.Spec.Name == nil { + return true + } + if ko.Spec.another != nil { + if ko.Spec.another.map != nil { + if ko.Spec.another.map["lastfield"] == nil { + return true + } + } + } + if ko.Spec.map != nil { + if ko.Spec.map["subfield"] != nil { + if ko.Spec.map["subfield"].x == nil { + return true + } + } + } + if ko.Spec.some != nil { + if ko.Spec.some.list == nil { + return true + } + } + if ko.Spec.structA != nil { + if ko.Spec.structA.mapB != nil { + if ko.Spec.structA.mapB["structC"] != nil { + if ko.Spec.structA.mapB["structC"].valueD == nil { + return true + } + } + } + } + return false` + assert.Equal(expected, code.IncompleteLateInitialization(crd.Config(), crd, "latest", 1)) +} diff --git a/pkg/generate/config/field.go b/pkg/generate/config/field.go index d241135a..7e72bfdf 100644 --- a/pkg/generate/config/field.go +++ b/pkg/generate/config/field.go @@ -136,6 +136,22 @@ type PrintFieldConfig struct { Index int `json:"index"` } +// LateInitializeConfig contains instructions for how to handle the +// retrieval and setting of server-side defaulted fields. +// NOTE: Currently the members of this have no effect on late initialization of fields. +// Currently the late initialization is requeued with static delay of 5 second. +// TODO: (vijat@) Add support of retry/backoff for late initialization. +type LateInitializeConfig struct { + // MinBackoffSeconds provides the minimum backoff to attempt late initialization again after an unsuccessful + // attempt to late initialized fields from ReadOne output + // For every attempt, the reconciler will calculate the delay between MinBackoffSeconds and MaxBackoffSeconds + // using exponential backoff and retry strategy + MinBackoffSeconds int `json:"min_backoff_seconds,omitempty"` + // MaxBackoffSeconds provide the maximum allowed backoff when retrying late initialization after an + // unsuccessful attempt. + MaxBackoffSeconds int `json:"max_backoff_seconds"` +} + // FieldConfig contains instructions to the code generator about how // to interpret the value of an Attribute and how to map it to a CRD's Spec or // Status field @@ -184,4 +200,7 @@ type FieldConfig struct { // influence hows field are printed in `kubectl get` response. If this field // is not nil, it will be added to the columns of `kubectl get`. Print *PrintFieldConfig `json:"print,omitempty"` + // Late Initialize instructs the code generator how to handle the late initialization + // of the field. + LateInitialize *LateInitializeConfig `json:"late_initialize,omitempty"` } diff --git a/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-config.yaml b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-config.yaml new file mode 100644 index 00000000..5fb7cba4 --- /dev/null +++ b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-config.yaml @@ -0,0 +1,12 @@ +resources: + Repository: + fields: + Name: + is_name: true + exceptions: + errors: + 404: + code: RepositoryNotFoundException + list_operation: + match_fields: + - RepositoryName diff --git a/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-late-initialize.yaml b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-late-initialize.yaml new file mode 100644 index 00000000..93a42c84 --- /dev/null +++ b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-late-initialize.yaml @@ -0,0 +1,15 @@ +resources: + Repository: + fields: + Name: + late_initialize: {} + ImageTagMutability: + late_initialize: + min_backoff_seconds: 5 + exceptions: + errors: + 404: + code: RepositoryNotFoundException + list_operation: + match_fields: + - RepositoryName diff --git a/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-nested-path-late-initialize.yaml b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-nested-path-late-initialize.yaml new file mode 100644 index 00000000..c87e8f60 --- /dev/null +++ b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-nested-path-late-initialize.yaml @@ -0,0 +1,28 @@ +resources: + Repository: + fields: + Name: + late_initialize: {} + ImageScanningConfiguration.ScanOnPush: + late_initialize: + min_backoff_seconds: 5 + max_backoff_seconds: 15 + map..subfield.x: + late_initialize: + min_backoff_seconds: 5 + another.map..lastfield: + late_initialize: + min_backoff_seconds: 5 + some.list: + late_initialize: + min_backoff_seconds: 10 + structA.mapB..structC.valueD: + late_initialize: + min_backoff_seconds: 20 + exceptions: + errors: + 404: + code: RepositoryNotFoundException + list_operation: + match_fields: + - RepositoryName diff --git a/templates/pkg/resource/manager.go.tpl b/templates/pkg/resource/manager.go.tpl index aa4fa8c4..649db9c1 100644 --- a/templates/pkg/resource/manager.go.tpl +++ b/templates/pkg/resource/manager.go.tpl @@ -5,12 +5,16 @@ package {{ .CRD.Names.Snake }} import ( "context" "fmt" + "time" ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1" - ackcfg "github.com/aws-controllers-k8s/runtime/pkg/config" ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare" + ackcfg "github.com/aws-controllers-k8s/runtime/pkg/config" + ackcondition "github.com/aws-controllers-k8s/runtime/pkg/condition" ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors" ackmetrics "github.com/aws-controllers-k8s/runtime/pkg/metrics" + ackrequeue "github.com/aws-controllers-k8s/runtime/pkg/requeue" + ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log" acktypes "github.com/aws-controllers-k8s/runtime/pkg/types" "github.com/aws/aws-sdk-go/aws/session" "github.com/go-logr/logr" @@ -23,6 +27,8 @@ import ( // +kubebuilder:rbac:groups={{ .APIGroup }},resources={{ ToLower .CRD.Plural }},verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups={{ .APIGroup }},resources={{ ToLower .CRD.Plural }}/status,verbs=get;update;patch +{{ GoCodeFindLateInitializedFieldNames .CRD "lateInitializeFieldNames" 1 }} + // resourceManager is responsible for providing a consistent way to perform // CRUD operations in a backend AWS service API for Book custom resources. type resourceManager struct { @@ -156,6 +162,69 @@ func (rm *resourceManager) ARNFromName(name string) string { ) } +// LateInitialize returns an acktypes.AWSResource after setting the late initialized +// fields from the readOne call. This method will initialize the optional fields +// which were not provided by the k8s user but were defaulted by the AWS service. +// If there are no such fields to be initialized, the returned object is similar to +// object passed in the parameter. +func (rm *resourceManager) LateInitialize( + ctx context.Context, + latest acktypes.AWSResource, +) (acktypes.AWSResource, error) { + rlog := ackrtlog.FromContext(ctx) + // If there are no fields to late initialize, do nothing + if len(lateInitializeFieldNames) == 0 { + rlog.Debug("no late initialization required.") + return latest, nil + } + lateInitConditionReason := "" + lateInitConditionMessage := "" +{{- if $hookCode := Hook .CRD "late_initialize_pre_read_one" }} +{{ $hookCode }} +{{- end }} + observed, err := rm.ReadOne(ctx, latest) + if err != nil { + lateInitConditionMessage = "Unable to complete Read operation required for late initialization" + lateInitConditionReason = "Late Initialization Failure" + ackcondition.SetLateInitialized(latest, corev1.ConditionFalse, &lateInitConditionMessage, &lateInitConditionReason) + return latest, err + } +{{- if $hookCode := Hook .CRD "late_initialize_post_read_one" }} +{{ $hookCode }} +{{- end }} + latest = rm.lateInitializeFromReadOneOutput(observed, latest) + incompleteInitialization := rm.incompleteLateInitialization(latest) + if incompleteInitialization { + // Add the condition with LateInitialized=False + lateInitConditionMessage = "Late initialization did not complete, requeuing with delay of 5 seconds" + lateInitConditionReason = "Delayed Late Initialization" + ackcondition.SetLateInitialized(latest, corev1.ConditionFalse, &lateInitConditionMessage, &lateInitConditionReason) + return latest, ackrequeue.NeededAfter(nil, time.Duration(5)*time.Second) + } + // Set LateIntialized condition to True + lateInitConditionMessage = "Late initialization successful" + lateInitConditionReason = "Late initialization successful" + ackcondition.SetLateInitialized(latest, corev1.ConditionTrue, &lateInitConditionMessage, &lateInitConditionReason) + return latest, nil +} + +// incompleteLateInitialization return true if there are fields which were supposed to be +// late initialized but are not. If all the fields are late initialized, false is returned +func (rm *resourceManager) incompleteLateInitialization( + latest acktypes.AWSResource, +) bool { +{{ GoCodeIncompleteLateInitialization .CRD "latest" 1 }} +} + +// lateInitializeFromReadOneOutput late initializes the 'latest' resource from the 'observed' +// resource and returns 'latest' resource +func (rm *resourceManager) lateInitializeFromReadOneOutput( + observed acktypes.AWSResource, + latest acktypes.AWSResource, +) acktypes.AWSResource { +{{ GoCodeLateInitializeFromReadOne .CRD "observed" "latest" 1 }} +} + // newResourceManager returns a new struct implementing // acktypes.AWSResourceManager func newResourceManager(