From 895b82ffeb0c3beff5ed21d97c23cfcd7abf22d2 Mon Sep 17 00:00:00 2001 From: Mateusz Gozdek Date: Tue, 9 Feb 2021 12:10:21 +0100 Subject: [PATCH 1/8] bootstrap/kubeadm: Add Ignition bootstrap format Add support for using Ignition as the format for bootstrap data. This allows using Ignition-based distros as the operating system for workload cluster nodes. Co-authored-by: Suraj Deshmukh Signed-off-by: Mateusz Gozdek Signed-off-by: Johanan Liebermann --- OWNERS_ALIASES | 10 + bootstrap/kubeadm/api/v1alpha3/conversion.go | 10 + .../api/v1alpha3/zz_generated.conversion.go | 16 +- bootstrap/kubeadm/api/v1alpha4/conversion.go | 48 +- .../kubeadm/api/v1alpha4/conversion_test.go | 19 +- .../api/v1alpha4/zz_generated.conversion.go | 40 +- .../api/v1beta1/kubeadmconfig_types.go | 32 +- .../api/v1beta1/kubeadmconfig_types_test.go | 73 +++ .../api/v1beta1/kubeadmconfig_webhook.go | 89 +++- .../api/v1beta1/zz_generated.deepcopy.go | 40 ++ ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 19 + ...uster.x-k8s.io_kubeadmconfigtemplates.yaml | 22 + .../controllers/kubeadmconfig_controller.go | 60 ++- .../kubeadmconfig_controller_test.go | 127 +++++- bootstrap/kubeadm/internal/ignition/OWNERS | 8 + .../kubeadm/internal/ignition/clc/clc.go | 373 ++++++++++++++++ .../kubeadm/internal/ignition/clc/clc_test.go | 418 ++++++++++++++++++ .../kubeadm/internal/ignition/ignition.go | 115 +++++ .../internal/ignition/ignition_test.go | 386 ++++++++++++++++ .../kubeadm/api/v1alpha3/conversion.go | 2 + .../kubeadm/api/v1alpha4/conversion.go | 43 +- .../kubeadm/api/v1alpha4/conversion_test.go | 21 +- ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 19 + ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 22 + go.mod | 9 +- go.sum | 34 +- test/e2e/Makefile | 1 + test/e2e/config/docker.yaml | 1 + .../cluster-template-ignition/ignition.yaml | 26 ++ .../kustomization.yaml | 7 + test/e2e/quick_start_test.go | 13 + test/go.sum | 24 + 32 files changed, 2070 insertions(+), 57 deletions(-) create mode 100644 bootstrap/kubeadm/internal/ignition/OWNERS create mode 100644 bootstrap/kubeadm/internal/ignition/clc/clc.go create mode 100644 bootstrap/kubeadm/internal/ignition/clc/clc_test.go create mode 100644 bootstrap/kubeadm/internal/ignition/ignition.go create mode 100644 bootstrap/kubeadm/internal/ignition/ignition_test.go create mode 100644 test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/ignition.yaml create mode 100644 test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/kustomization.yaml diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 4cca9bce6d14..b2c5e75a59a4 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -46,6 +46,16 @@ aliases: cluster-api-bootstrap-provider-kubeadm-maintainers: cluster-api-bootstrap-provider-kubeadm-reviewers: + # ----------------------------------------------------------- + # OWNER_ALIASES for bootstrap/kubeadm/internal/ignition + # ----------------------------------------------------------- + + cluster-api-bootstrap-provider-kubeadm-ignition-maintainers: + cluster-api-bootstrap-provider-kubeadm-ignition-reviewers: + - dongsupark + - invidian + - johananl + # ----------------------------------------------------------- # OWNER_ALIASES for controlplane/kubeadm # ----------------------------------------------------------- diff --git a/bootstrap/kubeadm/api/v1alpha3/conversion.go b/bootstrap/kubeadm/api/v1alpha3/conversion.go index 3ce22249f269..f0ce83ce2e43 100644 --- a/bootstrap/kubeadm/api/v1alpha3/conversion.go +++ b/bootstrap/kubeadm/api/v1alpha3/conversion.go @@ -52,6 +52,8 @@ func (src *KubeadmConfig) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.InitConfiguration.NodeRegistration.IgnorePreflightErrors = restored.Spec.InitConfiguration.NodeRegistration.IgnorePreflightErrors } + dst.Spec.Ignition = restored.Spec.Ignition + return nil } @@ -109,6 +111,8 @@ func (src *KubeadmConfigTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.InitConfiguration.NodeRegistration.IgnorePreflightErrors = restored.Spec.Template.Spec.InitConfiguration.NodeRegistration.IgnorePreflightErrors } + dst.Spec.Template.Spec.Ignition = restored.Spec.Template.Spec.Ignition + return nil } @@ -177,3 +181,9 @@ func Convert_v1beta1_JoinConfiguration_To_upstreamv1beta1_JoinConfiguration(in * // NodeRegistrationOptions.IgnorePreflightErrors does not exist in kubeadm v1beta1 API return upstreamv1beta1.Convert_v1beta1_JoinConfiguration_To_upstreamv1beta1_JoinConfiguration(in, out, s) } + +// Convert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec is an autogenerated conversion function. +func Convert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(in *v1beta1.KubeadmConfigSpec, out *KubeadmConfigSpec, s apiconversion.Scope) error { + // KubeadmConfigSpec.Ignition does not exist in kubeadm v1alpha3 API. + return autoConvert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(in, out, s) +} diff --git a/bootstrap/kubeadm/api/v1alpha3/zz_generated.conversion.go b/bootstrap/kubeadm/api/v1alpha3/zz_generated.conversion.go index 25c562be9e70..ea239590f8c0 100644 --- a/bootstrap/kubeadm/api/v1alpha3/zz_generated.conversion.go +++ b/bootstrap/kubeadm/api/v1alpha3/zz_generated.conversion.go @@ -104,11 +104,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeadmConfigSpec)(nil), (*KubeadmConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(a.(*v1beta1.KubeadmConfigSpec), b.(*KubeadmConfigSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*v1beta1.KubeadmConfigStatus)(nil), (*KubeadmConfigStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_KubeadmConfigStatus_To_v1alpha3_KubeadmConfigStatus(a.(*v1beta1.KubeadmConfigStatus), b.(*KubeadmConfigStatus), scope) }); err != nil { @@ -229,6 +224,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.KubeadmConfigSpec)(nil), (*KubeadmConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(a.(*v1beta1.KubeadmConfigSpec), b.(*KubeadmConfigSpec), scope) + }); err != nil { + return err + } return nil } @@ -498,14 +498,10 @@ func autoConvert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(in *v1b out.Format = Format(in.Format) out.Verbosity = (*int32)(unsafe.Pointer(in.Verbosity)) out.UseExperimentalRetryJoin = in.UseExperimentalRetryJoin + // WARNING: in.Ignition requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec is an autogenerated conversion function. -func Convert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(in *v1beta1.KubeadmConfigSpec, out *KubeadmConfigSpec, s conversion.Scope) error { - return autoConvert_v1beta1_KubeadmConfigSpec_To_v1alpha3_KubeadmConfigSpec(in, out, s) -} - func autoConvert_v1alpha3_KubeadmConfigStatus_To_v1beta1_KubeadmConfigStatus(in *KubeadmConfigStatus, out *v1beta1.KubeadmConfigStatus, s conversion.Scope) error { out.Ready = in.Ready out.DataSecretName = (*string)(unsafe.Pointer(in.DataSecretName)) diff --git a/bootstrap/kubeadm/api/v1alpha4/conversion.go b/bootstrap/kubeadm/api/v1alpha4/conversion.go index 489e25d4c544..b348fb89f521 100644 --- a/bootstrap/kubeadm/api/v1alpha4/conversion.go +++ b/bootstrap/kubeadm/api/v1alpha4/conversion.go @@ -17,21 +17,39 @@ limitations under the License. package v1alpha4 import ( + apiconversion "k8s.io/apimachinery/pkg/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) func (src *KubeadmConfig) ConvertTo(dstRaw conversion.Hub) error { dst := dstRaw.(*v1beta1.KubeadmConfig) - return Convert_v1alpha4_KubeadmConfig_To_v1beta1_KubeadmConfig(src, dst, nil) + if err := Convert_v1alpha4_KubeadmConfig_To_v1beta1_KubeadmConfig(src, dst, nil); err != nil { + return err + } + + // Manually restore data. + restored := &v1beta1.KubeadmConfig{} + if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + return err + } + + dst.Spec.Ignition = restored.Spec.Ignition + + return nil } func (dst *KubeadmConfig) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*v1beta1.KubeadmConfig) - return Convert_v1beta1_KubeadmConfig_To_v1alpha4_KubeadmConfig(src, dst, nil) + if err := Convert_v1beta1_KubeadmConfig_To_v1alpha4_KubeadmConfig(src, dst, nil); err != nil { + return err + } + // Preserve Hub data on down-conversion except for metadata. + return utilconversion.MarshalData(src, dst) } func (src *KubeadmConfigList) ConvertTo(dstRaw conversion.Hub) error { @@ -49,13 +67,29 @@ func (dst *KubeadmConfigList) ConvertFrom(srcRaw conversion.Hub) error { func (src *KubeadmConfigTemplate) ConvertTo(dstRaw conversion.Hub) error { dst := dstRaw.(*v1beta1.KubeadmConfigTemplate) - return Convert_v1alpha4_KubeadmConfigTemplate_To_v1beta1_KubeadmConfigTemplate(src, dst, nil) + if err := Convert_v1alpha4_KubeadmConfigTemplate_To_v1beta1_KubeadmConfigTemplate(src, dst, nil); err != nil { + return err + } + + // Manually restore data. + restored := &v1beta1.KubeadmConfigTemplate{} + if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + return err + } + + dst.Spec.Template.Spec.Ignition = restored.Spec.Template.Spec.Ignition + + return nil } func (dst *KubeadmConfigTemplate) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*v1beta1.KubeadmConfigTemplate) - return Convert_v1beta1_KubeadmConfigTemplate_To_v1alpha4_KubeadmConfigTemplate(src, dst, nil) + if err := Convert_v1beta1_KubeadmConfigTemplate_To_v1alpha4_KubeadmConfigTemplate(src, dst, nil); err != nil { + return err + } + // Preserve Hub data on down-conversion except for metadata. + return utilconversion.MarshalData(src, dst) } func (src *KubeadmConfigTemplateList) ConvertTo(dstRaw conversion.Hub) error { @@ -69,3 +103,9 @@ func (dst *KubeadmConfigTemplateList) ConvertFrom(srcRaw conversion.Hub) error { return Convert_v1beta1_KubeadmConfigTemplateList_To_v1alpha4_KubeadmConfigTemplateList(src, dst, nil) } + +// Convert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec is an autogenerated conversion function. +func Convert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(in *v1beta1.KubeadmConfigSpec, out *KubeadmConfigSpec, s apiconversion.Scope) error { + // KubeadmConfigSpec.Ignition does not exist in kubeadm v1alpha4 API. + return autoConvert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(in, out, s) +} diff --git a/bootstrap/kubeadm/api/v1alpha4/conversion_test.go b/bootstrap/kubeadm/api/v1alpha4/conversion_test.go index d651f03c99a2..b05f8037b958 100644 --- a/bootstrap/kubeadm/api/v1alpha4/conversion_test.go +++ b/bootstrap/kubeadm/api/v1alpha4/conversion_test.go @@ -28,6 +28,11 @@ import ( utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) +const ( + fakeID = "abcdef" + fakeSecret = "abcdef0123456789" +) + func TestFuzzyConversion(t *testing.T) { t.Run("for KubeadmConfig", utilconversion.FuzzTestFunc(utilconversion.FuzzTestFuncInput{ Hub: &v1beta1.KubeadmConfig{}, @@ -57,6 +62,7 @@ func fuzzFuncs(_ runtimeserializer.CodecFactory) []interface{} { // the values for ID and Secret to working alphanumeric values. kubeadmBootstrapTokenStringFuzzerV1UpstreamBeta1, kubeadmBootstrapTokenStringFuzzerV1Beta1, + kubeadmBootstrapTokenStringFuzzerV1Alpha4, } } @@ -75,11 +81,16 @@ func clusterConfigurationFuzzer(obj *upstreamv1beta1.ClusterConfiguration, c fuz } func kubeadmBootstrapTokenStringFuzzerV1UpstreamBeta1(in *upstreamv1beta1.BootstrapTokenString, c fuzz.Continue) { - in.ID = "abcdef" - in.Secret = "abcdef0123456789" + in.ID = fakeID + in.Secret = fakeSecret } func kubeadmBootstrapTokenStringFuzzerV1Beta1(in *v1beta1.BootstrapTokenString, c fuzz.Continue) { - in.ID = "abcdef" - in.Secret = "abcdef0123456789" + in.ID = fakeID + in.Secret = fakeSecret +} + +func kubeadmBootstrapTokenStringFuzzerV1Alpha4(in *BootstrapTokenString, c fuzz.Continue) { + in.ID = fakeID + in.Secret = fakeSecret } diff --git a/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go b/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go index 48c9073a4d6f..6cd521551442 100644 --- a/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go +++ b/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go @@ -285,11 +285,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.KubeadmConfigSpec)(nil), (*KubeadmConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(a.(*v1beta1.KubeadmConfigSpec), b.(*KubeadmConfigSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*KubeadmConfigStatus)(nil), (*v1beta1.KubeadmConfigStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_KubeadmConfigStatus_To_v1beta1_KubeadmConfigStatus(a.(*KubeadmConfigStatus), b.(*v1beta1.KubeadmConfigStatus), scope) }); err != nil { @@ -410,6 +405,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.KubeadmConfigSpec)(nil), (*KubeadmConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(a.(*v1beta1.KubeadmConfigSpec), b.(*KubeadmConfigSpec), scope) + }); err != nil { + return err + } return nil } @@ -1125,14 +1125,10 @@ func autoConvert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(in *v1b out.Format = Format(in.Format) out.Verbosity = (*int32)(unsafe.Pointer(in.Verbosity)) out.UseExperimentalRetryJoin = in.UseExperimentalRetryJoin + // WARNING: in.Ignition requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec is an autogenerated conversion function. -func Convert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(in *v1beta1.KubeadmConfigSpec, out *KubeadmConfigSpec, s conversion.Scope) error { - return autoConvert_v1beta1_KubeadmConfigSpec_To_v1alpha4_KubeadmConfigSpec(in, out, s) -} - func autoConvert_v1alpha4_KubeadmConfigStatus_To_v1beta1_KubeadmConfigStatus(in *KubeadmConfigStatus, out *v1beta1.KubeadmConfigStatus, s conversion.Scope) error { out.Ready = in.Ready out.DataSecretName = (*string)(unsafe.Pointer(in.DataSecretName)) @@ -1211,7 +1207,17 @@ func Convert_v1beta1_KubeadmConfigTemplate_To_v1alpha4_KubeadmConfigTemplate(in func autoConvert_v1alpha4_KubeadmConfigTemplateList_To_v1beta1_KubeadmConfigTemplateList(in *KubeadmConfigTemplateList, out *v1beta1.KubeadmConfigTemplateList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1beta1.KubeadmConfigTemplate)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1beta1.KubeadmConfigTemplate, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_KubeadmConfigTemplate_To_v1beta1_KubeadmConfigTemplate(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -1222,7 +1228,17 @@ func Convert_v1alpha4_KubeadmConfigTemplateList_To_v1beta1_KubeadmConfigTemplate func autoConvert_v1beta1_KubeadmConfigTemplateList_To_v1alpha4_KubeadmConfigTemplateList(in *v1beta1.KubeadmConfigTemplateList, out *KubeadmConfigTemplateList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]KubeadmConfigTemplate)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]KubeadmConfigTemplate, len(*in)) + for i := range *in { + if err := Convert_v1beta1_KubeadmConfigTemplate_To_v1alpha4_KubeadmConfigTemplate(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types.go index d4cc96663ba6..40bbe36a5505 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types.go @@ -23,12 +23,15 @@ import ( ) // Format specifies the output format of the bootstrap data -// +kubebuilder:validation:Enum=cloud-config +// +kubebuilder:validation:Enum=cloud-config;ignition type Format string const ( // CloudConfig make the bootstrap data to be of cloud-config format. CloudConfig Format = "cloud-config" + + // Ignition make the bootstrap data to be of Ignition format. + Ignition Format = "ignition" ) // KubeadmConfigSpec defines the desired state of KubeadmConfig. @@ -95,6 +98,33 @@ type KubeadmConfigSpec struct { // For more information, refer to https://github.com/kubernetes-sigs/cluster-api/pull/2763#discussion_r397306055. // +optional UseExperimentalRetryJoin bool `json:"useExperimentalRetryJoin,omitempty"` + + // Ignition contains Ignition specific configuration. + // +optional + Ignition *IgnitionSpec `json:"ignition,omitempty"` +} + +// IgnitionSpec contains Ignition specific configuration. +type IgnitionSpec struct { + // ContainerLinuxConfig contains CLC specific configuration. + // +optional + ContainerLinuxConfig *ContainerLinuxConfig `json:"containerLinuxConfig,omitempty"` +} + +// ContainerLinuxConfig contains CLC-specific configuration. +// +// We use a structured type here to allow adding additional fields, for example 'version'. +type ContainerLinuxConfig struct { + // AdditionalConfig contains additional configuration to be merged with the Ignition + // configuration generated by the bootstrapper controller. More info: https://coreos.github.io/ignition/operator-notes/#config-merging + // + // The data format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/ + // +optional + AdditionalConfig string `json:"additionalConfig,omitempty"` + + // Strict controls if AdditionalConfig should be strictly parsed. If so, warnings are treated as errors. + // +optional + Strict bool `json:"strict,omitempty"` } // KubeadmConfigStatus defines the observed state of KubeadmConfig. diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go index 1df1908fe2c8..47268b18fe0d 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ) func TestClusterValidate(t *testing.T) { @@ -141,6 +142,78 @@ func TestClusterValidate(t *testing.T) { }, expectErr: true, }, + "Ignition field is set, format is not Ignition": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Ignition: &IgnitionSpec{}, + }, + }, + expectErr: true, + }, + "Ignition field is not set, format is Ignition": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + }, + }, + }, + "format is Ignition, user is inactive": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + Users: []User{ + { + Inactive: pointer.BoolPtr(true), + }, + }, + }, + }, + expectErr: true, + }, + "format is Ignition, non-GPT partition configured": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + DiskSetup: &DiskSetup{ + Partitions: []Partition{ + { + TableType: pointer.StringPtr("MS-DOS"), + }, + }, + }, + }, + }, + expectErr: true, + }, + "format is Ignition, experimental retry join is set": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + UseExperimentalRetryJoin: true, + }, + }, + expectErr: true, + }, } for name, tt := range cases { diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go index d43964601837..26506f1e829e 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go @@ -17,6 +17,8 @@ limitations under the License. package v1beta1 import ( + "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" @@ -25,6 +27,7 @@ import ( ) var ( + cannotUseWithIgnition = fmt.Sprintf("not supported when spec.format is set to %q", Ignition) conflictingFileSourceMsg = "only one of content or contentFrom may be specified for a single file" missingSecretNameMsg = "secret file source must specify non-empty secret name" missingSecretKeyMsg = "secret file source must specify non-empty secret key" @@ -57,6 +60,25 @@ func (c *KubeadmConfig) ValidateDelete() error { } func (c *KubeadmConfigSpec) validate(name string) error { + allErrs := c.Validate() + + if len(allErrs) == 0 { + return nil + } + return apierrors.NewInvalid(GroupVersion.WithKind("KubeadmConfig").GroupKind(), name, allErrs) +} + +// Validate ensures the KubeadmConfigSpec is valid. +func (c *KubeadmConfigSpec) Validate() field.ErrorList { + var allErrs field.ErrorList + + allErrs = append(allErrs, c.validateFiles()...) + allErrs = append(allErrs, c.validateIgnition()...) + + return allErrs +} + +func (c *KubeadmConfigSpec) validateFiles() field.ErrorList { var allErrs field.ErrorList knownPaths := map[string]struct{}{} @@ -112,8 +134,69 @@ func (c *KubeadmConfigSpec) validate(name string) error { knownPaths[file.Path] = struct{}{} } - if len(allErrs) == 0 { - return nil + return allErrs +} + +func (c *KubeadmConfigSpec) validateIgnition() field.ErrorList { + var allErrs field.ErrorList + + if c.Format != Ignition { + if c.Ignition != nil { + allErrs = append( + allErrs, + field.Invalid( + field.NewPath("spec", "format"), + c.Format, + fmt.Sprintf("must be set to %q if spec.ignition is set", Ignition), + ), + ) + } + + return allErrs } - return apierrors.NewInvalid(GroupVersion.WithKind("KubeadmConfig").GroupKind(), name, allErrs) + + for i, user := range c.Users { + if user.Inactive != nil && *user.Inactive { + allErrs = append( + allErrs, + field.Forbidden( + field.NewPath("spec", "users").Index(i).Child("inactive"), + cannotUseWithIgnition, + ), + ) + } + } + + if c.UseExperimentalRetryJoin { + allErrs = append( + allErrs, + field.Forbidden( + field.NewPath("spec", "useExperimentalRetryJoin"), + cannotUseWithIgnition, + ), + ) + } + + if c.DiskSetup == nil { + return allErrs + } + + for i, partition := range c.DiskSetup.Partitions { + if partition.TableType != nil && *partition.TableType != "gpt" { + allErrs = append( + allErrs, + field.Invalid( + field.NewPath("spec", "diskSetup", "partitions").Index(i).Child("tableType"), + *partition.TableType, + fmt.Sprintf( + "only partition type %q is supported when spec.format is set to %q", + "gpt", + Ignition, + ), + ), + ) + } + } + + return allErrs } diff --git a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go index a90f85d39cc8..80ce9d284902 100644 --- a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go +++ b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go @@ -211,6 +211,21 @@ func (in *ClusterStatus) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerLinuxConfig) DeepCopyInto(out *ContainerLinuxConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerLinuxConfig. +func (in *ContainerLinuxConfig) DeepCopy() *ContainerLinuxConfig { + if in == nil { + return nil + } + out := new(ContainerLinuxConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { *out = *in @@ -459,6 +474,26 @@ func (in *HostPathMount) DeepCopy() *HostPathMount { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IgnitionSpec) DeepCopyInto(out *IgnitionSpec) { + *out = *in + if in.ContainerLinuxConfig != nil { + in, out := &in.ContainerLinuxConfig, &out.ContainerLinuxConfig + *out = new(ContainerLinuxConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IgnitionSpec. +func (in *IgnitionSpec) DeepCopy() *IgnitionSpec { + if in == nil { + return nil + } + out := new(IgnitionSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageMeta) DeepCopyInto(out *ImageMeta) { *out = *in @@ -681,6 +716,11 @@ func (in *KubeadmConfigSpec) DeepCopyInto(out *KubeadmConfigSpec) { *out = new(int32) **out = **in } + if in.Ignition != nil { + in, out := &in.Ignition, &out.Ignition + *out = new(IgnitionSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmConfigSpec. diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index 8ee3facdf9a3..9fbe3cfb347a 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -2467,7 +2467,26 @@ spec: description: Format specifies the output format of the bootstrap data enum: - cloud-config + - ignition type: string + ignition: + description: Ignition contains Ignition specific configuration. + properties: + containerLinuxConfig: + description: ContainerLinuxConfig contains CLC specific configuration. + properties: + additionalConfig: + description: "AdditionalConfig contains additional configuration + to be merged with the Ignition configuration generated by + the bootstrapper controller. More info: https://coreos.github.io/ignition/operator-notes/#config-merging + \n The data format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/" + type: string + strict: + description: Strict controls if AdditionalConfig should be + strictly parsed. If so, warnings are treated as errors. + type: boolean + type: object + type: object initConfiguration: description: InitConfiguration along with ClusterConfiguration are the configurations necessary for the init command diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml index 8788552838c8..baf390262927 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml @@ -2482,7 +2482,29 @@ spec: data enum: - cloud-config + - ignition type: string + ignition: + description: Ignition contains Ignition specific configuration. + properties: + containerLinuxConfig: + description: ContainerLinuxConfig contains CLC specific + configuration. + properties: + additionalConfig: + description: "AdditionalConfig contains additional + configuration to be merged with the Ignition configuration + generated by the bootstrapper controller. More info: + https://coreos.github.io/ignition/operator-notes/#config-merging + \n The data format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/" + type: string + strict: + description: Strict controls if AdditionalConfig should + be strictly parsed. If so, warnings are treated + as errors. + type: boolean + type: object + type: object initConfiguration: description: InitConfiguration along with ClusterConfiguration are the configurations necessary for the init command diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go index 3fc55d075b5c..a7be733a6ecb 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go @@ -41,6 +41,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/locking" kubeadmtypes "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types" bsutil "sigs.k8s.io/cluster-api/bootstrap/util" @@ -446,7 +447,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex return ctrl.Result{}, err } - cloudInitData, err := cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInput{ + controlPlaneInput := &cloudinit.ControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ AdditionalFiles: files, NTP: scope.Config.Spec.NTP, @@ -460,13 +461,25 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex InitConfiguration: initdata, ClusterConfiguration: clusterdata, Certificates: certificates, - }) + } + + var bootstrapInitData []byte + switch scope.Config.Spec.Format { + case bootstrapv1.Ignition: + bootstrapInitData, _, err = ignition.NewInitControlPlane(&ignition.ControlPlaneInput{ + ControlPlaneInput: controlPlaneInput, + Ignition: scope.Config.Spec.Ignition, + }) + default: + bootstrapInitData, err = cloudinit.NewInitControlPlane(controlPlaneInput) + } + if err != nil { - scope.Error(err, "Failed to generate cloud init for bootstrap control plane") + scope.Error(err, "Failed to generate user data for bootstrap control plane") return ctrl.Result{}, err } - if err := r.storeBootstrapData(ctx, scope, cloudInitData); err != nil { + if err := r.storeBootstrapData(ctx, scope, bootstrapInitData); err != nil { scope.Error(err, "Failed to store bootstrap data") return ctrl.Result{}, err } @@ -527,7 +540,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) return ctrl.Result{}, err } - cloudJoinData, err := cloudinit.NewNode(&cloudinit.NodeInput{ + nodeInput := &cloudinit.NodeInput{ BaseUserData: cloudinit.BaseUserData{ AdditionalFiles: files, NTP: scope.Config.Spec.NTP, @@ -540,13 +553,25 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin, }, JoinConfiguration: joinData, - }) + } + + var bootstrapJoinData []byte + switch scope.Config.Spec.Format { + case bootstrapv1.Ignition: + bootstrapJoinData, _, err = ignition.NewNode(&ignition.NodeInput{ + NodeInput: nodeInput, + Ignition: scope.Config.Spec.Ignition, + }) + default: + bootstrapJoinData, err = cloudinit.NewNode(nodeInput) + } + if err != nil { scope.Error(err, "Failed to create a worker join configuration") return ctrl.Result{}, err } - if err := r.storeBootstrapData(ctx, scope, cloudJoinData); err != nil { + if err := r.storeBootstrapData(ctx, scope, bootstrapJoinData); err != nil { scope.Error(err, "Failed to store bootstrap data") return ctrl.Result{}, err } @@ -610,7 +635,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S return ctrl.Result{}, err } - cloudJoinData, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{ + controlPlaneJoinInput := &cloudinit.ControlPlaneJoinInput{ JoinConfiguration: joinData, Certificates: certificates, BaseUserData: cloudinit.BaseUserData{ @@ -624,13 +649,25 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S KubeadmVerbosity: verbosityFlag, UseExperimentalRetry: scope.Config.Spec.UseExperimentalRetryJoin, }, - }) + } + + var bootstrapJoinData []byte + switch scope.Config.Spec.Format { + case bootstrapv1.Ignition: + bootstrapJoinData, _, err = ignition.NewJoinControlPlane(&ignition.ControlPlaneJoinInput{ + ControlPlaneJoinInput: controlPlaneJoinInput, + Ignition: scope.Config.Spec.Ignition, + }) + default: + bootstrapJoinData, err = cloudinit.NewJoinControlPlane(controlPlaneJoinInput) + } + if err != nil { scope.Error(err, "Failed to create a control plane join configuration") return ctrl.Result{}, err } - if err := r.storeBootstrapData(ctx, scope, cloudJoinData); err != nil { + if err := r.storeBootstrapData(ctx, scope, bootstrapJoinData); err != nil { scope.Error(err, "Failed to store bootstrap data") return ctrl.Result{}, err } @@ -891,7 +928,8 @@ func (r *KubeadmConfigReconciler) storeBootstrapData(ctx context.Context, scope }, }, Data: map[string][]byte{ - "value": data, + "value": data, + "format": []byte(scope.Config.Spec.Format), }, Type: clusterv1.ClusterSecretType, } diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go index de36de92e59f..d01a26fbc273 100644 --- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go +++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + ignition "github.com/flatcar-linux/ignition/config/v2_3" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/yaml" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" @@ -483,7 +485,7 @@ func TestReconcileIfJoinNodesAndControlPlaneIsReady(t *testing.T) { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} - var useCases = []struct { + useCases := []struct { name string machine *clusterv1.Machine configName string @@ -575,7 +577,7 @@ func TestReconcileIfJoinNodePoolsAndControlPlaneIsReady(t *testing.T) { conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} - var useCases = []struct { + useCases := []struct { name string machinePool *expv1.MachinePool configName string @@ -642,6 +644,127 @@ func TestReconcileIfJoinNodePoolsAndControlPlaneIsReady(t *testing.T) { } } +// Ensure bootstrap data is generated in the correct format based on the format specified in the +// KubeadmConfig resource. +func TestBootstrapDataFormat(t *testing.T) { + testcases := []struct { + name string + isWorker bool + format bootstrapv1.Format + clusterInitialized bool + }{ + { + name: "cloud-config init config", + format: bootstrapv1.CloudConfig, + }, + { + name: "Ignition init config", + format: bootstrapv1.Ignition, + }, + { + name: "Ignition control plane join config", + format: bootstrapv1.Ignition, + clusterInitialized: true, + }, + { + name: "Ignition worker join config", + isWorker: true, + format: bootstrapv1.Ignition, + clusterInitialized: true, + }, + { + name: "Empty format field", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + cluster := newCluster("cluster", metav1.NamespaceDefault) + cluster.Status.InfrastructureReady = true + cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443} + if tc.clusterInitialized { + conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition) + } + + var machine *clusterv1.Machine + var config *bootstrapv1.KubeadmConfig + var configName string + if tc.isWorker { + machine = newWorkerMachine(cluster) + configName = "worker-join-cfg" + config = newWorkerJoinKubeadmConfig(machine) + } else { + machine = newControlPlaneMachine(cluster, "machine") + configName = "cfg" + config = newControlPlaneInitKubeadmConfig(machine, configName) + } + config.Spec.Format = tc.format + + objects := []client.Object{ + cluster, + machine, + config, + } + objects = append(objects, createSecrets(t, cluster, config)...) + + myclient := fake.NewClientBuilder().WithObjects(objects...).Build() + + k := &KubeadmConfigReconciler{ + Client: myclient, + KubeadmInitLock: &myInitLocker{}, + remoteClientGetter: fakeremote.NewClusterClient, + } + request := ctrl.Request{ + NamespacedName: client.ObjectKey{ + Namespace: metav1.NamespaceDefault, + Name: configName, + }, + } + + // Reconcile the KubeadmConfig resource. + _, err := k.Reconcile(ctx, request) + g.Expect(err).NotTo(HaveOccurred()) + + // Verify the KubeadmConfig resource state is correct. + cfg, err := getKubeadmConfig(myclient, configName, metav1.NamespaceDefault) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cfg.Status.Ready).To(BeTrue()) + g.Expect(cfg.Status.DataSecretName).NotTo(BeNil()) + + // Read the secret containing the bootstrap data which was generated by the + // KubeadmConfig controller. + key := client.ObjectKey{ + Namespace: metav1.NamespaceDefault, + Name: *cfg.Status.DataSecretName, + } + secret := &corev1.Secret{} + err = myclient.Get(ctx, key, secret) + g.Expect(err).NotTo(HaveOccurred()) + + // Verify the format field of the bootstrap data secret is correct. + g.Expect(string(secret.Data["format"])).To(Equal(string(tc.format))) + + // Verify the bootstrap data value is in the correct format. + data := secret.Data["value"] + switch tc.format { + case bootstrapv1.CloudConfig, "": + // Verify the bootstrap data is valid YAML. + // TODO: Verify the YAML document is valid cloud-config? + var out interface{} + err = yaml.Unmarshal(data, &out) + g.Expect(err).NotTo(HaveOccurred()) + case bootstrapv1.Ignition: + // Verify the bootstrap data is valid Ignition. + _, reports, err := ignition.Parse(data) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(reports.IsFatal()).NotTo(BeTrue()) + } + }) + } +} + // during kubeadmconfig reconcile it is possible that bootstrap secret gets created // but kubeadmconfig is not patched, do not error if secret already exists. // ignore the alreadyexists error and update the status to ready. diff --git a/bootstrap/kubeadm/internal/ignition/OWNERS b/bootstrap/kubeadm/internal/ignition/OWNERS new file mode 100644 index 000000000000..1a54e5b08f81 --- /dev/null +++ b/bootstrap/kubeadm/internal/ignition/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - cluster-api-bootstrap-provider-kubeadm-ignition-maintainers + +reviewers: + - cluster-api-reviewers + - cluster-api-bootstrap-provider-kubeadm-ignition-reviewers diff --git a/bootstrap/kubeadm/internal/ignition/clc/clc.go b/bootstrap/kubeadm/internal/ignition/clc/clc.go new file mode 100644 index 000000000000..7c2c5d327d1b --- /dev/null +++ b/bootstrap/kubeadm/internal/ignition/clc/clc.go @@ -0,0 +1,373 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package clc generates bootstrap data in Ignition format using Container Linux Config Transpiler. +// +// CLC configuration defined in this package will run kubeadm command by creating a /etc/kubeadm.sh script +// file containing both pre and post kubeadm commands as well as the kubeadm command itself. +// +// /etc/kubeadm.sh script will be executed using kubeadm.service systemd unit, which will only happen +// if /etc/kubeadm.yml file exists, which ensures the script will run only once, as by the end of the +// script, /etc/kubeadm.yml is moved to /tmp filesystem, so it gets automatically cleaned up after a +// reboot. This is to align the implementation with cloud-init, which places kubeadm configuration in +// /tmp directory directly, which is not possible with Ignition. +// +// /etc/kubeadm.yml file contains generated kubeadm configuration and can be customized using pre kubeadm +// commands if needed, as a replacement for Jinja templates supported by cloud-init, for example +// using 'envsubst' or 'sed'. +// +// To override the behavior of kubeadm.service unit, one should create an override drop-in +// using AdditionalConfig field. Data from this field takes precedence and will be merged with +// configuration generated by the bootstrap provider, overriding already defined fields following the +// merge strategy described in https://coreos.github.io/ignition/operator-notes/#config-merging. +package clc + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "text/template" + + clct "github.com/flatcar-linux/container-linux-config-transpiler/config" + ignition "github.com/flatcar-linux/ignition/config/v2_3" + ignitionTypes "github.com/flatcar-linux/ignition/config/v2_3/types" + "github.com/pkg/errors" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit" +) + +const ( + clcTemplate = `--- +{{- if .Users }} +passwd: + users: + {{- range .Users }} + - name: {{ .Name }} + {{- with .Gecos }} + gecos: {{ . }} + {{- end }} + {{- if .Groups }} + groups: + {{- range Split .Groups ", " }} + - {{ . }} + {{- end }} + {{- end }} + {{- with .HomeDir }} + home_dir: {{ . }} + {{- end }} + {{- with .Shell }} + shell: {{ . }} + {{- end }} + {{- with .Passwd }} + password_hash: {{ . }} + {{- end }} + {{- with .PrimaryGroup }} + primary_group: {{ . }} + {{- end }} + {{- if .SSHAuthorizedKeys }} + ssh_authorized_keys: + {{- range .SSHAuthorizedKeys }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} +systemd: + units: + - name: kubeadm.service + enabled: true + contents: | + [Unit] + Description=kubeadm + # Run only once. After successful run, this file is moved to /tmp/. + ConditionPathExists=/etc/kubeadm.yml + [Service] + # To not restart the unit when it exits, as it is expected. + Type=oneshot + ExecStart=/etc/kubeadm.sh + [Install] + WantedBy=multi-user.target + {{- if .NTP }}{{ if .NTP.Enabled }} + - name: ntpd.service + enabled: true + {{- end }}{{- end }} + {{- range .Mounts }} + {{- $label := index . 0 }} + {{- $mountpoint := index . 1 }} + {{- $disk := index $.FilesystemDevicesByLabel $label }} + {{- $mountOptions := slice . 2 }} + - name: {{ $mountpoint | MountpointName }}.mount + enabled: true + contents: | + [Unit] + Description = Mount {{ $label }} + + [Mount] + What={{ $disk }} + Where={{ $mountpoint }} + Options={{ Join $mountOptions "," }} + + [Install] + WantedBy=multi-user.target + {{- end }} +storage: + {{- if .DiskSetup }}{{- if .DiskSetup.Partitions }} + disks: + {{- range .DiskSetup.Partitions }} + - device: {{ .Device }} + {{- with .Overwrite }} + wipe_table: {{ . }} + {{- end }} + {{- if .Layout }} + partitions: + - {} + {{- end }} + {{- end }} + {{- end }}{{- end }} + {{- if .DiskSetup }}{{- if .DiskSetup.Filesystems }} + filesystems: + {{- range .DiskSetup.Filesystems }} + - name: {{ .Label }} + mount: + device: {{ .Device }} + format: {{ .Filesystem }} + wipe_filesystem: {{ .Overwrite }} + label: {{ .Label }} + {{- if .ExtraOpts }} + options: + {{- range .ExtraOpts }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }}{{- end }} + files: + {{- range .Users }} + {{- if .Sudo }} + - path: /etc/sudoers.d/{{ .Name }} + mode: 0600 + contents: + inline: | + {{ .Name }} {{ .Sudo }} + {{- end }} + {{- end }} + {{- with .UsersWithPasswordAuth }} + - path: /etc/ssh/sshd_config + mode: 0600 + contents: + inline: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + + Match User {{ . }} + PasswordAuthentication yes + {{- end }} + {{- range .WriteFiles }} + - path: {{ .Path }} + # Owner + # + # If Encoding == gzip+base64 || Encoding == gzip + # compression: true + # + # If Encoding == gzip+base64 || Encoding == "base64" + # Put "!!binary" notation before the content to let YAML decoder treat data as + # base64 data. + # + {{ if ne .Permissions "" -}} + mode: {{ .Permissions }} + {{ end -}} + contents: + inline: | + {{ .Content | Indent 10 }} + {{- end }} + - path: /etc/kubeadm.sh + mode: 0700 + contents: + inline: | + #!/bin/bash + set -e + {{ range .PreKubeadmCommands }} + {{ . | Indent 10 }} + {{- end }} + + {{ .KubeadmCommand }} + mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete + mv /etc/kubeadm.yml /tmp/ + {{range .PostKubeadmCommands }} + {{ . | Indent 10 }} + {{- end }} + - path: /etc/kubeadm.yml + mode: 0600 + contents: + inline: | + --- + {{ .KubeadmConfig | Indent 10 }} + {{- if .NTP }}{{- if and .NTP.Enabled .NTP.Servers }} + - path: /etc/ntp.conf + mode: 0644 + contents: + inline: | + # Common pool + {{- range .NTP.Servers }} + server {{ . }} + {{- end }} + + # Warning: Using default NTP settings will leave your NTP + # server accessible to all hosts on the Internet. + + # If you want to deny all machines (including your own) + # from accessing the NTP server, uncomment: + #restrict default ignore + + # Default configuration: + # - Allow only time queries, at a limited rate, sending KoD when in excess. + # - Allow all local queries (IPv4, IPv6) + restrict default nomodify nopeer noquery notrap limited kod + restrict 127.0.0.1 + restrict [::1] + {{- end }}{{- end }} +` +) + +type render struct { + *cloudinit.BaseUserData + + KubeadmConfig string + UsersWithPasswordAuth string + FilesystemDevicesByLabel map[string]string +} + +func defaultTemplateFuncMap() template.FuncMap { + return template.FuncMap{ + "Indent": templateYAMLIndent, + "Split": strings.Split, + "Join": strings.Join, + "MountpointName": mountpointName, + } +} + +func mountpointName(name string) string { + return strings.TrimPrefix(strings.ReplaceAll(name, "/", "-"), "-") +} + +func templateYAMLIndent(i int, input string) string { + split := strings.Split(input, "\n") + ident := "\n" + strings.Repeat(" ", i) + return strings.Join(split, ident) +} + +func renderCLC(input *cloudinit.BaseUserData, kubeadmConfig string) ([]byte, error) { + t := template.Must(template.New("template").Funcs(defaultTemplateFuncMap()).Parse(clcTemplate)) + + usersWithPasswordAuth := []string{} + for _, user := range input.Users { + if user.LockPassword != nil && !*user.LockPassword { + usersWithPasswordAuth = append(usersWithPasswordAuth, user.Name) + } + } + + filesystemDevicesByLabel := map[string]string{} + if input.DiskSetup != nil { + for _, filesystem := range input.DiskSetup.Filesystems { + filesystemDevicesByLabel[filesystem.Label] = filesystem.Device + } + } + + data := render{ + BaseUserData: input, + KubeadmConfig: kubeadmConfig, + UsersWithPasswordAuth: strings.Join(usersWithPasswordAuth, ","), + FilesystemDevicesByLabel: filesystemDevicesByLabel, + } + + var out bytes.Buffer + if err := t.Execute(&out, data); err != nil { + return nil, errors.Wrapf(err, "failed to render template") + } + + return out.Bytes(), nil +} + +// Render renders the provided user data and CLC snippets into Ignition config. +func Render(input *cloudinit.BaseUserData, clc *bootstrapv1.ContainerLinuxConfig, kubeadmConfig string) ([]byte, string, error) { + if input == nil { + return nil, "", errors.New("empty base user data") + } + + clcBytes, err := renderCLC(input, kubeadmConfig) + if err != nil { + return nil, "", errors.Wrapf(err, "rendering CLC configuration") + } + + userData, warnings, err := buildIgnitionConfig(clcBytes, clc) + if err != nil { + return nil, "", errors.Wrapf(err, "building Ignition config") + } + + return userData, warnings, nil +} + +func buildIgnitionConfig(baseCLC []byte, clc *bootstrapv1.ContainerLinuxConfig) ([]byte, string, error) { + // We control baseCLC config, so treat it as strict. + ign, _, err := clcToIgnition(baseCLC, true) + if err != nil { + return nil, "", errors.Wrapf(err, "converting generated CLC to Ignition") + } + + var clcWarnings string + + if clc != nil && clc.AdditionalConfig != "" { + additionalIgn, warnings, err := clcToIgnition([]byte(clc.AdditionalConfig), clc.Strict) + if err != nil { + return nil, "", errors.Wrapf(err, "converting additional CLC to Ignition") + } + + clcWarnings = warnings + + ign = ignition.Append(ign, additionalIgn) + } + + userData, err := json.Marshal(&ign) + if err != nil { + return nil, "", errors.Wrapf(err, "marshaling generated Ignition config into JSON") + } + + return userData, clcWarnings, nil +} + +func clcToIgnition(data []byte, strict bool) (ignitionTypes.Config, string, error) { + clc, ast, reports := clct.Parse(data) + + if (len(reports.Entries) > 0 && strict) || reports.IsFatal() { + return ignitionTypes.Config{}, "", fmt.Errorf("error parsing Container Linux Config: %v", reports.String()) + } + + ign, report := clct.Convert(clc, "", ast) + if (len(report.Entries) > 0 && strict) || report.IsFatal() { + return ignitionTypes.Config{}, "", fmt.Errorf("error converting to Ignition: %v", report.String()) + } + + reports.Merge(report) + + return ign, reports.String(), nil +} diff --git a/bootstrap/kubeadm/internal/ignition/clc/clc_test.go b/bootstrap/kubeadm/internal/ignition/clc/clc_test.go new file mode 100644 index 000000000000..a87c9882f811 --- /dev/null +++ b/bootstrap/kubeadm/internal/ignition/clc/clc_test.go @@ -0,0 +1,418 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package clc_test tests clc package. +package clc_test + +import ( + "testing" + + ignition "github.com/flatcar-linux/ignition/config/v2_3" + "github.com/flatcar-linux/ignition/config/v2_3/types" + "github.com/google/go-cmp/cmp" + "k8s.io/utils/pointer" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition/clc" +) + +const ( + configWithWarning = `--- +storage: + files: + - path: /foo + contents: + inline: foo +` + + // Should generate an Ignition warning about the colon in the partition label. + configWithIgnitionWarning = `--- +storage: + disks: + - device: /dev/sda + partitions: + - label: foo:bar +` +) + +func TestRender(t *testing.T) { + t.Parallel() + + preKubeadmCommands := []string{ + "pre-command", + "another-pre-command", + // Test multi-line commands as well. + "cat < /etc/modules-load.d/containerd.conf\noverlay\nbr_netfilter\nEOF\n", + } + postKubeadmCommands := []string{ + "post-kubeadm-command", + "another-post-kubeamd-command", + // Test multi-line commands as well. + "cat < /etc/modules-load.d/containerd.conf\noverlay\nbr_netfilter\nEOF\n", + } + + tc := []struct { + desc string + input *cloudinit.BaseUserData + wantIgnition types.Config + }{ + { + desc: "renders valid Ignition JSON", + input: &cloudinit.BaseUserData{ + PreKubeadmCommands: preKubeadmCommands, + PostKubeadmCommands: postKubeadmCommands, + KubeadmCommand: "kubeadm join", + NTP: &bootstrapv1.NTP{ + Enabled: pointer.BoolPtr(true), + Servers: []string{ + "foo.bar", + "baz", + }, + }, + Users: []bootstrapv1.User{ + { + Name: "foo", + Gecos: pointer.StringPtr("Foo B. Bar"), + Groups: pointer.StringPtr("foo, bar"), + HomeDir: pointer.StringPtr("/home/foo"), + Shell: pointer.StringPtr("/bin/false"), + Passwd: pointer.StringPtr("$6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/"), + PrimaryGroup: pointer.StringPtr("foo"), + Sudo: pointer.StringPtr("ALL=(ALL) NOPASSWD:ALL"), + SSHAuthorizedKeys: []string{ + "foo", + "bar", + }, + }, + }, + DiskSetup: &bootstrapv1.DiskSetup{ + Partitions: []bootstrapv1.Partition{ + { + Device: "/dev/disk/azure/scsi1/lun0", + Layout: true, + Overwrite: pointer.BoolPtr(true), + TableType: pointer.StringPtr("gpt"), + }, + }, + Filesystems: []bootstrapv1.Filesystem{ + { + Device: "/dev/disk/azure/scsi1/lun0", + Filesystem: "ext4", + Label: "test_disk", + ExtraOpts: []string{"-F", "-E", "lazy_itable_init=1,lazy_journal_init=1"}, + Overwrite: pointer.BoolPtr(true), + }, + }, + }, + Mounts: []bootstrapv1.MountPoints{ + { + "test_disk", "/var/lib/testdir", "foo", + }, + }, + }, + wantIgnition: types.Config{ + Ignition: types.Ignition{ + Version: "2.3.0", + }, + Passwd: types.Passwd{ + Users: []types.PasswdUser{ + { + Gecos: "Foo B. Bar", + Groups: []types.Group{ + "foo", + "bar", + }, + HomeDir: "/home/foo", + Name: "foo", + PasswordHash: pointer.StringPtr("$6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/"), + PrimaryGroup: "foo", + SSHAuthorizedKeys: []types.SSHAuthorizedKey{ + "foo", + "bar", + }, + Shell: "/bin/false", + }, + }, + }, + Storage: types.Storage{ + Disks: []types.Disk{ + { + Device: "/dev/disk/azure/scsi1/lun0", + Partitions: []types.Partition{{}}, + WipeTable: true, + }, + }, + Files: []types.File{ + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/sudoers.d/foo", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,foo%20ALL%3D(ALL)%20NOPASSWD%3AALL%0A", + }, + Mode: pointer.IntPtr(384), + }, + }, + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/kubeadm.sh", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,%23!%2Fbin%2Fbash%0Aset%20-e%0A%0Apre-command%0Aanother-pre-command%0Acat%20%3C%3CEOF%20%3E%20%2Fetc%2Fmodules-load.d%2Fcontainerd.conf%0Aoverlay%0Abr_netfilter%0AEOF%0A%0A%0Akubeadm%20join%0Amkdir%20-p%20%2Frun%2Fcluster-api%20%26%26%20echo%20success%20%3E%20%2Frun%2Fcluster-api%2Fbootstrap-success.complete%0Amv%20%2Fetc%2Fkubeadm.yml%20%2Ftmp%2F%0A%0Apost-kubeadm-command%0Aanother-post-kubeamd-command%0Acat%20%3C%3CEOF%20%3E%20%2Fetc%2Fmodules-load.d%2Fcontainerd.conf%0Aoverlay%0Abr_netfilter%0AEOF%0A", + }, + Mode: pointer.IntPtr(448), + }, + }, + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/kubeadm.yml", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,---%0Afoo%0A", + }, + Mode: pointer.IntPtr(384), + }, + }, + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/ntp.conf", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,%23%20Common%20pool%0Aserver%20foo.bar%0Aserver%20baz%0A%0A%23%20Warning%3A%20Using%20default%20NTP%20settings%20will%20leave%20your%20NTP%0A%23%20server%20accessible%20to%20all%20hosts%20on%20the%20Internet.%0A%0A%23%20If%20you%20want%20to%20deny%20all%20machines%20(including%20your%20own)%0A%23%20from%20accessing%20the%20NTP%20server%2C%20uncomment%3A%0A%23restrict%20default%20ignore%0A%0A%23%20Default%20configuration%3A%0A%23%20-%20Allow%20only%20time%20queries%2C%20at%20a%20limited%20rate%2C%20sending%20KoD%20when%20in%20excess.%0A%23%20-%20Allow%20all%20local%20queries%20(IPv4%2C%20IPv6)%0Arestrict%20default%20nomodify%20nopeer%20noquery%20notrap%20limited%20kod%0Arestrict%20127.0.0.1%0Arestrict%20%5B%3A%3A1%5D%0A", + }, + Mode: pointer.IntPtr(420), + }, + }, + }, + Filesystems: []types.Filesystem{ + { + Mount: &types.Mount{ + Device: "/dev/disk/azure/scsi1/lun0", + Format: "ext4", + Label: pointer.StringPtr("test_disk"), + Options: []types.MountOption{ + "-F", + "-E", + "lazy_itable_init=1,lazy_journal_init=1", + }, + WipeFilesystem: true, + }, + Name: "test_disk", + }, + }, + }, + Systemd: types.Systemd{ + Units: []types.Unit{ + { + Contents: "[Unit]\nDescription=kubeadm\n# Run only once. After successful run, this file is moved to /tmp/.\nConditionPathExists=/etc/kubeadm.yml\n[Service]\n# To not restart the unit when it exits, as it is expected.\nType=oneshot\nExecStart=/etc/kubeadm.sh\n[Install]\nWantedBy=multi-user.target\n", + Enabled: pointer.BoolPtr(true), + Name: "kubeadm.service", + }, + { + Enabled: pointer.BoolPtr(true), + Name: "ntpd.service", + }, + { + Contents: "[Unit]\nDescription = Mount test_disk\n\n[Mount]\nWhat=/dev/disk/azure/scsi1/lun0\nWhere=/var/lib/testdir\nOptions=foo\n\n[Install]\nWantedBy=multi-user.target\n", + Enabled: pointer.BoolPtr(true), + Name: "var-lib-testdir.mount", + }, + }, + }, + }, + }, + { + desc: "multiple users with password auth", + input: &cloudinit.BaseUserData{ + PreKubeadmCommands: preKubeadmCommands, + PostKubeadmCommands: postKubeadmCommands, + KubeadmCommand: "kubeadm join", + Users: []bootstrapv1.User{ + { + Name: "foo", + LockPassword: pointer.BoolPtr(false), + }, + { + Name: "bar", + LockPassword: pointer.BoolPtr(false), + }, + }, + }, + wantIgnition: types.Config{ + Ignition: types.Ignition{ + Version: "2.3.0", + }, + Passwd: types.Passwd{ + Users: []types.PasswdUser{ + { + Name: "foo", + }, + { + Name: "bar", + }, + }, + }, + Storage: types.Storage{ + Files: []types.File{ + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/ssh/sshd_config", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0A%0AMatch%20User%20foo%2Cbar%0A%20%20PasswordAuthentication%20yes%0A", + }, + Mode: pointer.IntPtr(384), + }, + }, + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/kubeadm.sh", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,%23!%2Fbin%2Fbash%0Aset%20-e%0A%0Apre-command%0Aanother-pre-command%0Acat%20%3C%3CEOF%20%3E%20%2Fetc%2Fmodules-load.d%2Fcontainerd.conf%0Aoverlay%0Abr_netfilter%0AEOF%0A%0A%0Akubeadm%20join%0Amkdir%20-p%20%2Frun%2Fcluster-api%20%26%26%20echo%20success%20%3E%20%2Frun%2Fcluster-api%2Fbootstrap-success.complete%0Amv%20%2Fetc%2Fkubeadm.yml%20%2Ftmp%2F%0A%0Apost-kubeadm-command%0Aanother-post-kubeamd-command%0Acat%20%3C%3CEOF%20%3E%20%2Fetc%2Fmodules-load.d%2Fcontainerd.conf%0Aoverlay%0Abr_netfilter%0AEOF%0A", + }, + Mode: pointer.IntPtr(448), + }, + }, + { + Node: types.Node{ + Filesystem: "root", + Path: "/etc/kubeadm.yml", + }, + FileEmbedded1: types.FileEmbedded1{ + Contents: types.FileContents{ + Source: "data:,---%0Afoo%0A", + }, + Mode: pointer.IntPtr(384), + }, + }, + }, + }, + Systemd: types.Systemd{ + Units: []types.Unit{ + { + Contents: "[Unit]\nDescription=kubeadm\n# Run only once. After successful run, this file is moved to /tmp/.\nConditionPathExists=/etc/kubeadm.yml\n[Service]\n# To not restart the unit when it exits, as it is expected.\nType=oneshot\nExecStart=/etc/kubeadm.sh\n[Install]\nWantedBy=multi-user.target\n", + Enabled: pointer.BoolPtr(true), + Name: "kubeadm.service", + }, + }, + }, + }, + }, + } + + for _, tt := range tc { + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + + ignitionBytes, _, err := clc.Render(tt.input, &bootstrapv1.ContainerLinuxConfig{}, "foo") + if err != nil { + t.Fatalf("rendering: %v", err) + } + + ign, reports, err := ignition.Parse(ignitionBytes) + if err != nil { + t.Fatalf("Parsing generated Ignition: %v", err) + } + + if reports.IsFatal() { + t.Fatalf("Generated Ignition has fatal reports: %s", reports) + } + + if diff := cmp.Diff(tt.wantIgnition, ign); diff != "" { + t.Fatalf("Ignition mismatch (-want +got):\n%s", diff) + } + }) + } + + t.Run("validates input parameter", func(t *testing.T) { + t.Parallel() + + if _, _, err := clc.Render(nil, &bootstrapv1.ContainerLinuxConfig{}, "foo"); err == nil { + t.Fatal("expected error when passing empty input data") + } + }) + + t.Run("accepts empty clc parameter", func(t *testing.T) { + t.Parallel() + + if _, _, err := clc.Render(&cloudinit.BaseUserData{}, nil, "bar"); err != nil { + t.Fatalf("unexpected error while rendering: %v", err) + } + }) + + t.Run("treats warnings as errors in strict mode", func(t *testing.T) { + config := &bootstrapv1.ContainerLinuxConfig{ + Strict: true, + AdditionalConfig: configWithWarning, + } + + if _, _, err := clc.Render(&cloudinit.BaseUserData{}, config, "foo"); err == nil { + t.Fatalf("expected error") + } + }) + + t.Run("returns warnings", func(t *testing.T) { + config := &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: configWithWarning, + } + + data, warnings, err := clc.Render(&cloudinit.BaseUserData{}, config, "foo") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if warnings == "" { + t.Errorf("expected warnings to be not empty") + } + + if len(data) == 0 { + t.Errorf("expected data to be returned on config with warnings") + } + }) + + t.Run("returns Ignition warnings", func(t *testing.T) { + config := &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: configWithIgnitionWarning, + } + + data, warnings, err := clc.Render(&cloudinit.BaseUserData{}, config, "foo") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if warnings == "" { + t.Errorf("expected warnings to be not empty") + } + + if len(data) == 0 { + t.Errorf("expected data to be returned on config with warnings") + } + }) +} diff --git a/bootstrap/kubeadm/internal/ignition/ignition.go b/bootstrap/kubeadm/internal/ignition/ignition.go new file mode 100644 index 000000000000..ce9c231fee98 --- /dev/null +++ b/bootstrap/kubeadm/internal/ignition/ignition.go @@ -0,0 +1,115 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ignition aggregates all Ignition flavors into a single package to be consumed +// by the bootstrap provider by exposing an API similar to 'internal/cloudinit' package. +package ignition + +import ( + "fmt" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition/clc" +) + +const ( + joinSubcommand = "join" + initSubcommand = "init" + kubeadmCommandTemplate = "kubeadm %s --config /etc/kubeadm.yml %s" +) + +// NodeInput defines the context to generate a node user data. +type NodeInput struct { + *cloudinit.NodeInput + + Ignition *bootstrapv1.IgnitionSpec +} + +// ControlPlaneJoinInput defines context to generate controlplane instance user data for control plane node join. +type ControlPlaneJoinInput struct { + *cloudinit.ControlPlaneJoinInput + + Ignition *bootstrapv1.IgnitionSpec +} + +// ControlPlaneInput defines the context to generate a controlplane instance user data. +type ControlPlaneInput struct { + *cloudinit.ControlPlaneInput + + Ignition *bootstrapv1.IgnitionSpec +} + +// NewNode returns Ignition configuration for new worker node joining the cluster. +func NewNode(input *NodeInput) ([]byte, string, error) { + if input == nil { + return nil, "", fmt.Errorf("input can't be nil") + } + + if input.NodeInput == nil { + return nil, "", fmt.Errorf("node input can't be nil") + } + + input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) + input.KubeadmCommand = fmt.Sprintf(kubeadmCommandTemplate, joinSubcommand, input.KubeadmVerbosity) + + return render(&input.BaseUserData, input.Ignition, input.JoinConfiguration) +} + +// NewJoinControlPlane returns Ignition configuration for new controlplane node joining the cluster. +func NewJoinControlPlane(input *ControlPlaneJoinInput) ([]byte, string, error) { + if input == nil { + return nil, "", fmt.Errorf("input can't be nil") + } + + if input.ControlPlaneJoinInput == nil { + return nil, "", fmt.Errorf("controlplane join input can't be nil") + } + + input.WriteFiles = input.Certificates.AsFiles() + input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) + input.KubeadmCommand = fmt.Sprintf(kubeadmCommandTemplate, joinSubcommand, input.KubeadmVerbosity) + + return render(&input.BaseUserData, input.Ignition, input.JoinConfiguration) +} + +// NewInitControlPlane returns Ignition configuration for bootstrapping new cluster. +func NewInitControlPlane(input *ControlPlaneInput) ([]byte, string, error) { + if input == nil { + return nil, "", fmt.Errorf("input can't be nil") + } + + if input.ControlPlaneInput == nil { + return nil, "", fmt.Errorf("controlplane input can't be nil") + } + + input.WriteFiles = input.Certificates.AsFiles() + input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) + input.KubeadmCommand = fmt.Sprintf(kubeadmCommandTemplate, initSubcommand, input.KubeadmVerbosity) + + kubeadmConfig := fmt.Sprintf("%s\n---\n%s", input.ClusterConfiguration, input.InitConfiguration) + + return render(&input.BaseUserData, input.Ignition, kubeadmConfig) +} + +func render(input *cloudinit.BaseUserData, ignitionConfig *bootstrapv1.IgnitionSpec, kubeadmConfig string) ([]byte, string, error) { + clcConfig := &bootstrapv1.ContainerLinuxConfig{} + if ignitionConfig != nil && ignitionConfig.ContainerLinuxConfig != nil { + clcConfig = ignitionConfig.ContainerLinuxConfig + } + + return clc.Render(input, clcConfig, kubeadmConfig) +} diff --git a/bootstrap/kubeadm/internal/ignition/ignition_test.go b/bootstrap/kubeadm/internal/ignition/ignition_test.go new file mode 100644 index 000000000000..4db5b245a25d --- /dev/null +++ b/bootstrap/kubeadm/internal/ignition/ignition_test.go @@ -0,0 +1,386 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ignition_test + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + "testing" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition" +) + +const testString = "foo bar baz" + +func Test_NewNode(t *testing.T) { + t.Parallel() + + t.Run("returns error when", func(t *testing.T) { + t.Parallel() + + cases := map[string]*ignition.NodeInput{ + "nil input is given": nil, + "nil node input is given": {}, + } + + for name, input := range cases { + input := input + + t.Run(name, func(t *testing.T) { + t.Parallel() + + ignitionData, _, err := ignition.NewNode(input) + if err == nil { + t.Fatalf("Expected error") + } + + if ignitionData != nil { + t.Fatalf("Unexpected data returned %v", ignitionData) + } + }) + } + }) + + t.Run("returns JSON data without error", func(t *testing.T) { + t.Parallel() + + input := &ignition.NodeInput{ + NodeInput: &cloudinit.NodeInput{}, + Ignition: &bootstrapv1.IgnitionSpec{}, + } + + ignitionData, _, err := ignition.NewNode(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if ignitionData == nil { + t.Fatalf("Returned data is nil") + } + + decodedValue := map[string]interface{}{} + + if err := json.Unmarshal(ignitionData, &decodedValue); err != nil { + t.Fatalf("Decoding received Ignition data as JSON: %v", err) + } + }) + + t.Run("returns Ignition with user-specified snippet", func(t *testing.T) { + t.Parallel() + + input := &ignition.NodeInput{ + NodeInput: &cloudinit.NodeInput{}, + Ignition: &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: fmt.Sprintf(`storage: + files: + - path: /etc/foo + mode: 0644 + contents: + inline: | + %s +`, testString), + }, + }, + } + + ignitionData, _, err := ignition.NewNode(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Ignition stores content URL-encoded. + u := url.URL{Path: testString} + + if !strings.Contains(string(ignitionData), u.String()) { + t.Fatalf("Expected %q to be included in %q", testString, string(ignitionData)) + } + }) + + t.Run("returns warnings if any", func(t *testing.T) { + t.Parallel() + + input := &ignition.NodeInput{ + NodeInput: &cloudinit.NodeInput{}, + Ignition: &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: fmt.Sprintf(`storage: + files: + - path: /etc/foo + contents: + inline: | + %s +`, testString), + }, + }, + } + + ignitionData, warnings, err := ignition.NewNode(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if warnings == "" { + t.Fatalf("Expected warnings") + } + + if len(ignitionData) == 0 { + t.Fatalf("Data should be returned with warnings but no errors") + } + }) +} + +func Test_NewJoinControlPlane(t *testing.T) { + t.Parallel() + + t.Run("returns error when", func(t *testing.T) { + t.Parallel() + + cases := map[string]*ignition.ControlPlaneJoinInput{ + "nil input is given": nil, + "nil node input is given": {}, + } + + for name, input := range cases { + input := input + + t.Run(name, func(t *testing.T) { + t.Parallel() + + ignitionData, _, err := ignition.NewJoinControlPlane(input) + if err == nil { + t.Fatalf("Expected error") + } + + if ignitionData != nil { + t.Fatalf("Unexpected data returned %v", ignitionData) + } + }) + } + }) + + t.Run("returns JSON data without error", func(t *testing.T) { + t.Parallel() + + input := &ignition.ControlPlaneJoinInput{ + ControlPlaneJoinInput: &cloudinit.ControlPlaneJoinInput{}, + Ignition: &bootstrapv1.IgnitionSpec{}, + } + + ignitionData, _, err := ignition.NewJoinControlPlane(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if ignitionData == nil { + t.Fatalf("Returned data is nil") + } + + decodedValue := map[string]interface{}{} + + if err := json.Unmarshal(ignitionData, &decodedValue); err != nil { + t.Fatalf("Decoding received Ignition data as JSON: %v", err) + } + }) + + t.Run("returns Ignition with user specified snippet", func(t *testing.T) { + t.Parallel() + + input := &ignition.ControlPlaneJoinInput{ + ControlPlaneJoinInput: &cloudinit.ControlPlaneJoinInput{}, + Ignition: &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: fmt.Sprintf(`storage: + files: + - path: /etc/foo + mode: 0644 + contents: + inline: | + %s +`, testString), + }, + }, + } + + ignitionData, _, err := ignition.NewJoinControlPlane(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Ignition stores content URL-encoded. + u := url.URL{Path: testString} + + if !strings.Contains(string(ignitionData), u.String()) { + t.Fatalf("Expected %q to be included in %q", testString, string(ignitionData)) + } + }) + + t.Run("returns warnings if any", func(t *testing.T) { + t.Parallel() + + input := &ignition.ControlPlaneJoinInput{ + ControlPlaneJoinInput: &cloudinit.ControlPlaneJoinInput{}, + Ignition: &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: fmt.Sprintf(`storage: + files: + - path: /etc/foo + contents: + inline: | + %s +`, testString), + }, + }, + } + + ignitionData, warnings, err := ignition.NewJoinControlPlane(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if warnings == "" { + t.Fatalf("Expected to get some warnings") + } + + if len(ignitionData) == 0 { + t.Fatalf("Data should be returned with warnings but no errors") + } + }) +} + +func Test_NewInitControlPlane(t *testing.T) { + t.Parallel() + + t.Run("returns error when", func(t *testing.T) { + t.Parallel() + + cases := map[string]*ignition.ControlPlaneInput{ + "nil input is given": nil, + "nil node input is given": {}, + } + + for name, input := range cases { + input := input + + t.Run(name, func(t *testing.T) { + t.Parallel() + + ignitionData, _, err := ignition.NewInitControlPlane(input) + if err == nil { + t.Fatalf("Expected error") + } + + if ignitionData != nil { + t.Fatalf("Unexpected data returned %v", ignitionData) + } + }) + } + }) + + t.Run("returns without error", func(t *testing.T) { + t.Parallel() + + input := &ignition.ControlPlaneInput{ + ControlPlaneInput: &cloudinit.ControlPlaneInput{}, + } + + ignitionData, _, err := ignition.NewInitControlPlane(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if ignitionData == nil { + t.Fatalf("Returned data is nil") + } + + t.Run("valid_JSON_data", func(t *testing.T) { + decodedValue := map[string]interface{}{} + + if err := json.Unmarshal(ignitionData, &decodedValue); err != nil { + t.Fatalf("Decoding received Ignition data as JSON: %v", err) + } + }) + }) + + t.Run("returns Ignition with user-specified snippet", func(t *testing.T) { + t.Parallel() + + input := &ignition.ControlPlaneInput{ + ControlPlaneInput: &cloudinit.ControlPlaneInput{}, + Ignition: &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: fmt.Sprintf(`storage: + files: + - path: /etc/foo + mode: 0644 + contents: + inline: | + %s +`, testString), + }, + }, + } + + ignitionData, _, err := ignition.NewInitControlPlane(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // Ignition stores content URL-encoded. + u := url.URL{Path: testString} + + if !strings.Contains(string(ignitionData), u.String()) { + t.Fatalf("Expected %q to be included in %q", testString, string(ignitionData)) + } + }) + + t.Run("returns warnings if any", func(t *testing.T) { + t.Parallel() + + input := &ignition.ControlPlaneInput{ + ControlPlaneInput: &cloudinit.ControlPlaneInput{}, + Ignition: &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{ + AdditionalConfig: fmt.Sprintf(`storage: + files: + - path: /etc/foo + contents: + inline: | + %s +`, testString), + }, + }, + } + + ignitionData, warnings, err := ignition.NewInitControlPlane(input) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if warnings == "" { + t.Fatalf("Expected warnings") + } + + if len(ignitionData) == 0 { + t.Fatalf("Data should be returned with warnings but no errors") + } + }) +} diff --git a/controlplane/kubeadm/api/v1alpha3/conversion.go b/controlplane/kubeadm/api/v1alpha3/conversion.go index 0992f139f06f..54056351266f 100644 --- a/controlplane/kubeadm/api/v1alpha3/conversion.go +++ b/controlplane/kubeadm/api/v1alpha3/conversion.go @@ -55,6 +55,8 @@ func (src *KubeadmControlPlane) ConvertTo(destRaw conversion.Hub) error { dest.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.IgnorePreflightErrors = restored.Spec.KubeadmConfigSpec.InitConfiguration.NodeRegistration.IgnorePreflightErrors } + dest.Spec.KubeadmConfigSpec.Ignition = restored.Spec.KubeadmConfigSpec.Ignition + return nil } diff --git a/controlplane/kubeadm/api/v1alpha4/conversion.go b/controlplane/kubeadm/api/v1alpha4/conversion.go index 807c755fa514..84bd401606fc 100644 --- a/controlplane/kubeadm/api/v1alpha4/conversion.go +++ b/controlplane/kubeadm/api/v1alpha4/conversion.go @@ -20,18 +20,36 @@ import ( "sigs.k8s.io/controller-runtime/pkg/conversion" "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) func (src *KubeadmControlPlane) ConvertTo(destRaw conversion.Hub) error { dest := destRaw.(*v1beta1.KubeadmControlPlane) - return Convert_v1alpha4_KubeadmControlPlane_To_v1beta1_KubeadmControlPlane(src, dest, nil) + if err := Convert_v1alpha4_KubeadmControlPlane_To_v1beta1_KubeadmControlPlane(src, dest, nil); err != nil { + return err + } + + // Manually restore data. + restored := &v1beta1.KubeadmControlPlane{} + if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + return err + } + + dest.Spec.KubeadmConfigSpec.Ignition = restored.Spec.KubeadmConfigSpec.Ignition + + return nil } func (dest *KubeadmControlPlane) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*v1beta1.KubeadmControlPlane) - return Convert_v1beta1_KubeadmControlPlane_To_v1alpha4_KubeadmControlPlane(src, dest, nil) + if err := Convert_v1beta1_KubeadmControlPlane_To_v1alpha4_KubeadmControlPlane(src, dest, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + return utilconversion.MarshalData(src, dest) } func (src *KubeadmControlPlaneList) ConvertTo(destRaw conversion.Hub) error { @@ -49,13 +67,30 @@ func (dest *KubeadmControlPlaneList) ConvertFrom(srcRaw conversion.Hub) error { func (src *KubeadmControlPlaneTemplate) ConvertTo(destRaw conversion.Hub) error { dest := destRaw.(*v1beta1.KubeadmControlPlaneTemplate) - return Convert_v1alpha4_KubeadmControlPlaneTemplate_To_v1beta1_KubeadmControlPlaneTemplate(src, dest, nil) + if err := Convert_v1alpha4_KubeadmControlPlaneTemplate_To_v1beta1_KubeadmControlPlaneTemplate(src, dest, nil); err != nil { + return err + } + + // Manually restore data. + restored := &v1beta1.KubeadmControlPlaneTemplate{} + if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + return err + } + + dest.Spec.Template.Spec.KubeadmConfigSpec.Ignition = restored.Spec.Template.Spec.KubeadmConfigSpec.Ignition + + return nil } func (dest *KubeadmControlPlaneTemplate) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*v1beta1.KubeadmControlPlaneTemplate) - return Convert_v1beta1_KubeadmControlPlaneTemplate_To_v1alpha4_KubeadmControlPlaneTemplate(src, dest, nil) + if err := Convert_v1beta1_KubeadmControlPlaneTemplate_To_v1alpha4_KubeadmControlPlaneTemplate(src, dest, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + return utilconversion.MarshalData(src, dest) } func (src *KubeadmControlPlaneTemplateList) ConvertTo(destRaw conversion.Hub) error { diff --git a/controlplane/kubeadm/api/v1alpha4/conversion_test.go b/controlplane/kubeadm/api/v1alpha4/conversion_test.go index 67ad7f9520c1..8b2dc55143b7 100644 --- a/controlplane/kubeadm/api/v1alpha4/conversion_test.go +++ b/controlplane/kubeadm/api/v1alpha4/conversion_test.go @@ -23,12 +23,18 @@ import ( "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/upstreamv1beta1" "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) +const ( + fakeID = "abcdef" + fakeSecret = "abcdef0123456789" +) + func TestFuzzyConversion(t *testing.T) { t.Run("for KubeadmControlPlane", utilconversion.FuzzTestFunc(utilconversion.FuzzTestFuncInput{ Hub: &v1beta1.KubeadmControlPlane{}, @@ -58,16 +64,18 @@ func fuzzFuncs(_ runtimeserializer.CodecFactory) []interface{} { kubeadmBootstrapTokenStringFuzzer, cabpkBootstrapTokenStringFuzzer, dnsFuzzer, + kubeadmBootstrapTokenStringFuzzerV1Alpha4, } } func kubeadmBootstrapTokenStringFuzzer(in *upstreamv1beta1.BootstrapTokenString, c fuzz.Continue) { - in.ID = "abcdef" - in.Secret = "abcdef0123456789" + in.ID = fakeID + in.Secret = fakeSecret } + func cabpkBootstrapTokenStringFuzzer(in *cabpkv1.BootstrapTokenString, c fuzz.Continue) { - in.ID = "abcdef" - in.Secret = "abcdef0123456789" + in.ID = fakeID + in.Secret = fakeSecret } func dnsFuzzer(obj *upstreamv1beta1.DNS, c fuzz.Continue) { @@ -76,3 +84,8 @@ func dnsFuzzer(obj *upstreamv1beta1.DNS, c fuzz.Continue) { // DNS.Type does not exists in v1alpha4, so setting it to empty string in order to avoid v1alpha3 --> v1alpha4 --> v1alpha3 round trip errors. obj.Type = "" } + +func kubeadmBootstrapTokenStringFuzzerV1Alpha4(in *v1alpha4.BootstrapTokenString, c fuzz.Continue) { + in.ID = fakeID + in.Secret = fakeSecret +} diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 4cd43497a590..1715388bf8ea 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -2927,7 +2927,26 @@ spec: data enum: - cloud-config + - ignition type: string + ignition: + description: Ignition contains Ignition specific configuration. + properties: + containerLinuxConfig: + description: ContainerLinuxConfig contains CLC specific configuration. + properties: + additionalConfig: + description: "AdditionalConfig contains additional configuration + to be merged with the Ignition configuration generated + by the bootstrapper controller. More info: https://coreos.github.io/ignition/operator-notes/#config-merging + \n The data format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/" + type: string + strict: + description: Strict controls if AdditionalConfig should + be strictly parsed. If so, warnings are treated as errors. + type: boolean + type: object + type: object initConfiguration: description: InitConfiguration along with ClusterConfiguration are the configurations necessary for the init command diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index 493bcd0f4cfb..f2370c63915b 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -1707,7 +1707,29 @@ spec: bootstrap data enum: - cloud-config + - ignition type: string + ignition: + description: Ignition contains Ignition specific configuration. + properties: + containerLinuxConfig: + description: ContainerLinuxConfig contains CLC specific + configuration. + properties: + additionalConfig: + description: "AdditionalConfig contains additional + configuration to be merged with the Ignition + configuration generated by the bootstrapper + controller. More info: https://coreos.github.io/ignition/operator-notes/#config-merging + \n The data format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/" + type: string + strict: + description: Strict controls if AdditionalConfig + should be strictly parsed. If so, warnings are + treated as errors. + type: boolean + type: object + type: object initConfiguration: description: InitConfiguration along with ClusterConfiguration are the configurations necessary for the init command diff --git a/go.mod b/go.mod index d83fd64f67c5..6dd0e2883930 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,15 @@ require ( github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 github.com/evanphx/json-patch/v5 v5.6.0 github.com/fatih/color v1.13.0 + github.com/flatcar-linux/container-linux-config-transpiler v0.9.2 + github.com/flatcar-linux/ignition v0.36.1 github.com/go-logr/logr v1.2.0 github.com/gobuffalo/flect v0.2.4 github.com/google/go-cmp v0.5.6 github.com/google/go-github/v33 v33.0.0 github.com/google/gofuzz v1.2.0 github.com/gosuri/uitable v0.0.4 + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/pkg/errors v0.9.1 @@ -53,11 +56,14 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect + github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect @@ -86,7 +92,6 @@ require ( github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect @@ -111,12 +116,14 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/stretchr/testify v1.7.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect + go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect diff --git a/go.sum b/go.sum index 0dad7082857a..8bc2341b52bb 100644 --- a/go.sum +++ b/go.sum @@ -73,11 +73,16 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= +github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -86,6 +91,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -129,11 +135,14 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -180,6 +189,10 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flatcar-linux/container-linux-config-transpiler v0.9.2 h1:EZKQ25jmhNfj+VAvdhPLLc4jmnSnRwFrI4x4dlPWXqE= +github.com/flatcar-linux/container-linux-config-transpiler v0.9.2/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= +github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJkbTwcYhY= +github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= @@ -196,6 +209,7 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -221,6 +235,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I= github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= +github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -368,6 +383,7 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -484,12 +500,14 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -533,19 +551,24 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= +github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -592,6 +615,10 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -653,6 +680,9 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= +go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -710,6 +740,7 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -863,6 +894,7 @@ golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/test/e2e/Makefile b/test/e2e/Makefile index db70a5d1c694..18531a77e87c 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -84,6 +84,7 @@ cluster-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in --load_restrictor none > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-kcp-scale-in.yaml $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-ipv6 --load_restrictor none > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-ipv6.yaml $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-topology --load_restrictor none > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-topology.yaml + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-ignition --load_restrictor none > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-ignition.yaml ## -------------------------------------- ## Testing diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index 1577804ba4a1..ba7b3e77ad46 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -190,6 +190,7 @@ providers: - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-kcp-scale-in.yaml" - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-ipv6.yaml" - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-topology.yaml" + - sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template-ignition.yaml" - sourcePath: "../data/infrastructure-docker/v1beta1/clusterclass-quick-start.yaml" - sourcePath: "../data/shared/v1beta1/metadata.yaml" diff --git a/test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/ignition.yaml b/test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/ignition.yaml new file mode 100644 index 000000000000..11f07f4ce964 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/ignition.yaml @@ -0,0 +1,26 @@ +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + format: ignition +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + format: ignition + ignition: + containerLinuxConfig: + additionalConfig: | + storage: + files: + - path: /opt/foo + filesystem: root + contents: + inline: Howdy! + mode: 0644 diff --git a/test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/kustomization.yaml b/test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/kustomization.yaml new file mode 100644 index 000000000000..f1c273da9b6c --- /dev/null +++ b/test/e2e/data/infrastructure-docker/v1beta1/cluster-template-ignition/kustomization.yaml @@ -0,0 +1,7 @@ +bases: + - ../bases/cluster-with-kcp.yaml + - ../bases/md.yaml + - ../bases/crs.yaml + +patchesStrategicMerge: + - ./ignition.yaml diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index ff2c971ecac2..793154f8c407 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -48,3 +48,16 @@ var _ = Describe("When following the Cluster API quick-start with ClusterClass", } }) }) + +var _ = Describe("When following the Cluster API quick-start with Ignition", func() { + QuickStartSpec(ctx, func() QuickStartSpecInput { + return QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: pointer.String("ignition"), + } + }) +}) diff --git a/test/go.sum b/test/go.sum index d73e2eabecf6..632bf8cf80c5 100644 --- a/test/go.sum +++ b/test/go.sum @@ -96,11 +96,14 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= @@ -112,6 +115,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -254,11 +258,14 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -330,6 +337,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flatcar-linux/container-linux-config-transpiler v0.9.2/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= +github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -380,6 +389,7 @@ github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -707,12 +717,14 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -766,6 +778,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= @@ -774,6 +787,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= +github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -784,8 +799,10 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -847,12 +864,15 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -922,6 +942,8 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -986,6 +1008,7 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1168,6 +1191,7 @@ golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 03d9170d5efd4dcbda45dddbe13c49e1a7ba2b4f Mon Sep 17 00:00:00 2001 From: Mateusz Gozdek Date: Tue, 9 Mar 2021 11:05:50 +0100 Subject: [PATCH 2/8] bootstrap/kubeadm: add validation webhook for KubeadmConfigTemplate As users usually do not create KubeadmConfig directly, but rather create KubeadmConfigTemplate and now KubeadmConfig has more validation rules, it's good if users sees those errors directly. Signed-off-by: Mateusz Gozdek Signed-off-by: Johanan Liebermann --- .../v1beta1/kubeadmconfigtemplate_webhook.go | 35 ++++++++++++ .../kubeadmconfigtemplate_webhook_test.go | 54 +++++++++++++++++++ .../kubeadm/config/webhook/manifests.yaml | 22 ++++++++ 3 files changed, 111 insertions(+) create mode 100644 bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook.go index 9adfecf443f9..4459f7fa0bdf 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook.go @@ -17,7 +17,11 @@ limitations under the License. package v1beta1 import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) func (r *KubeadmConfigTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error { @@ -25,3 +29,34 @@ func (r *KubeadmConfigTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error For(r). Complete() } + +// +kubebuilder:webhook:verbs=create;update,path=/validate-bootstrap-cluster-x-k8s-io-v1beta1-kubeadmconfigtemplate,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=bootstrap.cluster.x-k8s.io,resources=kubeadmconfigtemplates,versions=v1beta1,name=validation.kubeadmconfigtemplate.bootstrap.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 + +var _ webhook.Validator = &KubeadmConfigTemplate{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *KubeadmConfigTemplate) ValidateCreate() error { + return r.Spec.validate(r.Name) +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *KubeadmConfigTemplate) ValidateUpdate(old runtime.Object) error { + return r.Spec.validate(r.Name) +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *KubeadmConfigTemplate) ValidateDelete() error { + return nil +} + +func (r *KubeadmConfigTemplateSpec) validate(name string) error { + var allErrs field.ErrorList + + allErrs = append(allErrs, r.Template.Spec.Validate()...) + + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid(GroupVersion.WithKind("KubeadmConfigTemplate").GroupKind(), name, allErrs) +} diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go new file mode 100644 index 000000000000..2295eaa155d7 --- /dev/null +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1_test + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +func TestKubeadmConfigTemplateValidation(t *testing.T) { + cases := map[string]struct { + in *bootstrapv1.KubeadmConfigTemplate + }{ + "valid configuration": { + in: &bootstrapv1.KubeadmConfigTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: bootstrapv1.KubeadmConfigTemplateSpec{ + Template: bootstrapv1.KubeadmConfigTemplateResource{ + Spec: bootstrapv1.KubeadmConfigSpec{}, + }, + }, + }, + }, + } + + for name, tt := range cases { + t.Run(name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(tt.in.ValidateCreate()).To(Succeed()) + g.Expect(tt.in.ValidateUpdate(nil)).To(Succeed()) + }) + } +} diff --git a/bootstrap/kubeadm/config/webhook/manifests.yaml b/bootstrap/kubeadm/config/webhook/manifests.yaml index bea0ea22b558..7f05595d297d 100644 --- a/bootstrap/kubeadm/config/webhook/manifests.yaml +++ b/bootstrap/kubeadm/config/webhook/manifests.yaml @@ -27,3 +27,25 @@ webhooks: resources: - kubeadmconfigs sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-kubeadmconfigtemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.kubeadmconfigtemplate.bootstrap.cluster.x-k8s.io + rules: + - apiGroups: + - bootstrap.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - kubeadmconfigtemplates + sideEffects: None From e6ffd8ef3a53600b4806893f23217d8accaf37d1 Mon Sep 17 00:00:00 2001 From: Mateusz Gozdek Date: Tue, 9 Mar 2021 11:06:58 +0100 Subject: [PATCH 3/8] controlplane/kubeadm/api/v1beta1: validate KubeadmConfigSpec KubeadmConfigSpec has now more validation rules, so we should trigger them, as KubeadmConfigSpec is embedded in KubeadmControlPlane object. Signed-off-by: Mateusz Gozdek Signed-off-by: Johanan Liebermann --- .../v1beta1/kubeadm_control_plane_webhook.go | 4 ++ .../kubeadm_control_plane_webhook_test.go | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index 74e106eacc03..7e68b62e4978 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -92,6 +92,7 @@ func (in *KubeadmControlPlane) ValidateCreate() error { spec := in.Spec allErrs := validateKubeadmControlPlaneSpec(spec, in.Namespace, field.NewPath("spec")) allErrs = append(allErrs, validateEtcd(&spec, nil)...) + allErrs = append(allErrs, in.Spec.KubeadmConfigSpec.Validate()...) if len(allErrs) > 0 { return apierrors.NewInvalid(GroupVersion.WithKind("KubeadmControlPlane").GroupKind(), in.Name, allErrs) } @@ -113,6 +114,7 @@ const ( controllerManager = "controllerManager" scheduler = "scheduler" ntp = "ntp" + ignition = "ignition" ) // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. @@ -138,6 +140,7 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { {spec, kubeadmConfigSpec, "verbosity"}, {spec, kubeadmConfigSpec, users}, {spec, kubeadmConfigSpec, ntp, "*"}, + {spec, kubeadmConfigSpec, ignition, "*"}, {spec, "machineTemplate", "metadata", "*"}, {spec, "machineTemplate", "infrastructureRef", "apiVersion"}, {spec, "machineTemplate", "infrastructureRef", "name"}, @@ -192,6 +195,7 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { allErrs = append(allErrs, in.validateVersion(prev.Spec.Version)...) allErrs = append(allErrs, validateEtcd(&in.Spec, &prev.Spec)...) allErrs = append(allErrs, in.validateCoreDNSVersion(prev)...) + allErrs = append(allErrs, in.Spec.KubeadmConfigSpec.Validate()...) if len(allErrs) > 0 { return apierrors.NewInvalid(GroupVersion.WithKind("KubeadmControlPlane").GroupKind(), in.Name, allErrs) diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go index 2d4624a1e0ee..74d3ce1d09f4 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go @@ -127,6 +127,13 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { invalidVersion2 := valid.DeepCopy() invalidVersion2.Spec.Version = "1.16.6" + invalidIgnitionConfiguration := valid.DeepCopy() + invalidIgnitionConfiguration.Spec.KubeadmConfigSpec.Ignition = &bootstrapv1.IgnitionSpec{} + + validIgnitionConfiguration := valid.DeepCopy() + validIgnitionConfiguration.Spec.KubeadmConfigSpec.Format = bootstrapv1.Ignition + validIgnitionConfiguration.Spec.KubeadmConfigSpec.Ignition = &bootstrapv1.IgnitionSpec{} + tests := []struct { name string expectErr bool @@ -182,6 +189,16 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { expectErr: true, kcp: invalidMaxSurge, }, + { + name: "should return error when Ignition configuration is invalid", + expectErr: true, + kcp: invalidIgnitionConfiguration, + }, + { + name: "should succeed when Ignition configuration is valid", + expectErr: false, + kcp: validIgnitionConfiguration, + }, } for _, tt := range tests { @@ -541,6 +558,18 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { disableNTPServers := before.DeepCopy() disableNTPServers.Spec.KubeadmConfigSpec.NTP.Enabled = pointer.BoolPtr(false) + invalidIgnitionConfiguration := before.DeepCopy() + invalidIgnitionConfiguration.Spec.KubeadmConfigSpec.Ignition = &bootstrapv1.IgnitionSpec{} + + validIgnitionConfigurationBefore := before.DeepCopy() + validIgnitionConfigurationBefore.Spec.KubeadmConfigSpec.Format = bootstrapv1.Ignition + validIgnitionConfigurationBefore.Spec.KubeadmConfigSpec.Ignition = &bootstrapv1.IgnitionSpec{ + ContainerLinuxConfig: &bootstrapv1.ContainerLinuxConfig{}, + } + + validIgnitionConfigurationAfter := validIgnitionConfigurationBefore.DeepCopy() + validIgnitionConfigurationAfter.Spec.KubeadmConfigSpec.Ignition.ContainerLinuxConfig.AdditionalConfig = "foo: bar" + tests := []struct { name string expectErr bool @@ -829,6 +858,18 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { before: before, kcp: disableNTPServers, }, + { + name: "should return error when Ignition configuration is invalid", + expectErr: true, + before: invalidIgnitionConfiguration, + kcp: invalidIgnitionConfiguration, + }, + { + name: "should succeed when Ignition configuration is modified", + expectErr: false, + before: validIgnitionConfigurationBefore, + kcp: validIgnitionConfigurationAfter, + }, } for _, tt := range tests { From edbb149174cf0ad3b2c0d6ee5a5d5d1d1add70a1 Mon Sep 17 00:00:00 2001 From: Johanan Liebermann Date: Thu, 2 Sep 2021 18:13:48 +0300 Subject: [PATCH 4/8] Add Ignition provisioning format for CAPD Allow provisioning CAPD machines using Ignition as an alternative to cloud-init. So far, cloud-init has been hardcoded as the only provisioning format supported by CAPD. - Add a package called "provisioning" which contains the interfaces and common logic for all supported provisioning formats. - Move the existing cloud-init code under the new package. - Add a new provisioning implementation for Ignition. Co-authored-by: Suraj Deshmukh Signed-off-by: Johanan Liebermann --- test/go.mod | 6 +- test/go.sum | 4 + .../docker/exp/internal/docker/nodepool.go | 22 ++- .../controllers/dockermachine_controller.go | 22 ++- .../docker/internal/docker/machine.go | 21 ++- .../{ => provisioning}/cloudinit/doc.go | 0 .../cloudinit/kindadapter.go | 14 +- .../cloudinit/kindadapter_test.go | 7 +- .../{ => provisioning}/cloudinit/runcmd.go | 47 +----- .../cloudinit/runcmd_test.go | 20 +-- .../{ => provisioning}/cloudinit/unknown.go | 6 +- .../cloudinit/unknown_test.go | 0 .../cloudinit/writefiles.go | 14 +- .../cloudinit/writefiles_test.go | 12 +- .../docker/internal/provisioning/commands.go | 61 ++++++++ .../internal/provisioning/ignition/OWNERS | 8 + .../internal/provisioning/ignition/doc.go | 23 +++ .../provisioning/ignition/kindadapter.go | 139 ++++++++++++++++++ .../provisioning/ignition/kindadapter_test.go | 95 ++++++++++++ 19 files changed, 429 insertions(+), 92 deletions(-) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/doc.go (100%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/kindadapter.go (86%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/kindadapter_test.go (99%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/runcmd.go (67%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/runcmd_test.go (76%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/unknown.go (88%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/unknown_test.go (100%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/writefiles.go (88%) rename test/infrastructure/docker/internal/{ => provisioning}/cloudinit/writefiles_test.go (94%) create mode 100644 test/infrastructure/docker/internal/provisioning/commands.go create mode 100644 test/infrastructure/docker/internal/provisioning/ignition/OWNERS create mode 100644 test/infrastructure/docker/internal/provisioning/ignition/doc.go create mode 100644 test/infrastructure/docker/internal/provisioning/ignition/kindadapter.go create mode 100644 test/infrastructure/docker/internal/provisioning/ignition/kindadapter_test.go diff --git a/test/go.mod b/test/go.mod index ae01b7db9d16..0545ed7ed6b3 100644 --- a/test/go.mod +++ b/test/go.mod @@ -6,13 +6,16 @@ replace sigs.k8s.io/cluster-api => ../ require ( github.com/blang/semver v3.5.1+incompatible + github.com/containerd/containerd v1.5.2 // indirect github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 + github.com/flatcar-linux/ignition v0.36.1 github.com/go-logr/logr v1.2.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 + github.com/vincent-petithory/dataurl v1.0.0 k8s.io/api v0.23.0-alpha.4 k8s.io/apiextensions-apiserver v0.23.0-alpha.4 k8s.io/apimachinery v0.23.0-alpha.4 @@ -34,9 +37,10 @@ require ( github.com/alessio/shellescape v1.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/containerd/containerd v1.5.2 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coredns/corefile-migration v1.0.14 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect diff --git a/test/go.sum b/test/go.sum index 632bf8cf80c5..8ae305991071 100644 --- a/test/go.sum +++ b/test/go.sum @@ -260,11 +260,13 @@ github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmeka github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -338,6 +340,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar-linux/container-linux-config-transpiler v0.9.2/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= +github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJkbTwcYhY= github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -864,6 +867,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= diff --git a/test/infrastructure/docker/exp/internal/docker/nodepool.go b/test/infrastructure/docker/exp/internal/docker/nodepool.go index 613cfcdcdc40..1c054556081e 100644 --- a/test/infrastructure/docker/exp/internal/docker/nodepool.go +++ b/test/infrastructure/docker/exp/internal/docker/nodepool.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/kind/pkg/cluster/constants" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api/test/infrastructure/docker/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker" @@ -255,15 +256,15 @@ func (np *NodePool) reconcileMachine(ctx context.Context, machine *docker.Machin return ctrl.Result{}, errors.Wrapf(err, "failed to pre-load images into the docker machine with instance name %s", machine.Name()) } - bootstrapData, err := getBootstrapData(ctx, np.client, np.machinePool) + bootstrapData, format, err := getBootstrapData(ctx, np.client, np.machinePool) if err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to get bootstrap data for instance named %s", machine.Name()) } timeoutctx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() - // Run the bootstrap script. Simulates cloud-init. - if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData); err != nil { + // Run the bootstrap script. Simulates cloud-init/Ignition. + if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData, format); err != nil { return ctrl.Result{}, errors.Wrapf(err, "failed to exec DockerMachinePool instance bootstrap for instance named %s", machine.Name()) } // Check for bootstrap success @@ -321,21 +322,26 @@ func (np *NodePool) reconcileMachine(ctx context.Context, machine *docker.Machin } // getBootstrapData fetches the bootstrap data for the machine pool. -func getBootstrapData(ctx context.Context, c client.Client, machinePool *clusterv1exp.MachinePool) (string, error) { +func getBootstrapData(ctx context.Context, c client.Client, machinePool *clusterv1exp.MachinePool) (string, bootstrapv1.Format, error) { if machinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil { - return "", errors.New("error retrieving bootstrap data: linked MachinePool's bootstrap.dataSecretName is nil") + return "", "", errors.New("error retrieving bootstrap data: linked MachinePool's bootstrap.dataSecretName is nil") } s := &corev1.Secret{} key := client.ObjectKey{Namespace: machinePool.GetNamespace(), Name: *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName} if err := c.Get(ctx, key, s); err != nil { - return "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for DockerMachinePool instance %s/%s", machinePool.GetNamespace(), machinePool.GetName()) + return "", "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for DockerMachinePool instance %s/%s", machinePool.GetNamespace(), machinePool.GetName()) } value, ok := s.Data["value"] if !ok { - return "", errors.New("error retrieving bootstrap data: secret value key is missing") + return "", "", errors.New("error retrieving bootstrap data: secret value key is missing") } - return base64.StdEncoding.EncodeToString(value), nil + format := s.Data["format"] + if string(format) == "" { + format = []byte(bootstrapv1.CloudConfig) + } + + return base64.StdEncoding.EncodeToString(value), bootstrapv1.Format(format), nil } diff --git a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go index 77a4305ab54c..0ac462a7ac31 100644 --- a/test/infrastructure/docker/internal/controllers/dockermachine_controller.go +++ b/test/infrastructure/docker/internal/controllers/dockermachine_controller.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/kind/pkg/cluster/constants" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker" "sigs.k8s.io/cluster-api/util" @@ -271,7 +272,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * // if the machine isn't bootstrapped, only then run bootstrap scripts if !dockerMachine.Spec.Bootstrapped { - bootstrapData, err := r.getBootstrapData(ctx, machine) + bootstrapData, format, err := r.getBootstrapData(ctx, machine) if err != nil { log.Error(err, "failed to get bootstrap data") return ctrl.Result{}, err @@ -279,8 +280,8 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster * timeoutctx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() - // Run the bootstrap script. Simulates cloud-init. - if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData); err != nil { + // Run the bootstrap script. Simulates cloud-init/Ignition. + if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData, format); err != nil { conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap") return ctrl.Result{}, errors.Wrap(err, "failed to exec DockerMachine bootstrap") } @@ -414,23 +415,28 @@ func (r *DockerMachineReconciler) DockerClusterToDockerMachines(o client.Object) return result } -func (r *DockerMachineReconciler) getBootstrapData(ctx context.Context, machine *clusterv1.Machine) (string, error) { +func (r *DockerMachineReconciler) getBootstrapData(ctx context.Context, machine *clusterv1.Machine) (string, bootstrapv1.Format, error) { if machine.Spec.Bootstrap.DataSecretName == nil { - return "", errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil") + return "", "", errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil") } s := &corev1.Secret{} key := client.ObjectKey{Namespace: machine.GetNamespace(), Name: *machine.Spec.Bootstrap.DataSecretName} if err := r.Client.Get(ctx, key, s); err != nil { - return "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for DockerMachine %s/%s", machine.GetNamespace(), machine.GetName()) + return "", "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for DockerMachine %s/%s", machine.GetNamespace(), machine.GetName()) } value, ok := s.Data["value"] if !ok { - return "", errors.New("error retrieving bootstrap data: secret value key is missing") + return "", "", errors.New("error retrieving bootstrap data: secret value key is missing") } - return base64.StdEncoding.EncodeToString(value), nil + format := s.Data["format"] + if string(format) == "" { + format = []byte(bootstrapv1.CloudConfig) + } + + return base64.StdEncoding.EncodeToString(value), bootstrapv1.Format(format), nil } // setMachineAddress gets the address from the container corresponding to a docker node and sets it on the Machine object. diff --git a/test/infrastructure/docker/internal/docker/machine.go b/test/infrastructure/docker/internal/docker/machine.go index 5a2ea4f071f3..48e43901800f 100644 --- a/test/infrastructure/docker/internal/docker/machine.go +++ b/test/infrastructure/docker/internal/docker/machine.go @@ -34,10 +34,13 @@ import ( "sigs.k8s.io/kind/pkg/cluster/constants" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/test/infrastructure/container" infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" - "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/cloudinit" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker/types" + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/cloudinit" + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/ignition" clusterapicontainer "sigs.k8s.io/cluster-api/util/container" ) @@ -317,7 +320,7 @@ func (m *Machine) PreloadLoadImages(ctx context.Context, images []string) error } // ExecBootstrap runs bootstrap on a node, this is generally `kubeadm `. -func (m *Machine) ExecBootstrap(ctx context.Context, data string) error { +func (m *Machine) ExecBootstrap(ctx context.Context, data string, format bootstrapv1.Format) error { log := ctrl.LoggerFrom(ctx) if m.container == nil { @@ -329,9 +332,19 @@ func (m *Machine) ExecBootstrap(ctx context.Context, data string) error { return errors.Wrap(err, "failed to decode machine's bootstrap data") } - commands, err := cloudinit.Commands(cloudConfig) + var commands []provisioning.Cmd + + switch format { + case bootstrapv1.CloudConfig: + commands, err = cloudinit.RawCloudInitToProvisioningCommands(cloudConfig) + case bootstrapv1.Ignition: + commands, err = ignition.RawIgnitionToProvisioningCommands(cloudConfig) + default: + return fmt.Errorf("unknown provisioning format %q", format) + } + if err != nil { - log.Info("cloud config failed to parse", "bootstrap data", data) + log.Info("provisioning code failed to parse", "bootstrap data", data) return errors.Wrap(err, "failed to join a control plane node with kubeadm") } diff --git a/test/infrastructure/docker/internal/cloudinit/doc.go b/test/infrastructure/docker/internal/provisioning/cloudinit/doc.go similarity index 100% rename from test/infrastructure/docker/internal/cloudinit/doc.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/doc.go diff --git a/test/infrastructure/docker/internal/cloudinit/kindadapter.go b/test/infrastructure/docker/internal/provisioning/cloudinit/kindadapter.go similarity index 86% rename from test/infrastructure/docker/internal/cloudinit/kindadapter.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/kindadapter.go index 215b67056cef..d19daf751994 100644 --- a/test/infrastructure/docker/internal/cloudinit/kindadapter.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/kindadapter.go @@ -24,6 +24,8 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" ) const ( @@ -48,23 +50,23 @@ func (a *actionFactory) action(name string) action { type action interface { Unmarshal(userData []byte) error - Commands() ([]Cmd, error) + Commands() ([]provisioning.Cmd, error) } -// Commands converts a cloudconfig to a list of commands to run in sequence on the node. -func Commands(cloudConfig []byte) ([]Cmd, error) { +// RawCloudInitToProvisioningCommands converts a cloudconfig to a list of commands to run in sequence on the node. +func RawCloudInitToProvisioningCommands(config []byte) ([]provisioning.Cmd, error) { // validate cloudConfigScript is a valid yaml, as required by the cloud config specification - if err := yaml.Unmarshal(cloudConfig, &map[string]interface{}{}); err != nil { + if err := yaml.Unmarshal(config, &map[string]interface{}{}); err != nil { return nil, errors.Wrapf(err, "cloud-config is not valid yaml") } // parse the cloud config yaml into a slice of cloud config actions. - actions, err := getActions(cloudConfig) + actions, err := getActions(config) if err != nil { return nil, err } - commands := []Cmd{} + commands := []provisioning.Cmd{} for _, action := range actions { cmds, err := action.Commands() if err != nil { diff --git a/test/infrastructure/docker/internal/cloudinit/kindadapter_test.go b/test/infrastructure/docker/internal/provisioning/cloudinit/kindadapter_test.go similarity index 99% rename from test/infrastructure/docker/internal/cloudinit/kindadapter_test.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/kindadapter_test.go index fa204e76ae9b..ae64a49f27fa 100644 --- a/test/infrastructure/docker/internal/cloudinit/kindadapter_test.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/kindadapter_test.go @@ -20,6 +20,8 @@ import ( "testing" . "github.com/onsi/gomega" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" ) func TestRealUseCase(t *testing.T) { @@ -104,7 +106,7 @@ write_files: permissions: '0640' `) - expectedCmds := []Cmd{ + expectedCmds := []provisioning.Cmd{ // ca {Cmd: "mkdir", Args: []string{"-p", "/etc/kubernetes/pki"}}, {Cmd: "/bin/sh", Args: []string{"-c", "cat > /etc/kubernetes/pki/ca.crt /dev/stdin"}}, @@ -139,7 +141,8 @@ write_files: {Cmd: "chmod", Args: []string{"0640", "/run/kubeadm/kubeadm.yaml"}}, } - commands, err := Commands(cloudData) + commands, err := RawCloudInitToProvisioningCommands(cloudData) + g.Expect(err).NotTo(HaveOccurred()) g.Expect(commands).To(HaveLen(len(expectedCmds))) diff --git a/test/infrastructure/docker/internal/cloudinit/runcmd.go b/test/infrastructure/docker/internal/provisioning/cloudinit/runcmd.go similarity index 67% rename from test/infrastructure/docker/internal/cloudinit/runcmd.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/runcmd.go index b05e2075c3ac..9b01270851f4 100644 --- a/test/infrastructure/docker/internal/cloudinit/runcmd.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/runcmd.go @@ -17,52 +17,17 @@ limitations under the License. package cloudinit import ( - "encoding/json" "strings" "github.com/pkg/errors" "sigs.k8s.io/yaml" -) - -// Cmd defines a shell command. -type Cmd struct { - Cmd string - Args []string - Stdin string -} - -// UnmarshalJSON a runcmd command -// It can be either a list or a string. -// If the item is a list, the head of the list is the command and the tail are the args. -// If the item is a string, the whole command will be wrapped in `/bin/sh -c`. -func (c *Cmd) UnmarshalJSON(data []byte) error { - // First, try to decode the input as a list - var s1 []string - if err := json.Unmarshal(data, &s1); err != nil { - if _, ok := err.(*json.UnmarshalTypeError); !ok { - return errors.WithStack(err) - } - } else { - c.Cmd = s1[0] - c.Args = s1[1:] - return nil - } - // If it's not a list, it must be a string - var s2 string - if err := json.Unmarshal(data, &s2); err != nil { - return errors.WithStack(err) - } - - c.Cmd = "/bin/sh" - c.Args = []string{"-c", s2} - - return nil -} + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" +) // runCmd defines parameters of a shell command that is equivalent to an action found in the cloud init rundcmd module. type runCmd struct { - Cmds []Cmd `json:"runcmd,"` + Cmds []provisioning.Cmd `json:"runcmd,"` } func newRunCmdAction() action { @@ -78,8 +43,8 @@ func (a *runCmd) Unmarshal(userData []byte) error { } // Commands returns a list of commands to run on the node. -func (a *runCmd) Commands() ([]Cmd, error) { - cmds := make([]Cmd, 0) +func (a *runCmd) Commands() ([]provisioning.Cmd, error) { + cmds := make([]provisioning.Cmd, 0) for _, c := range a.Cmds { // kubeadm in docker requires to ignore some errors, and this requires to modify the cmd generate by CABPK by default... c = hackKubeadmIgnoreErrors(c) @@ -88,7 +53,7 @@ func (a *runCmd) Commands() ([]Cmd, error) { return cmds, nil } -func hackKubeadmIgnoreErrors(c Cmd) Cmd { +func hackKubeadmIgnoreErrors(c provisioning.Cmd) provisioning.Cmd { // case kubeadm commands are defined as a string if c.Cmd == "/bin/sh" && len(c.Args) >= 2 { if c.Args[0] == "-c" { diff --git a/test/infrastructure/docker/internal/cloudinit/runcmd_test.go b/test/infrastructure/docker/internal/provisioning/cloudinit/runcmd_test.go similarity index 76% rename from test/infrastructure/docker/internal/cloudinit/runcmd_test.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/runcmd_test.go index 0de69b08141f..1afb666a09e4 100644 --- a/test/infrastructure/docker/internal/cloudinit/runcmd_test.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/runcmd_test.go @@ -20,6 +20,8 @@ import ( "testing" . "github.com/onsi/gomega" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" ) func TestRunCmdUnmarshal(t *testing.T) { @@ -34,10 +36,10 @@ runcmd: g.Expect(err).NotTo(HaveOccurred()) g.Expect(r.Cmds).To(HaveLen(2)) - expected0 := Cmd{Cmd: "ls", Args: []string{"-l", "/"}} + expected0 := provisioning.Cmd{Cmd: "ls", Args: []string{"-l", "/"}} g.Expect(r.Cmds[0]).To(Equal(expected0)) - expected1 := Cmd{Cmd: "/bin/sh", Args: []string{"-c", "ls -l /"}} + expected1 := provisioning.Cmd{Cmd: "/bin/sh", Args: []string{"-c", "ls -l /"}} g.Expect(r.Cmds[1]).To(Equal(expected1)) } @@ -45,17 +47,17 @@ func TestRunCmdRun(t *testing.T) { var useCases = []struct { name string r runCmd - expectedCmds []Cmd + expectedCmds []provisioning.Cmd }{ { name: "two command pass", r: runCmd{ - Cmds: []Cmd{ + Cmds: []provisioning.Cmd{ {Cmd: "foo", Args: []string{"bar"}}, {Cmd: "baz", Args: []string{"bbb"}}, }, }, - expectedCmds: []Cmd{ + expectedCmds: []provisioning.Cmd{ {Cmd: "foo", Args: []string{"bar"}}, {Cmd: "baz", Args: []string{"bbb"}}, }, @@ -63,11 +65,11 @@ func TestRunCmdRun(t *testing.T) { { name: "hack kubeadm ingore errors", r: runCmd{ - Cmds: []Cmd{ + Cmds: []provisioning.Cmd{ {Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --config /run/kubeadm/kubeadm.yaml"}}, }, }, - expectedCmds: []Cmd{ + expectedCmds: []provisioning.Cmd{ {Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config /run/kubeadm/kubeadm.yaml"}}, }, }, @@ -98,11 +100,11 @@ runcmd: r.Cmds[0] = hackKubeadmIgnoreErrors(r.Cmds[0]) - expected0 := Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml"}} + expected0 := provisioning.Cmd{Cmd: "/bin/sh", Args: []string{"-c", "kubeadm init --ignore-preflight-errors=all --config=/run/kubeadm/kubeadm.yaml"}} g.Expect(r.Cmds[0]).To(Equal(expected0)) r.Cmds[1] = hackKubeadmIgnoreErrors(r.Cmds[1]) - expected1 := Cmd{Cmd: "kubeadm", Args: []string{"join", "--ignore-preflight-errors=all", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml"}} + expected1 := provisioning.Cmd{Cmd: "kubeadm", Args: []string{"join", "--ignore-preflight-errors=all", "--config=/run/kubeadm/kubeadm-controlplane-join-config.yaml"}} g.Expect(r.Cmds[1]).To(Equal(expected1)) } diff --git a/test/infrastructure/docker/internal/cloudinit/unknown.go b/test/infrastructure/docker/internal/provisioning/cloudinit/unknown.go similarity index 88% rename from test/infrastructure/docker/internal/cloudinit/unknown.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/unknown.go index 37c3b004beb9..cb5af5ff4a25 100644 --- a/test/infrastructure/docker/internal/cloudinit/unknown.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/unknown.go @@ -20,6 +20,8 @@ import ( "encoding/json" "github.com/pkg/errors" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" ) type unknown struct { @@ -54,6 +56,6 @@ func (u *unknown) Unmarshal(data []byte) error { return nil } -func (u *unknown) Commands() ([]Cmd, error) { - return []Cmd{}, nil +func (u *unknown) Commands() ([]provisioning.Cmd, error) { + return []provisioning.Cmd{}, nil } diff --git a/test/infrastructure/docker/internal/cloudinit/unknown_test.go b/test/infrastructure/docker/internal/provisioning/cloudinit/unknown_test.go similarity index 100% rename from test/infrastructure/docker/internal/cloudinit/unknown_test.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/unknown_test.go diff --git a/test/infrastructure/docker/internal/cloudinit/writefiles.go b/test/infrastructure/docker/internal/provisioning/cloudinit/writefiles.go similarity index 88% rename from test/infrastructure/docker/internal/cloudinit/writefiles.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/writefiles.go index 46871046442a..c7fdb6aa398e 100644 --- a/test/infrastructure/docker/internal/cloudinit/writefiles.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/writefiles.go @@ -27,6 +27,8 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" ) const ( @@ -69,8 +71,8 @@ func (a *writeFilesAction) Unmarshal(userData []byte) error { // Commands return a list of commands to run on the node. // Each command defines the parameters of a shell command necessary to generate a file replicating the cloud-init write_files module. -func (a *writeFilesAction) Commands() ([]Cmd, error) { - commands := make([]Cmd, 0) +func (a *writeFilesAction) Commands() ([]provisioning.Cmd, error) { + commands := make([]provisioning.Cmd, 0) for _, f := range a.Files { // Fix attributes and apply defaults path := fixPath(f.Path) // NB. the real cloud init module for writes files converts path into absolute paths; this is not possible here... @@ -87,7 +89,7 @@ func (a *writeFilesAction) Commands() ([]Cmd, error) { // Make the directory so cat + redirection will work directory := filepath.Dir(path) - commands = append(commands, Cmd{Cmd: "mkdir", Args: []string{"-p", directory}}) + commands = append(commands, provisioning.Cmd{Cmd: "mkdir", Args: []string{"-p", directory}}) redirects := ">" if f.Append { @@ -95,16 +97,16 @@ func (a *writeFilesAction) Commands() ([]Cmd, error) { } // generate a command that will create a file with the expected contents. - commands = append(commands, Cmd{Cmd: "/bin/sh", Args: []string{"-c", fmt.Sprintf("cat %s %s /dev/stdin", redirects, path)}, Stdin: content}) + commands = append(commands, provisioning.Cmd{Cmd: "/bin/sh", Args: []string{"-c", fmt.Sprintf("cat %s %s /dev/stdin", redirects, path)}, Stdin: content}) // if permissions are different than default ownership, add a command to modify the permissions. if permissions != "0644" { - commands = append(commands, Cmd{Cmd: "chmod", Args: []string{permissions, path}}) + commands = append(commands, provisioning.Cmd{Cmd: "chmod", Args: []string{permissions, path}}) } // if ownership is different than default ownership, add a command to modify file ownerhsip. if owner != "root:root" { - commands = append(commands, Cmd{Cmd: "chown", Args: []string{owner, path}}) + commands = append(commands, provisioning.Cmd{Cmd: "chown", Args: []string{owner, path}}) } } return commands, nil diff --git a/test/infrastructure/docker/internal/cloudinit/writefiles_test.go b/test/infrastructure/docker/internal/provisioning/cloudinit/writefiles_test.go similarity index 94% rename from test/infrastructure/docker/internal/cloudinit/writefiles_test.go rename to test/infrastructure/docker/internal/provisioning/cloudinit/writefiles_test.go index 275cb4968e8b..e3e7fd3c2d91 100644 --- a/test/infrastructure/docker/internal/cloudinit/writefiles_test.go +++ b/test/infrastructure/docker/internal/provisioning/cloudinit/writefiles_test.go @@ -22,13 +22,15 @@ import ( "testing" . "github.com/onsi/gomega" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" ) func TestWriteFiles(t *testing.T) { var useCases = []struct { name string w writeFilesAction - expectedCmds []Cmd + expectedCmds []provisioning.Cmd }{ { name: "two files pass", @@ -38,7 +40,7 @@ func TestWriteFiles(t *testing.T) { {Path: "baz", Content: "qux"}, }, }, - expectedCmds: []Cmd{ + expectedCmds: []provisioning.Cmd{ {Cmd: "mkdir", Args: []string{"-p", "."}}, {Cmd: "/bin/sh", Args: []string{"-c", "cat > foo /dev/stdin"}, Stdin: "bar"}, {Cmd: "mkdir", Args: []string{"-p", "."}}, @@ -52,7 +54,7 @@ func TestWriteFiles(t *testing.T) { {Path: "foo", Content: "bar", Owner: "baz:baz"}, }, }, - expectedCmds: []Cmd{ + expectedCmds: []provisioning.Cmd{ {Cmd: "mkdir", Args: []string{"-p", "."}}, {Cmd: "/bin/sh", Args: []string{"-c", "cat > foo /dev/stdin"}, Stdin: "bar"}, {Cmd: "chown", Args: []string{"baz:baz", "foo"}}, @@ -65,7 +67,7 @@ func TestWriteFiles(t *testing.T) { {Path: "foo", Content: "bar", Permissions: "755"}, }, }, - expectedCmds: []Cmd{ + expectedCmds: []provisioning.Cmd{ {Cmd: "mkdir", Args: []string{"-p", "."}}, {Cmd: "/bin/sh", Args: []string{"-c", "cat > foo /dev/stdin"}, Stdin: "bar"}, {Cmd: "chmod", Args: []string{"755", "foo"}}, @@ -78,7 +80,7 @@ func TestWriteFiles(t *testing.T) { {Path: "foo", Content: "bar", Append: true}, }, }, - expectedCmds: []Cmd{ + expectedCmds: []provisioning.Cmd{ {Cmd: "mkdir", Args: []string{"-p", "."}}, {Cmd: "/bin/sh", Args: []string{"-c", "cat >> foo /dev/stdin"}, Stdin: "bar"}, }, diff --git a/test/infrastructure/docker/internal/provisioning/commands.go b/test/infrastructure/docker/internal/provisioning/commands.go new file mode 100644 index 000000000000..5b402ebcd5ce --- /dev/null +++ b/test/infrastructure/docker/internal/provisioning/commands.go @@ -0,0 +1,61 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package provisioning deals with various machine initialization methods viz. cloud-init, Ignition, +// etc. +package provisioning + +import ( + "encoding/json" + + "github.com/pkg/errors" +) + +// Cmd defines a shell command. +type Cmd struct { + Cmd string + Args []string + Stdin string +} + +// UnmarshalJSON a runcmd command +// It can be either a list or a string. +// If the item is a list, the head of the list is the command and the tail are the args. +// If the item is a string, the whole command will be wrapped in `/bin/sh -c`. +func (c *Cmd) UnmarshalJSON(data []byte) error { + // First, try to decode the input as a list + var s1 []string + if err := json.Unmarshal(data, &s1); err != nil { + if _, ok := err.(*json.UnmarshalTypeError); !ok { + return errors.WithStack(err) + } + } else { + c.Cmd = s1[0] + c.Args = s1[1:] + return nil + } + + // If it's not a list, it must be a string + var s2 string + if err := json.Unmarshal(data, &s2); err != nil { + return errors.WithStack(err) + } + + c.Cmd = "/bin/sh" + c.Args = []string{"-c", s2} + + return nil +} diff --git a/test/infrastructure/docker/internal/provisioning/ignition/OWNERS b/test/infrastructure/docker/internal/provisioning/ignition/OWNERS new file mode 100644 index 000000000000..1a54e5b08f81 --- /dev/null +++ b/test/infrastructure/docker/internal/provisioning/ignition/OWNERS @@ -0,0 +1,8 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - cluster-api-bootstrap-provider-kubeadm-ignition-maintainers + +reviewers: + - cluster-api-reviewers + - cluster-api-bootstrap-provider-kubeadm-ignition-reviewers diff --git a/test/infrastructure/docker/internal/provisioning/ignition/doc.go b/test/infrastructure/docker/internal/provisioning/ignition/doc.go new file mode 100644 index 000000000000..b4f997e4853f --- /dev/null +++ b/test/infrastructure/docker/internal/provisioning/ignition/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package ignition defines an Ignition adapter for kind nodes. + +The adapter supports a limited set of Ignition features necessary for testing CABPK. Additionally, +for the sake of simplicity, the adapter is designed to work with existing kind node images. +*/ +package ignition diff --git a/test/infrastructure/docker/internal/provisioning/ignition/kindadapter.go b/test/infrastructure/docker/internal/provisioning/ignition/kindadapter.go new file mode 100644 index 000000000000..cc5b4c91daf1 --- /dev/null +++ b/test/infrastructure/docker/internal/provisioning/ignition/kindadapter.go @@ -0,0 +1,139 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ignition + +import ( + "encoding/json" + "fmt" + "net/url" + "path/filepath" + "strconv" + "strings" + + ignitionTypes "github.com/flatcar-linux/ignition/config/v2_3/types" + "github.com/pkg/errors" + "github.com/vincent-petithory/dataurl" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" +) + +// RawIgnitionToProvisioningCommands converts an Ignition YAML document to a slice of commands. +func RawIgnitionToProvisioningCommands(config []byte) ([]provisioning.Cmd, error) { + // Ensure Ignition is a valid YAML document. + if err := yaml.Unmarshal(config, &map[string]interface{}{}); err != nil { + return nil, errors.Wrapf(err, "invalid YAML") + } + + // Parse the Ignition YAML into a slice of Ignition actions. + actions, err := getActions(config) + if err != nil { + return nil, err + } + + return actions, nil +} + +// getActions parses the cloud config YAML into a slice of actions to run. +// Parsing manually is required because the order of the cloud config's actions must be maintained. +func getActions(userData []byte) ([]provisioning.Cmd, error) { + var commands []provisioning.Cmd + + var ignition ignitionTypes.Config + if err := json.Unmarshal(userData, &ignition); err != nil { + return nil, fmt.Errorf("unmarshalling Ignition JSON: %w", err) + } + + // Generate commands for files. + for _, f := range ignition.Storage.Files { + raw := strings.TrimSpace(f.Contents.Source) + contents, err := decodeFileContents(raw) + if err != nil { + return nil, fmt.Errorf("decoding file contents: %w", err) + } + + mode := strconv.FormatInt(int64(*f.Mode), 8) + if len(mode) == 3 { + // Sticky bit isn't specified - pad with a zero. + mode = "0" + mode + } + + if f.Path == "/etc/kubeadm.sh" { + contents = hackKubeadmIgnoreErrors(contents) + } + + commands = append(commands, []provisioning.Cmd{ + // Idempotently create the directory. + {Cmd: "mkdir", Args: []string{"-p", filepath.Dir(f.Path)}}, + // Write the file. + {Cmd: "/bin/sh", Args: []string{"-c", fmt.Sprintf("cat > %s /dev/stdin", f.Path)}, Stdin: contents}, + // Set file permissions. + {Cmd: "chmod", Args: []string{mode, f.Path}}, + }...) + } + + for _, u := range ignition.Systemd.Units { + contents := strings.TrimSpace(u.Contents) + path := filepath.Join("/etc/systemd/system", u.Name) + + commands = append(commands, []provisioning.Cmd{ + {Cmd: "/bin/sh", Args: []string{"-c", fmt.Sprintf("cat > %s /dev/stdin", path)}, Stdin: contents}, + {Cmd: "systemctl", Args: []string{"daemon-reload"}}, + }...) + + if u.Enable || (u.Enabled != nil && *u.Enabled) { + commands = append(commands, provisioning.Cmd{Cmd: "systemctl", Args: []string{"enable", "--now", u.Name}}) + } + } + + return commands, nil +} + +// Add `--ignore-preflight-errors=all` to `kubeadm init` and `kubeadm join`. +func hackKubeadmIgnoreErrors(s string) string { + lines := strings.Split(s, "\n") + + for idx, line := range lines { + if !(strings.Contains(line, "kubeadm init") || strings.Contains(line, "kubeadm join")) { + continue + } + + lines[idx] = line + " --ignore-preflight-errors=all" + } + + return strings.Join(lines, "\n") +} + +// decodeFileContents accepts a string representing the contents of a file encoded in Ignition +// format and returns a decoded version of the string. +func decodeFileContents(s string) (string, error) { + u, err := url.Parse(s) + if err != nil { + return "", err + } + + if u.Scheme != "data" { + return s, nil + } + + rendered, err := dataurl.DecodeString(s) + if err != nil { + return "", err + } + + return string(rendered.Data), nil +} diff --git a/test/infrastructure/docker/internal/provisioning/ignition/kindadapter_test.go b/test/infrastructure/docker/internal/provisioning/ignition/kindadapter_test.go new file mode 100644 index 000000000000..20afc225af6e --- /dev/null +++ b/test/infrastructure/docker/internal/provisioning/ignition/kindadapter_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ignition + +import ( + "testing" + + . "github.com/onsi/gomega" + + "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" +) + +func TestRealUseCase(t *testing.T) { + g := NewWithT(t) + + cloudData := []byte(`{ + "storage": { + "files": [ + { + "filesystem": "root", + "path": "/etc/kubernetes/pki/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIC6jCCAdKgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl%0Acm5ldGVzMB4XDTIxMDkxNDExNTYyMVoXDTMxMDkxMjEyMDEyMVowFTETMBEGA1UE%0AAxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKpd%0AhNm74fN6MxrAlLkuRGVJIdrUP9D0JDm84yE71d8g22Bt%2BVrMXuaHthUWMsF0F%2FHF%0AaIo4zb7Rj8lgC5xnifrSaBFtd%2BdTRXe6kQHCzIvFhKpnOT3Q7NHnAzOXrkt8tOE0%0AnChh9wY%2BSQyTUZd5v5Mghsdi3vSzoQSLtrCD%2BNyMx600BWO8ME%2Bv4mqvnriT8xYs%0Atci45R4nzDauyJCawKQVvQl5Prl34hexptYIVbtVAt0Qm8Zqh2MTmKlIvnpr7kpl%0A7l%2FKvaRYZEgw58ge1vz91bOCuqDunRUQDtybJDayRUPyBhZHv6XP8QU%2FC2YR0GIE%0Ae8%2Fkszd1BvoT4likKz0CAwEAAaNFMEMwDgYDVR0PAQH%2FBAQDAgKkMBIGA1UdEwEB%0A%2FwQIMAYBAf8CAQAwHQYDVR0OBBYEFBGrJ3PFFKipZVwK7te%2BHE2z2fRjMA0GCSqG%0ASIb3DQEBCwUAA4IBAQA6SMrz10EoIhf2f%2BNOKp60TXhPnyFjd5YicPAo2b3zHAkJ%0AL05M1dQHTN626ue%2BuBQrWQ0MzZv9NNlIkhRtuf%2B6qmMi2Rq2UJWSavH5bm%2B2nxLl%0Amh1iaK%2BG8%2FmUxSWsTC%2BsexXMNNJAEM%2BwOQhcYd04ia%2BrmjuO7rI0gS8JCkW4%2Bh4B%0A0374kzTJrCZ%2BMhkyJqa0jVEW5fDUS02FwKP2Jf7nSWOP1T9e82tzwgIu6cHioU55%0AQg0raZj7neSdigRArT9488stSmwompwzpP6pwizcA2NLsjQx%2BbHT6x09VQrLuRiD%0AhVvcFSj2HhoXFTl2CJ1eYVzRpJhsvzjxZi6%2F8eIa%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 416 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/pki/ca.key", + "contents": { + "source": "data:,-----BEGIN%20RSA%20PRIVATE%20KEY-----%0AMIIEogIBAAKCAQEAql2E2bvh83ozGsCUuS5EZUkh2tQ%2F0PQkObzjITvV3yDbYG35%0AWsxe5oe2FRYywXQX8cVoijjNvtGPyWALnGeJ%2BtJoEW1351NFd7qRAcLMi8WEqmc5%0APdDs0ecDM5euS3y04TScKGH3Bj5JDJNRl3m%2FkyCGx2Le9LOhBIu2sIP43IzHrTQF%0AY7wwT6%2Fiaq%2BeuJPzFiy1yLjlHifMNq7IkJrApBW9CXk%2BuXfiF7Gm1ghVu1UC3RCb%0AxmqHYxOYqUi%2BemvuSmXuX8q9pFhkSDDnyB7W%2FP3Vs4K6oO6dFRAO3JskNrJFQ%2FIG%0AFke%2Fpc%2FxBT8LZhHQYgR7z%2BSzN3UG%2BhPiWKQrPQIDAQABAoIBAFGSvc3TnHkMhfPF%0ASnDwqmclAUTaZEQU4lOTEd4T3HAeN2yQu9iyCq6vRIwMOPlQMTbeoxOr5zf697Ig%0Afu7A1Nx4asQNemAVCyos9sm1EGPMi51cF5h1tS88QdguRJJ4f9NlcXAUmEcxA6E1%0A2NeCwCweYuqNeNwKNosKqssSJdLT%2Fbm0kiXKE2h4chxz7oqcn1bjtC%2BIHOQIIZ77%0Ahiof3qgiSUJ6E9GOoHk9Y%2FoXTrTmHT0WvoODc53rGT42T8UydzHH6nQIIxz1Mt%2F9%0AoRWwUay2cNf3WsOc1K6OM3qxHNVz2Z7JMf23BgSXl4Illo1cFl2b0vc0e18bouSc%0ArYVsaIECgYEA0XO137fFSp3Aq6PAHucSOCdA9xXoWRnmPoU0sQfplg4Mmjf6%2FIny%0A1rphhMt4d2Aa0p5LInX8WkhkhyvUauFOHBQYxtonjWJ90NcrcP3tRI4Vt4H6XTgF%0AGb51Xgc3vb%2FtGZvPMdyE8oVkj8x7U%2FXls3%2B5wk2q2AJBDBVXRxyQPVUCgYEA0DoP%0AuIe9pofnwLTO17V7g6AniOTnF6Yai2I0Ly3Lbc74pTXodTFcG2NVtAJxQMYGpQri%0A4PfqagzTXLq1ccEa%2BpvcNlZlv7t4vu46KAVo44xwHdCc%2F7LLfS0B23oDXg540ZH9%0ACNZz5g%2BRbZuZQZyYjVIjK2%2BQmrrXF6P3GT5H9kkCgYAw%2BkrURqfW2%2B669C6vyz7i%0AbKNvY%2BsSMtE5W3LH1t7TXPOreF2zghqMBcdaAy5nU8zR5XwSUd6xye3gAerJF2hp%0AfnWQwmCvWhGrrTUWVfqOpl8Dq1w9QiVHMNdHJo7tSx0JePrJYRShlXm%2FeoR4TK7q%0A%2B3oXqovBuT02syLWmSJNhQKBgCr%2FYlGrjgj%2BVWfgrjmy2w%2BCGcfV5LZocWDI5Ze8%0AcB57t7J94EOa7rclGwRx4KsMeUDJb7Ie34QIo%2FipAWC9DHIljyKVUqt17egXT2EG%0ARPOAA4LUmibe59AwZArLNjjM6jv0Vnjlt8cQ%2FenRUKNQz9uW03ZbslORM2tJS3Ql%0A%2FTwpAoGAFBYQarNj%2B4ruEvGAENP2oECq9RIKReJ5C%2FhuaITFC7qP15YmjsoTyjiG%0AJIi1FLtpDsBDEyMkOfjIJrfdGkKcWoSxAM%2FV0smuF1lc2SJECh9ESHx5IVZVcuQg%0ASBxQDUUoeu0jy2Ugkr6%2B06q%2BbUt4PkCBiczdZz1RchHnw6pyid0%3D%0A-----END%20RSA%20PRIVATE%20KEY-----%0A", + "verification": {} + }, + "mode": 384 + } + ] + }, + "systemd": { + "units": [ + { + "contents": "[Unit]\nDescription=kubeadm\n# Run only once. After successful run, this file is moved to /tmp/.\nConditionPathExists=/etc/kubeadm.yml\n[Service]\n# To not restart the unit when it exits, as it is expected.\nType=oneshot\nExecStart=/etc/kubeadm.sh\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "kubeadm.service" + } + ] + } + }`) + + expectedCmds := []provisioning.Cmd{ + {Cmd: "mkdir", Args: []string{"-p", "/etc/kubernetes/pki"}}, + {Cmd: "/bin/sh", Args: []string{"-c", "cat > /etc/kubernetes/pki/ca.crt /dev/stdin"}}, + {Cmd: "chmod", Args: []string{"0640", "/etc/kubernetes/pki/ca.crt"}}, + {Cmd: "mkdir", Args: []string{"-p", "/etc/kubernetes/pki"}}, + {Cmd: "/bin/sh", Args: []string{"-c", "cat > /etc/kubernetes/pki/ca.key /dev/stdin"}}, + {Cmd: "chmod", Args: []string{"0600", "/etc/kubernetes/pki/ca.key"}}, + {Cmd: "/bin/sh", Args: []string{"-c", "cat > /etc/systemd/system/kubeadm.service /dev/stdin"}}, + {Cmd: "systemctl", Args: []string{"daemon-reload"}}, + {Cmd: "systemctl", Args: []string{"enable", "--now", "kubeadm.service"}}, + } + + commands, err := RawIgnitionToProvisioningCommands(cloudData) + + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(commands).To(HaveLen(len(expectedCmds))) + + for i, cmd := range commands { + expected := expectedCmds[i] + g.Expect(cmd.Cmd).To(Equal(expected.Cmd)) + g.Expect(cmd.Args).To(ConsistOf(expected.Args)) + } +} + +func TestDecodeFileContents(t *testing.T) { + g := NewWithT(t) + + in := "data:,foo%20bar" + out, err := decodeFileContents(in) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(Equal("foo bar")) +} From 523663fc469b808aa88f7fb735c2b7d494dc9a52 Mon Sep 17 00:00:00 2001 From: Suraj Deshmukh Date: Fri, 17 Sep 2021 18:25:21 +0530 Subject: [PATCH 5/8] Add feature gate KubeadmBootstrapFormatIgnition This commit adds the feature gate KubeadmBootstrapFormatIgnition that will control the usage of field Ignition in KubeadmConfig. If user provides ignition field then the webhook config rejects the request with a validation error. Signed-off-by: Suraj Deshmukh Signed-off-by: Johanan Liebermann --- .../api/v1beta1/kubeadmconfig_types_test.go | 45 +++++++++++++- .../api/v1beta1/kubeadmconfig_webhook.go | 28 +++++++-- .../kubeadmconfigtemplate_webhook_test.go | 2 + bootstrap/kubeadm/config/manager/manager.yaml | 2 +- .../kubeadm_control_plane_webhook_test.go | 62 ++++++++++++------- .../kubeadm/config/manager/manager.yaml | 2 +- feature/feature.go | 13 +++- test/e2e/config/docker.yaml | 1 + 8 files changed, 122 insertions(+), 33 deletions(-) diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go index 47268b18fe0d..824347103feb 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go @@ -21,13 +21,17 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/component-base/featuregate/testing" "k8s.io/utils/pointer" + + "sigs.k8s.io/cluster-api/feature" ) func TestClusterValidate(t *testing.T) { cases := map[string]struct { - in *KubeadmConfig - expectErr bool + in *KubeadmConfig + enableIgnitionFeature bool + expectErr bool }{ "valid content": { in: &KubeadmConfig{ @@ -143,6 +147,7 @@ func TestClusterValidate(t *testing.T) { expectErr: true, }, "Ignition field is set, format is not Ignition": { + enableIgnitionFeature: true, in: &KubeadmConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", @@ -155,6 +160,7 @@ func TestClusterValidate(t *testing.T) { expectErr: true, }, "Ignition field is not set, format is Ignition": { + enableIgnitionFeature: true, in: &KubeadmConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", @@ -166,6 +172,7 @@ func TestClusterValidate(t *testing.T) { }, }, "format is Ignition, user is inactive": { + enableIgnitionFeature: true, in: &KubeadmConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", @@ -183,6 +190,7 @@ func TestClusterValidate(t *testing.T) { expectErr: true, }, "format is Ignition, non-GPT partition configured": { + enableIgnitionFeature: true, in: &KubeadmConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", @@ -202,6 +210,7 @@ func TestClusterValidate(t *testing.T) { expectErr: true, }, "format is Ignition, experimental retry join is set": { + enableIgnitionFeature: true, in: &KubeadmConfig{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", @@ -214,10 +223,42 @@ func TestClusterValidate(t *testing.T) { }, expectErr: true, }, + "feature gate disabled, format is Ignition": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + }, + }, + expectErr: true, + }, + "feature gate disabled, Ignition field is set": { + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + Ignition: &IgnitionSpec{ + ContainerLinuxConfig: &ContainerLinuxConfig{}, + }, + }, + }, + expectErr: true, + }, } for name, tt := range cases { t.Run(name, func(t *testing.T) { + if tt.enableIgnitionFeature { + // NOTE: KubeadmBootstrapFormatIgnition feature flag is disabled by default. + // Enabling the feature flag temporarily for this test. + defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.KubeadmBootstrapFormatIgnition, true)() + } g := NewWithT(t) if tt.expectErr { g.Expect(tt.in.ValidateCreate()).NotTo(Succeed()) diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go index 26506f1e829e..07f08a0d01e8 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go @@ -24,14 +24,17 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" + + "sigs.k8s.io/cluster-api/feature" ) var ( - cannotUseWithIgnition = fmt.Sprintf("not supported when spec.format is set to %q", Ignition) - conflictingFileSourceMsg = "only one of content or contentFrom may be specified for a single file" - missingSecretNameMsg = "secret file source must specify non-empty secret name" - missingSecretKeyMsg = "secret file source must specify non-empty secret key" - pathConflictMsg = "path property must be unique among all files" + cannotUseWithIgnition = fmt.Sprintf("not supported when spec.format is set to %q", Ignition) + conflictingFileSourceMsg = "only one of content or contentFrom may be specified for a single file" + kubeadmBootstrapFormatIgnitionFeatureDisabledMsg = "can be set only if the KubeadmBootstrapFormatIgnition feature gate is enabled" + missingSecretNameMsg = "secret file source must specify non-empty secret name" + missingSecretKeyMsg = "secret file source must specify non-empty secret key" + pathConflictMsg = "path property must be unique among all files" ) func (c *KubeadmConfig) SetupWebhookWithManager(mgr ctrl.Manager) error { @@ -65,6 +68,7 @@ func (c *KubeadmConfigSpec) validate(name string) error { if len(allErrs) == 0 { return nil } + return apierrors.NewInvalid(GroupVersion.WithKind("KubeadmConfig").GroupKind(), name, allErrs) } @@ -140,6 +144,20 @@ func (c *KubeadmConfigSpec) validateFiles() field.ErrorList { func (c *KubeadmConfigSpec) validateIgnition() field.ErrorList { var allErrs field.ErrorList + if !feature.Gates.Enabled(feature.KubeadmBootstrapFormatIgnition) { + if c.Format == Ignition { + allErrs = append(allErrs, field.Forbidden( + field.NewPath("spec", "format"), kubeadmBootstrapFormatIgnitionFeatureDisabledMsg)) + } + + if c.Ignition != nil { + allErrs = append(allErrs, field.Forbidden( + field.NewPath("spec", "ignition"), kubeadmBootstrapFormatIgnitionFeatureDisabledMsg)) + } + + return allErrs + } + if c.Format != Ignition { if c.Ignition != nil { allErrs = append( diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go index 2295eaa155d7..09b18ce2cb04 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfigtemplate_webhook_test.go @@ -45,6 +45,8 @@ func TestKubeadmConfigTemplateValidation(t *testing.T) { } for name, tt := range cases { + tt := tt + t.Run(name, func(t *testing.T) { g := NewWithT(t) g.Expect(tt.in.ValidateCreate()).To(Succeed()) diff --git a/bootstrap/kubeadm/config/manager/manager.yaml b/bootstrap/kubeadm/config/manager/manager.yaml index 233ba3d5fd90..8f5805b24425 100644 --- a/bootstrap/kubeadm/config/manager/manager.yaml +++ b/bootstrap/kubeadm/config/manager/manager.yaml @@ -21,7 +21,7 @@ spec: args: - "--leader-elect" - "--metrics-bind-addr=localhost:8080" - - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false}" + - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false}" image: controller:latest name: manager ports: diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go index 74d3ce1d09f4..e09aba21e190 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go @@ -24,9 +24,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + utilfeature "k8s.io/component-base/featuregate/testing" "k8s.io/utils/pointer" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/feature" utildefaulting "sigs.k8s.io/cluster-api/util/defaulting" ) @@ -135,9 +137,10 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { validIgnitionConfiguration.Spec.KubeadmConfigSpec.Ignition = &bootstrapv1.IgnitionSpec{} tests := []struct { - name string - expectErr bool - kcp *KubeadmControlPlane + name string + enableIgnitionFeature bool + expectErr bool + kcp *KubeadmControlPlane }{ { name: "should succeed when given a valid config", @@ -190,19 +193,27 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { kcp: invalidMaxSurge, }, { - name: "should return error when Ignition configuration is invalid", - expectErr: true, - kcp: invalidIgnitionConfiguration, + name: "should return error when Ignition configuration is invalid", + enableIgnitionFeature: true, + expectErr: true, + kcp: invalidIgnitionConfiguration, }, { - name: "should succeed when Ignition configuration is valid", - expectErr: false, - kcp: validIgnitionConfiguration, + name: "should succeed when Ignition configuration is valid", + enableIgnitionFeature: true, + expectErr: false, + kcp: validIgnitionConfiguration, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.enableIgnitionFeature { + // NOTE: KubeadmBootstrapFormatIgnition feature flag is disabled by default. + // Enabling the feature flag temporarily for this test. + defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.KubeadmBootstrapFormatIgnition, true)() + } + g := NewWithT(t) if tt.expectErr { @@ -571,10 +582,11 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { validIgnitionConfigurationAfter.Spec.KubeadmConfigSpec.Ignition.ContainerLinuxConfig.AdditionalConfig = "foo: bar" tests := []struct { - name string - expectErr bool - before *KubeadmControlPlane - kcp *KubeadmControlPlane + name string + enableIgnitionFeature bool + expectErr bool + before *KubeadmControlPlane + kcp *KubeadmControlPlane }{ { name: "should succeed when given a valid config", @@ -859,21 +871,29 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { kcp: disableNTPServers, }, { - name: "should return error when Ignition configuration is invalid", - expectErr: true, - before: invalidIgnitionConfiguration, - kcp: invalidIgnitionConfiguration, + name: "should return error when Ignition configuration is invalid", + enableIgnitionFeature: true, + expectErr: true, + before: invalidIgnitionConfiguration, + kcp: invalidIgnitionConfiguration, }, { - name: "should succeed when Ignition configuration is modified", - expectErr: false, - before: validIgnitionConfigurationBefore, - kcp: validIgnitionConfigurationAfter, + name: "should succeed when Ignition configuration is modified", + enableIgnitionFeature: true, + expectErr: false, + before: validIgnitionConfigurationBefore, + kcp: validIgnitionConfigurationAfter, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.enableIgnitionFeature { + // NOTE: KubeadmBootstrapFormatIgnition feature flag is disabled by default. + // Enabling the feature flag temporarily for this test. + defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.KubeadmBootstrapFormatIgnition, true)() + } + g := NewWithT(t) err := tt.kcp.ValidateUpdate(tt.before.DeepCopy()) diff --git a/controlplane/kubeadm/config/manager/manager.yaml b/controlplane/kubeadm/config/manager/manager.yaml index b5e31734e031..99b372b44e30 100644 --- a/controlplane/kubeadm/config/manager/manager.yaml +++ b/controlplane/kubeadm/config/manager/manager.yaml @@ -21,7 +21,7 @@ spec: args: - "--leader-elect" - "--metrics-bind-addr=localhost:8080" - - "--feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false}" + - "--feature-gates=ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false}" image: controller:latest name: manager ports: diff --git a/feature/feature.go b/feature/feature.go index 666f3fef8f5e..354e4bf40826 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -44,6 +44,12 @@ const ( // // alpha: v0.4 ClusterTopology featuregate.Feature = "ClusterTopology" + + // KubeadmBootstrapFormatIgnition is a feature gate for the Ignition bootstrap format + // functionality. + // + // alpha: v1.1 + KubeadmBootstrapFormatIgnition featuregate.Feature = "KubeadmBootstrapFormatIgnition" ) func init() { @@ -54,7 +60,8 @@ func init() { // To add a new feature, define a key for it above and add it here. var defaultClusterAPIFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ // Every feature should be initiated here: - MachinePool: {Default: false, PreRelease: featuregate.Alpha}, - ClusterResourceSet: {Default: true, PreRelease: featuregate.Beta}, - ClusterTopology: {Default: false, PreRelease: featuregate.Alpha}, + MachinePool: {Default: false, PreRelease: featuregate.Alpha}, + ClusterResourceSet: {Default: true, PreRelease: featuregate.Beta}, + ClusterTopology: {Default: false, PreRelease: featuregate.Alpha}, + KubeadmBootstrapFormatIgnition: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index ba7b3e77ad46..6868aed186a9 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -214,6 +214,7 @@ variables: NODE_DRAIN_TIMEOUT: "60s" # Enabling the feature flags by setting the env variables. EXP_CLUSTER_RESOURCE_SET: "true" + EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" EXP_MACHINE_POOL: "true" CLUSTER_TOPOLOGY: "true" # NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize From de8748e8acd1bed1588dc9e557d9b99b19cb2073 Mon Sep 17 00:00:00 2001 From: Suraj Deshmukh Date: Fri, 24 Sep 2021 16:09:49 +0530 Subject: [PATCH 6/8] Document Ignition feature gate Signed-off-by: Suraj Deshmukh Signed-off-by: Johanan Liebermann --- docs/book/src/SUMMARY.md | 1 + .../experimental-features.md | 1 + .../tasks/experimental-features/ignition.md | 171 ++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 docs/book/src/tasks/experimental-features/ignition.md diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index f96a5d41460e..a131822eb76b 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -20,6 +20,7 @@ - [ClusterResourceSet](./tasks/experimental-features/cluster-resource-set.md) - [ClusterClass](./tasks/experimental-features/cluster-classes.md) - [ClusterClass Operations](./tasks/experimental-features/cluster-class-operations.md) + - [Ignition Bootstrap configuration](./tasks/experimental-features/ignition.md) - [clusterctl CLI](./clusterctl/overview.md) - [clusterctl Commands](clusterctl/commands/commands.md) - [init](clusterctl/commands/init.md) diff --git a/docs/book/src/tasks/experimental-features/experimental-features.md b/docs/book/src/tasks/experimental-features/experimental-features.md index 34a56d87bdc3..baf487f5ee22 100644 --- a/docs/book/src/tasks/experimental-features/experimental-features.md +++ b/docs/book/src/tasks/experimental-features/experimental-features.md @@ -80,6 +80,7 @@ Similarly, to **validate** if a particular feature is enabled, see cluster-api-p * [ClusterResourceSet](./cluster-resource-set.md) * [ClusterClass](./cluster-classes.md) * [ClusterClass Operations](./cluster-class-operations.md) +* [Ignition Bootstrap configuration](./ignition.md) **Warning**: Experimental features are unreliable, i.e., some may one day be promoted to the main repository, or they may be modified arbitrarily or even disappear altogether. In short, they are not subject to any compatibility or deprecation promise. diff --git a/docs/book/src/tasks/experimental-features/ignition.md b/docs/book/src/tasks/experimental-features/ignition.md new file mode 100644 index 000000000000..bfd2e3d28a08 --- /dev/null +++ b/docs/book/src/tasks/experimental-features/ignition.md @@ -0,0 +1,171 @@ +# Experimental Feature: Ignition Bootstrap Config (alpha) + +The default configuration engine for bootstrapping workload cluster machines is [cloud-init](https://cloudinit.readthedocs.io/). **Ignition** is an alternative engine used by Linux distributions such as [Flatcar Container Linux](https://www.flatcar-linux.org/docs/latest/provisioning/ignition/) and [Fedora CoreOS](https://docs.fedoraproject.org/en-US/fedora-coreos/producing-ign/) and therefore should be used when choosing an Ignition-based distribution as the underlying OS for workload clusters. + + + +This guide explains how to deploy an AWS workload cluster using Ignition. + +## Prerequisites + +- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) installed locally +- [clusterawsadm](https://cluster-api-aws.sigs.k8s.io/introduction.html#clusterawsadm) installed locally - download from the [releases](https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases) page of the AWS provider +- [Kind](https://kind.sigs.k8s.io/) and [Docker](https://www.docker.com/) installed locally (when using Kind to create a management cluster) + +## Configure a management cluster + +Follow [this](../../user/quick-start.md#install-andor-configure-a-kubernetes-cluster) section of the quick start guide to deploy a Kubernetes cluster or connect to an existing one. + +Follow [this](../../user/quick-start.md#install-clusterctl) section of the quick start guide to install `clusterctl`. + +## Initialize the management cluster + +Before workload clusters can be deployed, Cluster API components must be deployed to the management cluster. + +Initialize the management cluster: + +```bash +export AWS_REGION=us-east-1 +export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +# Workload clusters need to call the AWS API as part of their normal operation. +# The following command creates a CloudFormation stack which provisions the +# necessary IAM resources to be used by workload clusters. +clusterawsadm bootstrap iam create-cloudformation-stack + +# The management cluster needs to call the AWS API in order to manage cloud +# resources for workload clusters. The following command tells clusterctl to +# store the AWS credentials provided before in a Kubernetes secret where they +# can be retrieved by the AWS provider running on the management cluster. +export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile) + +# Enable the feature gates controlling Ignition bootstrap. +export EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true # Used by the kubeadm bootstrap provider +export BOOTSTRAP_FORMAT_IGNITION=true # Used by the AWS provider + +# Initialize the management cluster. +clusterctl init --infrastructure aws +``` + +## Generate a workload cluster configuration + +```bash +# Deploy the workload cluster in the following AWS region. +export AWS_REGION=us-east-1 + +# Authorize the following SSH public key on cluster nodes. +export AWS_SSH_KEY_NAME=my-key + +# Ignition bootstrap data needs to be stored in an S3 bucket so that nodes can +# read them at boot time. Store Ignition bootstrap data in the following bucket. +export AWS_S3_BUCKET_NAME=my-bucket + +# Set the EC2 machine size for controllers and workers. +export AWS_CONTROL_PLANE_MACHINE_TYPE=t3a.small +export AWS_NODE_MACHINE_TYPE=t3a.small + +# TODO: Update --from URL once https://github.com/kubernetes-sigs/cluster-api-provider-aws/pull/2271 is merged. +clusterctl generate cluster ignition-cluster \ + --from https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/e7c89c9add92a4b233b26a1712518d9616d99e7a/templates/cluster-template-flatcar.yaml \ + --kubernetes-version v1.22.2 \ + --worker-machine-count 2 \ + > ignition-cluster.yaml +``` + +## Apply the workload cluster + +```bash +kubectl apply -f ignition-cluster.yaml +``` + +Wait for the control plane of the workload cluster to become initialized: + +```bash +kubectl get kubeadmcontrolplane ignition-cluster-control-plane +``` + +This could take a while. When the control plane is initialized, the `INITIALIZED` field should be `true`: + +``` +NAME CLUSTER INITIALIZED API SERVER AVAILABLE REPLICAS READY UPDATED UNAVAILABLE AGE VERSION +ignition-cluster-control-plane ignition-cluster true 1 1 1 7m7s v1.22.2 +``` + +## Connect to the workload cluster + +Generate a kubeconfig for the workload cluster: + +```bash +clusterctl get kubeconfig ignition-cluster > ./kubeconfig +``` + +Set `kubectl` to use the generated kubeconfig: + +```bash +export KUBECONFIG=$(pwd)/kubeconfig +``` + +Verify connectivity with the workload cluster's API server: + +```bash +kubectl cluster-info +``` + +Sample output: + +``` +Kubernetes control plane is running at https://ignition-cluster-apiserver-284992524.us-east-1.elb.amazonaws.com:6443 +CoreDNS is running at https://ignition-cluster-apiserver-284992524.us-east-1.elb.amazonaws.com:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy + +To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. +``` + +## Deploy a CNI plugin + +A [CNI plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) must be deployed to the workload cluster for the cluster to become ready. We use [Calico](https://www.tigera.io/project-calico/) here, however other CNI plugins could be used, too. + +```bash +kubectl apply -f https://docs.projectcalico.org/v3.20/manifests/calico.yaml +``` + +Ensure all cluster nodes become ready: + +```bash +kubectl get nodes +``` + +Sample output: + +``` +NAME STATUS ROLES AGE VERSION +ip-10-0-122-154.us-east-1.compute.internal Ready control-plane,master 14m v1.22.2 +ip-10-0-127-59.us-east-1.compute.internal Ready 13m v1.22.2 +ip-10-0-89-169.us-east-1.compute.internal Ready 13m v1.22.2 +``` + +## Clean up + +Delete the workload cluster (from a shell connected to the *management* cluster): + +```bash +kubectl delete cluster ignition-cluster +``` + +## Caveats + +### Supported infrastructure providers + +Cluster API has multiple [infrastructure providers](../../user/concepts.md#infrastructure-provider) which can be used to deploy workload clusters. + +The following infrastructure providers already have Ignition support: + +- [AWS](https://cluster-api-aws.sigs.k8s.io/) + +Ignition support will be added to more providers in the future. From 1df07964051be68fff7cd22d76428ef588d09484 Mon Sep 17 00:00:00 2001 From: Johanan Liebermann Date: Thu, 7 Oct 2021 10:16:52 +0300 Subject: [PATCH 7/8] Rename kubeadmconfig_types_test.go This file tests functions in kubeadmconfig_webhook_test.go. Signed-off-by: Johanan Liebermann --- ...{kubeadmconfig_types_test.go => kubeadmconfig_webhook_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bootstrap/kubeadm/api/v1beta1/{kubeadmconfig_types_test.go => kubeadmconfig_webhook_test.go} (100%) diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook_test.go similarity index 100% rename from bootstrap/kubeadm/api/v1beta1/kubeadmconfig_types_test.go rename to bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook_test.go From 87df81f4e4dd0c0ca78abdaa909da69f1112d4b8 Mon Sep 17 00:00:00 2001 From: Johanan Liebermann Date: Wed, 20 Oct 2021 17:59:08 +0300 Subject: [PATCH 8/8] Disallow unsupported storage params with Ignition Ignition doesn't support the replace_fs filesystem parameter and the partition filesystem parameter supported by cloud-init. Disallow using these parameters with Ignition. Signed-off-by: Johanan Liebermann --- .../api/v1beta1/kubeadmconfig_webhook.go | 22 ++++++++++ .../api/v1beta1/kubeadmconfig_webhook_test.go | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go index 07f08a0d01e8..1f24357766af 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook.go @@ -216,5 +216,27 @@ func (c *KubeadmConfigSpec) validateIgnition() field.ErrorList { } } + for i, fs := range c.DiskSetup.Filesystems { + if fs.ReplaceFS != nil { + allErrs = append( + allErrs, + field.Forbidden( + field.NewPath("spec", "diskSetup", "filesystems").Index(i).Child("replaceFS"), + cannotUseWithIgnition, + ), + ) + } + + if fs.Partition != nil { + allErrs = append( + allErrs, + field.Forbidden( + field.NewPath("spec", "diskSetup", "filesystems").Index(i).Child("partition"), + cannotUseWithIgnition, + ), + ) + } + } + return allErrs } diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook_test.go b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook_test.go index 824347103feb..58fc1d49629f 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook_test.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadmconfig_webhook_test.go @@ -250,6 +250,46 @@ func TestClusterValidate(t *testing.T) { }, expectErr: true, }, + "replaceFS specified with Ignition": { + enableIgnitionFeature: true, + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + DiskSetup: &DiskSetup{ + Filesystems: []Filesystem{ + { + ReplaceFS: pointer.StringPtr("ntfs"), + }, + }, + }, + }, + }, + expectErr: true, + }, + "filesystem partition specified with Ignition": { + enableIgnitionFeature: true, + in: &KubeadmConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + Namespace: "default", + }, + Spec: KubeadmConfigSpec{ + Format: Ignition, + DiskSetup: &DiskSetup{ + Filesystems: []Filesystem{ + { + Partition: pointer.StringPtr("1"), + }, + }, + }, + }, + }, + expectErr: true, + }, } for name, tt := range cases {