From b87e7373a27ca9079e0b107c7cd35314119ee212 Mon Sep 17 00:00:00 2001 From: Xu Deng Date: Tue, 18 Oct 2022 17:34:19 -0400 Subject: [PATCH] Handle kube-vip deployment in single-node cluster (#3667) Services can be exposed as LoadBalancer type supported by kube-vip. But in single-node cluster, kube-vip daemonset cannot be deployed correctly as it has to run in controlplane nodes. So we need this adjustment, to let the kube-vip daemonset of cp nodes to handle LoadBalancer services. --- .../tinkerbell/config/template-cp.yaml | 8 +- pkg/providers/tinkerbell/create.go | 4 +- pkg/providers/tinkerbell/template.go | 5 +- ...luster_tinkerbell_single_node_skip_lb.yaml | 52 ++++ ...lts_cluster_tinkerbell_cp_single_node.yaml | 4 + ...ter_tinkerbell_cp_single_node_skip_lb.yaml | 238 ++++++++++++++++++ pkg/providers/tinkerbell/tinkerbell_test.go | 36 +++ 7 files changed, 343 insertions(+), 4 deletions(-) create mode 100644 pkg/providers/tinkerbell/testdata/cluster_tinkerbell_single_node_skip_lb.yaml create mode 100644 pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node_skip_lb.yaml diff --git a/pkg/providers/tinkerbell/config/template-cp.yaml b/pkg/providers/tinkerbell/config/template-cp.yaml index 0dc250f31cc5..e6dabbcf5f7e 100644 --- a/pkg/providers/tinkerbell/config/template-cp.yaml +++ b/pkg/providers/tinkerbell/config/template-cp.yaml @@ -182,7 +182,13 @@ spec: value: "2" - name: address value: {{.controlPlaneEndpointIp}} - image: {{.kubeVipImage}} +{{- if and (not .workerNodeGroupConfigurations) (not .skipLoadBalancerDeployment) }} + # kube-vip daemon in worker node watches for LoadBalancer services. + # When there is no worker node, make kube-vip in control-plane nodes watch + - name: svc_enable + value: "true" +{{- end }} + image: {{ .kubeVipImage }} imagePullPolicy: IfNotPresent name: kube-vip resources: {} diff --git a/pkg/providers/tinkerbell/create.go b/pkg/providers/tinkerbell/create.go index 4cfb860efea6..3e9526c364ed 100644 --- a/pkg/providers/tinkerbell/create.go +++ b/pkg/providers/tinkerbell/create.go @@ -95,7 +95,9 @@ func (p *Provider) PostWorkloadInit(ctx context.Context, cluster *types.Cluster, stack.WithBootsOnKubernetes(), stack.WithHostPortEnabled(false), // disable host port on workload cluster stack.WithEnvoyEnabled(true), // use envoy on workload cluster - stack.WithLoadBalancerEnabled(!p.datacenterConfig.Spec.SkipLoadBalancerDeployment), // configure load balancer based on datacenterConfig.Spec.SkipLoadBalancerDeployment + stack.WithLoadBalancerEnabled( + len(clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations) != 0 && // load balancer is handled by kube-vip in control plane nodes + !p.datacenterConfig.Spec.SkipLoadBalancerDeployment), // configure load balancer based on datacenterConfig.Spec.SkipLoadBalancerDeployment ) if err != nil { return fmt.Errorf("installing stack on workload cluster: %v", err) diff --git a/pkg/providers/tinkerbell/template.go b/pkg/providers/tinkerbell/template.go index dec6450ee5e5..47edd95c08fe 100644 --- a/pkg/providers/tinkerbell/template.go +++ b/pkg/providers/tinkerbell/template.go @@ -98,7 +98,7 @@ func (tb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Spe return nil, fmt.Errorf("failed to get ETCD TinkerbellTemplateConfig: %v", err) } } - values := buildTemplateMapCP(clusterSpec, *tb.controlPlaneMachineSpec, etcdMachineSpec, cpTemplateString, etcdTemplateString) + values := buildTemplateMapCP(clusterSpec, *tb.controlPlaneMachineSpec, etcdMachineSpec, cpTemplateString, etcdTemplateString, *tb.datacenterSpec) for _, buildOption := range buildOptions { buildOption(values) @@ -367,7 +367,7 @@ func machineDeploymentName(clusterName, nodeGroupName string) string { return fmt.Sprintf("%s-%s", clusterName, nodeGroupName) } -func buildTemplateMapCP(clusterSpec *cluster.Spec, controlPlaneMachineSpec, etcdMachineSpec v1alpha1.TinkerbellMachineConfigSpec, cpTemplateOverride, etcdTemplateOverride string) map[string]interface{} { +func buildTemplateMapCP(clusterSpec *cluster.Spec, controlPlaneMachineSpec, etcdMachineSpec v1alpha1.TinkerbellMachineConfigSpec, cpTemplateOverride, etcdTemplateOverride string, datacenterSpec v1alpha1.TinkerbellDatacenterConfigSpec) map[string]interface{} { bundle := clusterSpec.VersionsBundle format := "cloud-config" @@ -410,6 +410,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec, controlPlaneMachineSpec, etcd "hardwareSelector": controlPlaneMachineSpec.HardwareSelector, "controlPlaneTaints": clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Taints, "workerNodeGroupConfigurations": clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations, + "skipLoadBalancerDeployment": datacenterSpec.SkipLoadBalancerDeployment, } if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil { diff --git a/pkg/providers/tinkerbell/testdata/cluster_tinkerbell_single_node_skip_lb.yaml b/pkg/providers/tinkerbell/testdata/cluster_tinkerbell_single_node_skip_lb.yaml new file mode 100644 index 000000000000..c2e06f9e6271 --- /dev/null +++ b/pkg/providers/tinkerbell/testdata/cluster_tinkerbell_single_node_skip_lb.yaml @@ -0,0 +1,52 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: single-node +spec: + clusterNetwork: + cniConfig: + cilium: {} + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneConfiguration: + count: 1 + endpoint: + host: 1.2.3.4 + machineGroupRef: + kind: TinkerbellMachineConfig + name: single-node-cp + taints: [] + datacenterRef: + kind: TinkerbellDatacenterConfig + name: single-node + kubernetesVersion: "1.21" + managementCluster: + name: single-node +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: TinkerbellDatacenterConfig +metadata: + name: single-node +spec: + tinkerbellIP: "5.6.7.8" + osImageURL: "https://ubuntu.gz" + skipLoadBalancerDeployment: true +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: TinkerbellMachineConfig +metadata: + name: single-node-cp +spec: + hardwareSelector: + type: cp + osFamily: ubuntu + templateRef: {} + users: + - name: tink-user + sshAuthorizedKeys: + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ== testemail@test.com" +--- \ No newline at end of file diff --git a/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node.yaml b/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node.yaml index a86bb7ffd390..7bde2562a812 100644 --- a/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node.yaml +++ b/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node.yaml @@ -95,6 +95,10 @@ spec: value: "2" - name: address value: 1.2.3.4 + # kube-vip daemon in worker node watches for LoadBalancer services. + # When there is no worker node, make kube-vip in control-plane nodes watch + - name: svc_enable + value: "true" image: public.ecr.aws/l0g8r8j6/kube-vip/kube-vip:v0.3.7-eks-a-v0.0.0-dev-build.581 imagePullPolicy: IfNotPresent name: kube-vip diff --git a/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node_skip_lb.yaml b/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node_skip_lb.yaml new file mode 100644 index 000000000000..a86bb7ffd390 --- /dev/null +++ b/pkg/providers/tinkerbell/testdata/expected_results_cluster_tinkerbell_cp_single_node_skip_lb.yaml @@ -0,0 +1,238 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: single-node + name: single-node + namespace: eksa-system +spec: + clusterNetwork: + pods: + cidrBlocks: [192.168.0.0/16] + services: + cidrBlocks: [10.96.0.0/12] + controlPlaneEndpoint: + host: 1.2.3.4 + port: 6443 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: single-node + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: TinkerbellCluster + name: single-node +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: single-node + namespace: eksa-system +spec: + kubeadmConfigSpec: + clusterConfiguration: + imageRepository: public.ecr.aws/eks-distro/kubernetes + etcd: + local: + imageRepository: public.ecr.aws/eks-distro/etcd-io + imageTag: v3.4.16-eks-1-21-4 + dns: + imageRepository: public.ecr.aws/eks-distro/coredns + imageTag: v1.8.3-eks-1-21-4 + apiServer: + extraArgs: + feature-gates: ServiceLoadBalancerClass=true + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + provider-id: PROVIDER_ID + read-only-port: "0" + anonymous-auth: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + taints: [] + joinConfiguration: + nodeRegistration: + ignorePreflightErrors: + - DirAvailable--etc-kubernetes-manifests + kubeletExtraArgs: + provider-id: PROVIDER_ID + read-only-port: "0" + anonymous-auth: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + taints: [] + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: address + value: 1.2.3.4 + image: public.ecr.aws/l0g8r8j6/kube-vip/kube-vip:v0.3.7-eks-a-v0.0.0-dev-build.581 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + users: + - name: tink-user + sshAuthorizedKeys: + - 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1BK73XhIzjX+meUr7pIYh6RHbvI3tmHeQIXY5lv7aztN1UoX+bhPo3dwo2sfSQn5kuxgQdnxIZ/CTzy0p0GkEYVv3gwspCeurjmu0XmrdmaSGcGxCEWT/65NtvYrQtUE5ELxJ+N/aeZNlK2B7IWANnw/82913asXH4VksV1NYNduP0o1/G4XcwLLSyVFB078q/oEnmvdNIoS61j4/o36HVtENJgYr0idcBvwJdvcGxGnPaqOhx477t+kfJAa5n5dSA5wilIaoXH5i1Tf/HsTCM52L+iNCARvQzJYZhzbWI1MDQwzILtIBEQCJsl2XSqIupleY8CxqQ6jCXt2mhae+wPc3YmbO5rFvr2/EvC57kh3yDs1Nsuj8KOvD78KeeujbR8n8pScm3WDp62HFQ8lEKNdeRNj6kB8WnuaJvPnyZfvzOhwG65/9w13IBl7B1sWxbFnq2rMpm5uHVK7mAmjL0Tt8zoDhcE1YJEnp9xte3/pvmKPkST5Q/9ZtR9P5sI+02jY0fvPkPyC03j2gsPixG7rpOCwpOdbny4dcj0TDeeXJX8er+oVfJuLYz0pNWJcT2raDdFfcqvYA0B0IyNYlj5nWX4RuEcyT3qocLReWPnZojetvAG/H8XwOh7fEVGqHAKOVSnPXCSQJPl6s0H12jPJBDJMTydtYPEszl4/CeQ==' + sudo: ALL=(ALL) NOPASSWD:ALL + format: cloud-config + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: TinkerbellMachineTemplate + name: single-node-control-plane-template-1234567890000 + replicas: 1 + version: v1.21.2-eks-1-21-4 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: TinkerbellMachineTemplate +metadata: + name: single-node-control-plane-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + hardwareAffinity: + required: + - labelSelector: + matchLabels: + type: cp + templateOverride: | + global_timeout: 6000 + id: "" + name: single-node + tasks: + - actions: + - environment: + COMPRESSED: "true" + DEST_DISK: /dev/sda + IMG_URL: https://ubuntu.gz + image: "" + name: stream-image + timeout: 600 + - environment: + DEST_DISK: /dev/sda2 + DEST_PATH: /etc/netplan/config.yaml + DIRMODE: "0755" + FS_TYPE: ext4 + GID: "0" + MODE: "0644" + STATIC_NETPLAN: "true" + UID: "0" + image: "" + name: write-netplan + pid: host + timeout: 90 + - environment: + CONTENTS: 'network: {config: disabled}' + DEST_DISK: /dev/sda2 + DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg + DIRMODE: "0700" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + image: "" + name: disable-cloud-init-network-capabilities + timeout: 90 + - environment: + CONTENTS: | + datasource: + Ec2: + metadata_urls: [http://5.6.7.8:50061,http://5.6.7.8:50061] + strict_id: false + manage_etc_hosts: localhost + warnings: + dsid_missing_source: off + DEST_DISK: /dev/sda2 + DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg + DIRMODE: "0700" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + image: "" + name: add-tink-cloud-init-config + timeout: 90 + - environment: + CONTENTS: | + datasource: Ec2 + DEST_DISK: /dev/sda2 + DEST_PATH: /etc/cloud/ds-identify.cfg + DIRMODE: "0700" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + image: "" + name: add-tink-cloud-init-ds-config + timeout: 90 + - environment: + BLOCK_DEVICE: /dev/sda2 + FS_TYPE: ext4 + image: "" + name: kexec-image + pid: host + timeout: 90 + name: single-node + volumes: + - /dev:/dev + - /dev/console:/dev/console + - /lib/firmware:/lib/firmware:ro + worker: '{{.device_1}}' + version: "0.1" + +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: TinkerbellCluster +metadata: + name: single-node + namespace: eksa-system +spec: + imageLookupFormat: --kube-v1.21.2-eks-1-21-4.raw.gz + imageLookupBaseRegistry: / diff --git a/pkg/providers/tinkerbell/tinkerbell_test.go b/pkg/providers/tinkerbell/tinkerbell_test.go index 399a11410092..13e4ade3bacc 100644 --- a/pkg/providers/tinkerbell/tinkerbell_test.go +++ b/pkg/providers/tinkerbell/tinkerbell_test.go @@ -821,3 +821,39 @@ func TestProviderGenerateDeploymentFileForSingleNodeCluster(t *testing.T) { } test.AssertContentToFile(t, string(cp), "testdata/expected_results_cluster_tinkerbell_cp_single_node.yaml") } + +func TestProviderGenerateDeploymentFileForSingleNodeClusterSkipLB(t *testing.T) { + clusterSpecManifest := "cluster_tinkerbell_single_node_skip_lb.yaml" + mockCtrl := gomock.NewController(t) + docker := stackmocks.NewMockDocker(mockCtrl) + helm := stackmocks.NewMockHelm(mockCtrl) + kubectl := mocks.NewMockProviderKubectlClient(mockCtrl) + stackInstaller := stackmocks.NewMockStackInstaller(mockCtrl) + writer := filewritermocks.NewMockFileWriter(mockCtrl) + cluster := &types.Cluster{Name: "test"} + forceCleanup := false + + clusterSpec := givenClusterSpec(t, clusterSpecManifest) + datacenterConfig := givenDatacenterConfig(t, clusterSpecManifest) + machineConfigs := givenMachineConfigs(t, clusterSpecManifest) + ctx := context.Background() + + provider := newProvider(datacenterConfig, machineConfigs, clusterSpec.Cluster, writer, docker, helm, kubectl, forceCleanup) + provider.stackInstaller = stackInstaller + + stackInstaller.EXPECT().CleanupLocalBoots(ctx, forceCleanup) + + if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil { + t.Fatalf("failed to setup and validate: %v", err) + } + + cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), cluster, clusterSpec) + if err != nil { + t.Fatalf("failed to generate cluster api spec contents: %v", err) + } + + if len(md) != 0 { + t.Fatalf("expect nothing to be generated for worker node") + } + test.AssertContentToFile(t, string(cp), "testdata/expected_results_cluster_tinkerbell_cp_single_node_skip_lb.yaml") +}