Merge pull request #14080 from hakman/etcd-manager_ig

Create etcd-manager config for each instance group

cmd/kops/integration_test.go

@@ -487,7 +487,7 @@ func TestPrivateCiliumAdvanced(t *testing.T) {
 	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
 		withPrivate().
 		withCiliumEtcd().
-		withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium").
+		withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium-master-us-test-1a").
 		withAddons(ciliumAddon, dnsControllerAddon).
 		runTestTerraformAWS(t)
 	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
@@ -1202,8 +1202,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 		"aws_s3_object_etcd-cluster-spec-events_content",
 		"aws_s3_object_etcd-cluster-spec-main_content",
 		"aws_s3_object_kops-version.txt_content",
-		"aws_s3_object_manifests-etcdmanager-events_content",
-		"aws_s3_object_manifests-etcdmanager-main_content",
 		"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
 		"aws_s3_object_nodeupconfig-nodes_content",
 		"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
@@ -1234,6 +1232,8 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 	for j := 0; j < i.zones; j++ {
 		zone := "us-test-1" + string([]byte{byte('a') + byte(j)})
 		expectedFilenames = append(expectedFilenames,
+			"aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content",
+			"aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content",
 			"aws_s3_object_nodeupconfig-master-"+zone+"_content",
 			"aws_launch_template_master-"+zone+".masters."+i.clusterName+"_user_data")
 	}
@@ -1333,8 +1333,6 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
 		"aws_s3_object_etcd-cluster-spec-events_content",
 		"aws_s3_object_etcd-cluster-spec-main_content",
 		"aws_s3_object_kops-version.txt_content",
-		"aws_s3_object_manifests-etcdmanager-events_content",
-		"aws_s3_object_manifests-etcdmanager-main_content",
 		"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
 		"aws_s3_object_nodeupconfig-nodes_content",
 		"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
@@ -1349,6 +1347,8 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
 		zone := "us-test1-" + string([]byte{byte('a') + byte(j)})
 		prefix := "google_compute_instance_template_master-" + zone + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_"
 
+		expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content")
+		expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content")
 		expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-master-"+zone+"_content")
 		expectedFilenames = append(expectedFilenames, prefix+"startup-script")
 		expectedFilenames = append(expectedFilenames, prefix+"ssh-keys")

pkg/model/components/etcdmanager/model.go

@@ -57,9 +57,6 @@ var _ fi.ModelBuilder = &EtcdManagerBuilder{}
 // Build creates the tasks
 func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 	for _, etcdCluster := range b.Cluster.Spec.EtcdClusters {
-		name := etcdCluster.Name
-		version := etcdCluster.Version
-
 		backupStore := ""
 		if etcdCluster.Backups != nil {
 			backupStore = etcdCluster.Backups.BackupStore
@@ -68,25 +65,29 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 			return fmt.Errorf("backupStore must be set for use with etcd-manager")
 		}
 
-		manifest, err := b.buildManifest(etcdCluster)
-		if err != nil {
-			return err
-		}
+		for _, member := range etcdCluster.Members {
+			instanceGroupName := fi.StringValue(member.InstanceGroup)
+			manifest, err := b.buildManifest(etcdCluster, instanceGroupName)
+			if err != nil {
+				return err
+			}
 
-		manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
-		if err != nil {
-			return fmt.Errorf("error marshaling manifest to yaml: %v", err)
-		}
+			manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
+			if err != nil {
+				return fmt.Errorf("error marshaling manifest to yaml: %v", err)
+			}
 
-		c.AddTask(&fitasks.ManagedFile{
-			Contents:  fi.NewBytesResource(manifestYAML),
-			Lifecycle: b.Lifecycle,
-			Location:  fi.String("manifests/etcd/" + name + ".yaml"),
-			Name:      fi.String("manifests-etcdmanager-" + name),
-		})
+			name := fmt.Sprintf("%s-%s", etcdCluster.Name, instanceGroupName)
+			c.AddTask(&fitasks.ManagedFile{
+				Contents:  fi.NewBytesResource(manifestYAML),
+				Lifecycle: b.Lifecycle,
+				Location:  fi.String("manifests/etcd/" + name + ".yaml"),
+				Name:      fi.String("manifests-etcdmanager-" + name),
+			})
+		}
 
 		info := &etcdClusterSpec{
-			EtcdVersion: version,
+			EtcdVersion: etcdCluster.Version,
 			MemberCount: int32(len(etcdCluster.Members)),
 		}
 
@@ -108,7 +109,7 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 			Base:      fi.String(backupStore),
 			// TODO: We need this to match the backup base (currently)
 			Location: fi.String(location + "/control/etcd-cluster-spec"),
-			Name:     fi.String("etcd-cluster-spec-" + name),
+			Name:     fi.String("etcd-cluster-spec-" + etcdCluster.Name),
 		})
 
 		// We create a CA keypair to enable secure communication
@@ -166,8 +167,8 @@ type etcdClusterSpec struct {
 	EtcdVersion string `json:"etcdVersion,omitempty"`
 }
 
-func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) {
-	return b.buildPod(etcdCluster)
+func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
+	return b.buildPod(etcdCluster, instanceGroupName)
 }
 
 // Until we introduce the bundle, we hard-code the manifest
@@ -214,7 +215,7 @@ spec:
 `
 
 // buildPod creates the pod spec, based on the EtcdClusterSpec
-func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) {
+func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
 	var pod *v1.Pod
 	var container *v1.Container
 

pkg/model/components/etcdmanager/tests/interval/tasks.yaml

@@ -128,8 +128,8 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/events.yaml
-Name: manifests-etcdmanager-events
+Location: manifests/etcd/events-master-us-test-1a.yaml
+Name: manifests-etcdmanager-events-master-us-test-1a
 Public: null
 ---
 Base: null
@@ -197,6 +197,6 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/main.yaml
-Name: manifests-etcdmanager-main
+Location: manifests/etcd/main-master-us-test-1a.yaml
+Name: manifests-etcdmanager-main-master-us-test-1a
 Public: null

pkg/model/components/etcdmanager/tests/minimal/tasks.yaml

@@ -127,8 +127,8 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/events.yaml
-Name: manifests-etcdmanager-events
+Location: manifests/etcd/events-master-us-test-1a.yaml
+Name: manifests-etcdmanager-events-master-us-test-1a
 Public: null
 ---
 Base: null
@@ -195,6 +195,6 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/main.yaml
-Name: manifests-etcdmanager-main
+Location: manifests/etcd/main-master-us-test-1a.yaml
+Name: manifests-etcdmanager-main-master-us-test-1a
 Public: null

pkg/model/components/etcdmanager/tests/overwrite_settings/tasks.yaml

@@ -130,8 +130,8 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/events.yaml
-Name: manifests-etcdmanager-events
+Location: manifests/etcd/events-master-us-test-1a.yaml
+Name: manifests-etcdmanager-events-master-us-test-1a
 Public: null
 ---
 Base: null
@@ -201,6 +201,6 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/main.yaml
-Name: manifests-etcdmanager-main
+Location: manifests/etcd/main-master-us-test-1a.yaml
+Name: manifests-etcdmanager-main-master-us-test-1a
 Public: null

pkg/model/components/etcdmanager/tests/proxy/tasks.yaml

@@ -136,8 +136,8 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/events.yaml
-Name: manifests-etcdmanager-events
+Location: manifests/etcd/events-master-us-test-1a.yaml
+Name: manifests-etcdmanager-events-master-us-test-1a
 Public: null
 ---
 Base: null
@@ -213,6 +213,6 @@ Contents: |
       name: varlogetcd
   status: {}
 Lifecycle: ""
-Location: manifests/etcd/main.yaml
-Name: manifests-etcdmanager-main
+Location: manifests/etcd/main-master-us-test-1a.yaml
+Name: manifests-etcdmanager-main-master-us-test-1a
 Public: null

tests/integration/update_cluster/additionalobjects/data/aws_launch_template_master-us-test-1a.masters.additionalobjects.example.com_user_data

@@ -269,7 +269,7 @@ CloudProvider: aws
 ConfigBase: memfs://tests/additionalobjects.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: 9pzqS4P6CQBoUaJX9isvi855VbNwglocW5mQd9MxF4c=
+NodeupConfigHash: fakCiYsFNoBTHwDYQgsVgXkkCkARXy/uzu+PDD7NOAs=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/additionalobjects/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -281,8 +281,8 @@ containerdConfig:
     version: 1.1.3
   version: 1.6.8
 etcdManifests:
-- memfs://tests/additionalobjects.example.com/manifests/etcd/main.yaml
-- memfs://tests/additionalobjects.example.com/manifests/etcd/events.yaml
+- memfs://tests/additionalobjects.example.com/manifests/etcd/main-master-us-test-1a.yaml
+- memfs://tests/additionalobjects.example.com/manifests/etcd/events-master-us-test-1a.yaml
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml

tests/integration/update_cluster/additionalobjects/kubernetes.tf

@@ -609,18 +609,18 @@ resource "aws_s3_object" "kops-version-txt" {
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-events" {
+resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
-  key                    = "tests/additionalobjects.example.com/manifests/etcd/events.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
+  key                    = "tests/additionalobjects.example.com/manifests/etcd/events-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-main" {
+resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
-  key                    = "tests/additionalobjects.example.com/manifests/etcd/main.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
+  key                    = "tests/additionalobjects.example.com/manifests/etcd/main-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }

tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml

@@ -439,7 +439,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
-  NodeupConfigHash: fQk0XYnSl+mPW9dF85gn2ny0ga76H9fyudV6BhJUkl4=
+  NodeupConfigHash: o14f+FINvjAsuxKdAZ0TjAfBMvJ2DDjSLodwewNgm2w=
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/apiservernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -262,7 +262,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: fQk0XYnSl+mPW9dF85gn2ny0ga76H9fyudV6BhJUkl4=
+NodeupConfigHash: o14f+FINvjAsuxKdAZ0TjAfBMvJ2DDjSLodwewNgm2w=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -275,8 +275,8 @@ containerdConfig:
   logLevel: info
   version: 1.4.13
 etcdManifests:
-- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
-- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml

tests/integration/update_cluster/apiservernodes/kubernetes.tf

@@ -698,18 +698,18 @@ resource "aws_s3_object" "kops-version-txt" {
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-events" {
+resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
-  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/events.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
+  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-main" {
+resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
-  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/main.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
+  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }

tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -244,7 +244,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: pqJn3OPb3BOn0/zySTUOO9Ohmxs7XHR6+NnA93T1/Wc=
+NodeupConfigHash: F56oipBerHI/IM58aPmR1lXYLb5nkwRq2LaknRGeeNQ=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -267,8 +267,8 @@ containerdConfig:
   logLevel: info
   version: 1.4.13
 etcdManifests:
-- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
-- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml
+- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml

tests/integration/update_cluster/aws-lb-controller/kubernetes.tf

@@ -612,18 +612,18 @@ resource "aws_s3_object" "kops-version-txt" {
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-events" {
+resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
-  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/events.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
+  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-main" {
+resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
-  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/main.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
+  key                    = "clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }

tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data

@@ -244,7 +244,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/bastionuserdata.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: E+5eXSXOWJB4JVbmF015qXc8xA6ST6eqrnawd0g3bQw=
+NodeupConfigHash: Yieg04ujnxQRIa+INE/cglnO0ggsDndm1PGNXvi2ejw=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -267,8 +267,8 @@ containerdConfig:
   logLevel: info
   version: 1.4.13
 etcdManifests:
-- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/main.yaml
-- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/events.yaml
+- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/main-master-us-test-1a.yaml
+- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/events-master-us-test-1a.yaml
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml

tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf

@@ -862,18 +862,18 @@ resource "aws_s3_object" "kops-version-txt" {
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-events" {
+resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
-  key                    = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/events.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
+  key                    = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/events-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "manifests-etcdmanager-main" {
+resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
-  key                    = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/main.yaml"
+  content                = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
+  key                    = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/main-master-us-test-1a.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }