Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create etcd-manager config for each instance group #14080

Merged
merged 2 commits into from
Aug 15, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
10 changes: 5 additions & 5 deletions cmd/kops/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ func TestPrivateCiliumAdvanced(t *testing.T) {
newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
withPrivate().
withCiliumEtcd().
withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium").
withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium-master-us-test-1a").
withAddons(ciliumAddon, dnsControllerAddon).
runTestTerraformAWS(t)
newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
Expand Down Expand Up @@ -1202,8 +1202,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
"aws_s3_object_etcd-cluster-spec-events_content",
"aws_s3_object_etcd-cluster-spec-main_content",
"aws_s3_object_kops-version.txt_content",
"aws_s3_object_manifests-etcdmanager-events_content",
"aws_s3_object_manifests-etcdmanager-main_content",
"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
"aws_s3_object_nodeupconfig-nodes_content",
"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
Expand Down Expand Up @@ -1234,6 +1232,8 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
for j := 0; j < i.zones; j++ {
zone := "us-test-1" + string([]byte{byte('a') + byte(j)})
expectedFilenames = append(expectedFilenames,
"aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content",
"aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content",
"aws_s3_object_nodeupconfig-master-"+zone+"_content",
"aws_launch_template_master-"+zone+".masters."+i.clusterName+"_user_data")
}
Expand Down Expand Up @@ -1333,8 +1333,6 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
"aws_s3_object_etcd-cluster-spec-events_content",
"aws_s3_object_etcd-cluster-spec-main_content",
"aws_s3_object_kops-version.txt_content",
"aws_s3_object_manifests-etcdmanager-events_content",
"aws_s3_object_manifests-etcdmanager-main_content",
"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
"aws_s3_object_nodeupconfig-nodes_content",
"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
Expand All @@ -1349,6 +1347,8 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
zone := "us-test1-" + string([]byte{byte('a') + byte(j)})
prefix := "google_compute_instance_template_master-" + zone + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_"

expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content")
expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content")
expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-master-"+zone+"_content")
expectedFilenames = append(expectedFilenames, prefix+"startup-script")
expectedFilenames = append(expectedFilenames, prefix+"ssh-keys")
Expand Down
45 changes: 23 additions & 22 deletions pkg/model/components/etcdmanager/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,6 @@ var _ fi.ModelBuilder = &EtcdManagerBuilder{}
// Build creates the tasks
func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
for _, etcdCluster := range b.Cluster.Spec.EtcdClusters {
name := etcdCluster.Name
version := etcdCluster.Version

backupStore := ""
if etcdCluster.Backups != nil {
backupStore = etcdCluster.Backups.BackupStore
Expand All @@ -68,25 +65,29 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
return fmt.Errorf("backupStore must be set for use with etcd-manager")
}

manifest, err := b.buildManifest(etcdCluster)
if err != nil {
return err
}
for _, member := range etcdCluster.Members {
instanceGroupName := fi.StringValue(member.InstanceGroup)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be that we should use the member name here. On the one hand, it's the primary key and therefore maybe "more correct", and maybe avoids the problem of then wanting to lock instance groups to members. On the other hand, it means users must be more aware of etcd member names, when they are probably more familiar with instance group names.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Edit: but now I see that it the mapping in nodeup below becomes much more straightforward if we stick to instance groups...

manifest, err := b.buildManifest(etcdCluster, instanceGroupName)
if err != nil {
return err
}

manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
if err != nil {
return fmt.Errorf("error marshaling manifest to yaml: %v", err)
}
manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
if err != nil {
return fmt.Errorf("error marshaling manifest to yaml: %v", err)
}

c.AddTask(&fitasks.ManagedFile{
Contents: fi.NewBytesResource(manifestYAML),
Lifecycle: b.Lifecycle,
Location: fi.String("manifests/etcd/" + name + ".yaml"),
Name: fi.String("manifests-etcdmanager-" + name),
})
name := fmt.Sprintf("%s-%s", etcdCluster.Name, instanceGroupName)
c.AddTask(&fitasks.ManagedFile{
Contents: fi.NewBytesResource(manifestYAML),
Lifecycle: b.Lifecycle,
Location: fi.String("manifests/etcd/" + name + ".yaml"),
Name: fi.String("manifests-etcdmanager-" + name),
})
}

info := &etcdClusterSpec{
EtcdVersion: version,
EtcdVersion: etcdCluster.Version,
MemberCount: int32(len(etcdCluster.Members)),
}

Expand All @@ -108,7 +109,7 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
Base: fi.String(backupStore),
// TODO: We need this to match the backup base (currently)
Location: fi.String(location + "/control/etcd-cluster-spec"),
Name: fi.String("etcd-cluster-spec-" + name),
Name: fi.String("etcd-cluster-spec-" + etcdCluster.Name),
})

// We create a CA keypair to enable secure communication
Expand Down Expand Up @@ -166,8 +167,8 @@ type etcdClusterSpec struct {
EtcdVersion string `json:"etcdVersion,omitempty"`
}

func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) {
return b.buildPod(etcdCluster)
func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
return b.buildPod(etcdCluster, instanceGroupName)
}

// Until we introduce the bundle, we hard-code the manifest
Expand Down Expand Up @@ -214,7 +215,7 @@ spec:
`

// buildPod creates the pod spec, based on the EtcdClusterSpec
func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) {
func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
var pod *v1.Pod
var container *v1.Container

Expand Down
8 changes: 4 additions & 4 deletions pkg/model/components/etcdmanager/tests/interval/tasks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/events.yaml
Name: manifests-etcdmanager-events
Location: manifests/etcd/events-master-us-test-1a.yaml
Name: manifests-etcdmanager-events-master-us-test-1a
Public: null
---
Base: null
Expand Down Expand Up @@ -197,6 +197,6 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/main.yaml
Name: manifests-etcdmanager-main
Location: manifests/etcd/main-master-us-test-1a.yaml
Name: manifests-etcdmanager-main-master-us-test-1a
Public: null
8 changes: 4 additions & 4 deletions pkg/model/components/etcdmanager/tests/minimal/tasks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,8 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/events.yaml
Name: manifests-etcdmanager-events
Location: manifests/etcd/events-master-us-test-1a.yaml
Name: manifests-etcdmanager-events-master-us-test-1a
Public: null
---
Base: null
Expand Down Expand Up @@ -195,6 +195,6 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/main.yaml
Name: manifests-etcdmanager-main
Location: manifests/etcd/main-master-us-test-1a.yaml
Name: manifests-etcdmanager-main-master-us-test-1a
Public: null
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/events.yaml
Name: manifests-etcdmanager-events
Location: manifests/etcd/events-master-us-test-1a.yaml
Name: manifests-etcdmanager-events-master-us-test-1a
Public: null
---
Base: null
Expand Down Expand Up @@ -201,6 +201,6 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/main.yaml
Name: manifests-etcdmanager-main
Location: manifests/etcd/main-master-us-test-1a.yaml
Name: manifests-etcdmanager-main-master-us-test-1a
Public: null
8 changes: 4 additions & 4 deletions pkg/model/components/etcdmanager/tests/proxy/tasks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,8 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/events.yaml
Name: manifests-etcdmanager-events
Location: manifests/etcd/events-master-us-test-1a.yaml
Name: manifests-etcdmanager-events-master-us-test-1a
Public: null
---
Base: null
Expand Down Expand Up @@ -213,6 +213,6 @@ Contents: |
name: varlogetcd
status: {}
Lifecycle: ""
Location: manifests/etcd/main.yaml
Name: manifests-etcdmanager-main
Location: manifests/etcd/main-master-us-test-1a.yaml
Name: manifests-etcdmanager-main-master-us-test-1a
Public: null
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ CloudProvider: aws
ConfigBase: memfs://tests/additionalobjects.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: Master
NodeupConfigHash: 9pzqS4P6CQBoUaJX9isvi855VbNwglocW5mQd9MxF4c=
NodeupConfigHash: fakCiYsFNoBTHwDYQgsVgXkkCkARXy/uzu+PDD7NOAs=

__EOF_KUBE_ENV

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ containerdConfig:
version: 1.1.3
version: 1.6.8
etcdManifests:
- memfs://tests/additionalobjects.example.com/manifests/etcd/main.yaml
- memfs://tests/additionalobjects.example.com/manifests/etcd/events.yaml
- memfs://tests/additionalobjects.example.com/manifests/etcd/main-master-us-test-1a.yaml
- memfs://tests/additionalobjects.example.com/manifests/etcd/events-master-us-test-1a.yaml
staticManifests:
- key: kube-apiserver-healthcheck
path: manifests/static/kube-apiserver-healthcheck.yaml
12 changes: 6 additions & 6 deletions tests/integration/update_cluster/additionalobjects/kubernetes.tf
Original file line number Diff line number Diff line change
Expand Up @@ -609,18 +609,18 @@ resource "aws_s3_object" "kops-version-txt" {
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-events" {
resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
key = "tests/additionalobjects.example.com/manifests/etcd/events.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
key = "tests/additionalobjects.example.com/manifests/etcd/events-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-main" {
resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
key = "tests/additionalobjects.example.com/manifests/etcd/main.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
key = "tests/additionalobjects.example.com/manifests/etcd/main-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
ConfigBase: memfs://clusters.example.com/minimal.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: Master
NodeupConfigHash: fQk0XYnSl+mPW9dF85gn2ny0ga76H9fyudV6BhJUkl4=
NodeupConfigHash: o14f+FINvjAsuxKdAZ0TjAfBMvJ2DDjSLodwewNgm2w=

__EOF_KUBE_ENV

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ CloudProvider: aws
ConfigBase: memfs://clusters.example.com/minimal.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: Master
NodeupConfigHash: fQk0XYnSl+mPW9dF85gn2ny0ga76H9fyudV6BhJUkl4=
NodeupConfigHash: o14f+FINvjAsuxKdAZ0TjAfBMvJ2DDjSLodwewNgm2w=

__EOF_KUBE_ENV

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,8 +275,8 @@ containerdConfig:
logLevel: info
version: 1.4.13
etcdManifests:
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml
staticManifests:
- key: kube-apiserver-healthcheck
path: manifests/static/kube-apiserver-healthcheck.yaml
12 changes: 6 additions & 6 deletions tests/integration/update_cluster/apiservernodes/kubernetes.tf
Original file line number Diff line number Diff line change
Expand Up @@ -698,18 +698,18 @@ resource "aws_s3_object" "kops-version-txt" {
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-events" {
resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/events.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-main" {
resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/main.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ CloudProvider: aws
ConfigBase: memfs://clusters.example.com/minimal.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: Master
NodeupConfigHash: pqJn3OPb3BOn0/zySTUOO9Ohmxs7XHR6+NnA93T1/Wc=
NodeupConfigHash: F56oipBerHI/IM58aPmR1lXYLb5nkwRq2LaknRGeeNQ=

__EOF_KUBE_ENV

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,8 @@ containerdConfig:
logLevel: info
version: 1.4.13
etcdManifests:
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml
- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml
staticManifests:
- key: kube-apiserver-healthcheck
path: manifests/static/kube-apiserver-healthcheck.yaml
12 changes: 6 additions & 6 deletions tests/integration/update_cluster/aws-lb-controller/kubernetes.tf
Original file line number Diff line number Diff line change
Expand Up @@ -612,18 +612,18 @@ resource "aws_s3_object" "kops-version-txt" {
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-events" {
resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/events.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-main" {
resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/main.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
key = "clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ CloudProvider: aws
ConfigBase: memfs://clusters.example.com/bastionuserdata.example.com
InstanceGroupName: master-us-test-1a
InstanceGroupRole: Master
NodeupConfigHash: E+5eXSXOWJB4JVbmF015qXc8xA6ST6eqrnawd0g3bQw=
NodeupConfigHash: Yieg04ujnxQRIa+INE/cglnO0ggsDndm1PGNXvi2ejw=

__EOF_KUBE_ENV

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,8 @@ containerdConfig:
logLevel: info
version: 1.4.13
etcdManifests:
- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/main.yaml
- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/events.yaml
- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/main-master-us-test-1a.yaml
- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/events-master-us-test-1a.yaml
staticManifests:
- key: kube-apiserver-healthcheck
path: manifests/static/kube-apiserver-healthcheck.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -862,18 +862,18 @@ resource "aws_s3_object" "kops-version-txt" {
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-events" {
resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events_content")
key = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/events.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content")
key = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/events-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}

resource "aws_s3_object" "manifests-etcdmanager-main" {
resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main_content")
key = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/main.yaml"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content")
key = "clusters.example.com/bastionuserdata.example.com/manifests/etcd/main-master-us-test-1a.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
Expand Down
Loading