From e20900a2de586f98f843e1e8641986db712ec838 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 13 Jan 2021 08:29:33 +0200 Subject: [PATCH 1/2] Add CF integration test for gp3 volumes --- cmd/kops/integration_test.go | 1 + .../minimal-gp3/cloudformation.json | 1129 +++++++++++++++++ .../cloudformation.json.extracted.yaml | 550 ++++++++ 3 files changed, 1680 insertions(+) create mode 100644 tests/integration/update_cluster/minimal-gp3/cloudformation.json create mode 100644 tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 4f9ac43f9dbe3..69e587e76ae13 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -184,6 +184,7 @@ func TestMinimalCloudformation(t *testing.T) { // TestMinimalGp3 runs the test on a minimum configuration using gp3 volumes, similar to kops create cluster minimal.example.com --zones us-west-1a func TestMinimalGp3(t *testing.T) { newIntegrationTest("minimal.example.com", "minimal-gp3").runTestTerraformAWS(t) + newIntegrationTest("minimal.example.com", "minimal-gp3").runTestCloudformation(t) } // TestExistingIAMCloudformation runs the test with existing IAM instance profiles, similar to kops create cluster minimal.example.com --zones us-west-1a diff --git a/tests/integration/update_cluster/minimal-gp3/cloudformation.json b/tests/integration/update_cluster/minimal-gp3/cloudformation.json new file mode 100644 index 0000000000000..4007d4b5bd627 --- /dev/null +++ b/tests/integration/update_cluster/minimal-gp3/cloudformation.json @@ -0,0 +1,1129 @@ +{ + "Resources": { + "AWSAutoScalingAutoScalingGroupmasterustest1amastersminimalexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "master-us-test-1a.masters.minimal.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "1", + "MinSize": "1", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1aminimalexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/master", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSAutoScalingAutoScalingGroupnodesminimalexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "nodes.minimal.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatenodesminimalexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatenodesminimalexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "2", + "MinSize": "2", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1aminimalexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/node", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSEC2DHCPOptionsminimalexamplecom": { + "Type": "AWS::EC2::DHCPOptions", + "Properties": { + "DomainName": "us-test-1.compute.internal", + "DomainNameServers": [ + "AmazonProvidedDNS" + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2InternetGatewayminimalexamplecom": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "master-us-test-1a.masters.minimal.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp3", + "VolumeSize": 64, + "Iops": 4000, + "Throughput": 200, + "DeleteOnTermination": true, + "Encrypted": false + } + }, + { + "DeviceName": "/dev/sdc", + "VirtualName": "ephemeral0" + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilemastersminimalexamplecom" + } + }, + "ImageId": "ami-12345678", + "InstanceType": "m3.medium", + "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2LaunchTemplatenodesminimalexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "nodes.minimal.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp3", + "VolumeSize": 128, + "Iops": 3000, + "Throughput": 125, + "DeleteOnTermination": true, + "Encrypted": false + } + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilenodesminimalexamplecom" + } + }, + "ImageId": "ami-12345678", + "InstanceType": "t2.medium", + "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2Route00000": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "AWSEC2RouteTableminimalexamplecom" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "AWSEC2InternetGatewayminimalexamplecom" + } + } + }, + "AWSEC2RouteTableminimalexamplecom": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/kops/role", + "Value": "public" + } + ] + } + }, + "AWSEC2SecurityGroupEgressfrommastersminimalexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupEgressfromnodesminimalexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 443, + "ToPort": 443, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrommastersminimalexamplecomingressall0to0mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressfrommastersminimalexamplecomingressall0to0nodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingressall0to0nodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingresstcp1to2379mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 1, + "ToPort": 2379, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingresstcp2382to4000mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 2382, + "ToPort": 4000, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingresstcp4003to65535mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 4003, + "ToPort": 65535, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingressudp1to65535mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 1, + "ToPort": 65535, + "IpProtocol": "udp" + } + }, + "AWSEC2SecurityGroupmastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "masters.minimal.example.com", + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "GroupDescription": "Security group for masters", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "masters.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SecurityGroupnodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "nodes.minimal.example.com", + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "GroupDescription": "Security group for nodes", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SubnetRouteTableAssociationustest1aminimalexamplecom": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { + "Ref": "AWSEC2Subnetustest1aminimalexamplecom" + }, + "RouteTableId": { + "Ref": "AWSEC2RouteTableminimalexamplecom" + } + } + }, + "AWSEC2Subnetustest1aminimalexamplecom": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "CidrBlock": "172.20.32.0/19", + "AvailabilityZone": "us-test-1a", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.minimal.example.com" + }, + { + "Key": "SubnetType", + "Value": "Public" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + } + ] + } + }, + "AWSEC2VPCDHCPOptionsAssociationminimalexamplecom": { + "Type": "AWS::EC2::VPCDHCPOptionsAssociation", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "DhcpOptionsId": { + "Ref": "AWSEC2DHCPOptionsminimalexamplecom" + } + } + }, + "AWSEC2VPCGatewayAttachmentminimalexamplecom": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "InternetGatewayId": { + "Ref": "AWSEC2InternetGatewayminimalexamplecom" + } + } + }, + "AWSEC2VPCminimalexamplecom": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "172.20.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdeventsminimalexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp2", + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-events.minimal.example.com" + }, + { + "Key": "k8s.io/etcd/events", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdmainminimalexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp2", + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-main.minimal.example.com" + }, + { + "Key": "k8s.io/etcd/main", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSIAMInstanceProfilemastersminimalexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "masters.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemastersminimalexamplecom" + } + ] + } + }, + "AWSIAMInstanceProfilenodesminimalexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "nodes.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodesminimalexamplecom" + } + ] + } + }, + "AWSIAMPolicymastersminimalexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "masters.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemastersminimalexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DescribeVolumesModifications", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress" + ], + "Condition": { + "StringEquals": { + "ec2:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "iam:ListServerCertificates", + "iam:GetServerCertificate" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMPolicynodesminimalexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "nodes.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodesminimalexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMRolemastersminimalexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "masters.minimal.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "masters.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSIAMRolenodesminimalexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "nodes.minimal.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + } + } +} diff --git a/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml new file mode 100644 index 0000000000000..59b99c29e56b2 --- /dev/null +++ b/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml @@ -0,0 +1,550 @@ +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup + NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup + NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87 + + export AWS_REGION=us-test-1 + + + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, url1, url2... + download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: null + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 19.03.14 + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.19.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.19.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.19.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.19.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet + - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 9f1ec28e357a8f18e9561129239caf9c0807d74756e21cc63637c7fdeaafe847@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.14.tgz + arm64: + - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet + - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 8350eaa0c0965bb8eb9d45a014f4b6728c985715f56466077dfe6feb271d9518@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.14.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nodeLabels: + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml + protokubeImage: + amd64: + hash: 7b3c7f6adbda11b1ec740bd6b969c84f249b7eee818af95f2d321963088245a8 + name: protokube:1.19.0-alpha.3 + sources: + - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-amd64.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz + arm64: + hash: 69270ca9c1c950be65af40337adfccec0a728930fa3224bb0d2e88f181f39ead + name: protokube:1.19.0-alpha.3 + sources: + - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-arm64.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup + NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup + NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87 + + export AWS_REGION=us-test-1 + + + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, url1, url2... + download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: null + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 19.03.14 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.19.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet + - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 9f1ec28e357a8f18e9561129239caf9c0807d74756e21cc63637c7fdeaafe847@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.14.tgz + arm64: + - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet + - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 8350eaa0c0965bb8eb9d45a014f4b6728c985715f56466077dfe6feb271d9518@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.14.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" From 65ebf4760dbb7f1023a9b671de7635487b243599 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 13 Jan 2021 09:22:15 +0200 Subject: [PATCH 2/2] Update integration test for gp3 with etcd volumes --- .../minimal-gp3/cloudformation.json | 16 ++++++++-------- .../update_cluster/minimal-gp3/in-v1alpha2.yaml | 8 ++++++-- .../update_cluster/minimal-gp3/kubernetes.tf | 14 ++++++-------- upup/pkg/fi/cloudup/awstasks/ebsvolume.go | 8 +++++++- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/tests/integration/update_cluster/minimal-gp3/cloudformation.json b/tests/integration/update_cluster/minimal-gp3/cloudformation.json index 4007d4b5bd627..efe1aaea85309 100644 --- a/tests/integration/update_cluster/minimal-gp3/cloudformation.json +++ b/tests/integration/update_cluster/minimal-gp3/cloudformation.json @@ -215,10 +215,6 @@ "DeleteOnTermination": true, "Encrypted": false } - }, - { - "DeviceName": "/dev/sdc", - "VirtualName": "ephemeral0" } ], "IamInstanceProfile": { @@ -227,7 +223,7 @@ } }, "ImageId": "ami-12345678", - "InstanceType": "m3.medium", + "InstanceType": "m5.large", "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", "MetadataOptions": { "HttpPutResponseHopLimit": 1, @@ -349,7 +345,7 @@ } }, "ImageId": "ami-12345678", - "InstanceType": "t2.medium", + "InstanceType": "t3.large", "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", "MetadataOptions": { "HttpPutResponseHopLimit": 1, @@ -775,7 +771,9 @@ "Properties": { "AvailabilityZone": "us-test-1a", "Size": 20, - "VolumeType": "gp2", + "VolumeType": "gp3", + "Iops": 3000, + "Throughput": 125, "Encrypted": false, "Tags": [ { @@ -806,7 +804,9 @@ "Properties": { "AvailabilityZone": "us-test-1a", "Size": 20, - "VolumeType": "gp2", + "VolumeType": "gp3", + "Iops": 5000, + "Throughput": 125, "Encrypted": false, "Tags": [ { diff --git a/tests/integration/update_cluster/minimal-gp3/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal-gp3/in-v1alpha2.yaml index 3ccf940adefe9..69cefd5ecc005 100644 --- a/tests/integration/update_cluster/minimal-gp3/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/minimal-gp3/in-v1alpha2.yaml @@ -13,10 +13,14 @@ spec: - etcdMembers: - instanceGroup: master-us-test-1a name: us-test-1a + volumeIops: 5000 + volumeThroughput: 125 + volumeType: gp3 name: main - etcdMembers: - instanceGroup: master-us-test-1a name: us-test-1a + volumeType: gp3 name: events iam: {} kubelet: @@ -51,7 +55,7 @@ metadata: spec: associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium + machineType: t3.large rootVolumeType: gp3 maxSize: 2 minSize: 2 @@ -71,7 +75,7 @@ metadata: spec: associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium + machineType: m5.large maxSize: 1 minSize: 1 rootVolumeType: gp3 diff --git a/tests/integration/update_cluster/minimal-gp3/kubernetes.tf b/tests/integration/update_cluster/minimal-gp3/kubernetes.tf index d6dd118cd7a23..5ef48f991af4d 100644 --- a/tests/integration/update_cluster/minimal-gp3/kubernetes.tf +++ b/tests/integration/update_cluster/minimal-gp3/kubernetes.tf @@ -184,6 +184,7 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" { availability_zone = "us-test-1a" encrypted = false + iops = 3000 size = 20 tags = { "KubernetesCluster" = "minimal.example.com" @@ -192,12 +193,13 @@ resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" { "k8s.io/role/master" = "1" "kubernetes.io/cluster/minimal.example.com" = "owned" } - type = "gp2" + type = "gp3" } resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" { availability_zone = "us-test-1a" encrypted = false + iops = 5000 size = 20 tags = { "KubernetesCluster" = "minimal.example.com" @@ -206,7 +208,7 @@ resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" { "k8s.io/role/master" = "1" "kubernetes.io/cluster/minimal.example.com" = "owned" } - type = "gp2" + type = "gp3" } resource "aws_iam_instance_profile" "masters-minimal-example-com" { @@ -282,15 +284,11 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { volume_type = "gp3" } } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } iam_instance_profile { name = aws_iam_instance_profile.masters-minimal-example-com.id } image_id = "ami-12345678" - instance_type = "m3.medium" + instance_type = "m5.large" key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id lifecycle { create_before_destroy = true @@ -361,7 +359,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" { name = aws_iam_instance_profile.nodes-minimal-example-com.id } image_id = "ami-12345678" - instance_type = "t2.medium" + instance_type = "t3.large" key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id lifecycle { create_before_destroy = true diff --git a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go index e2f733b30bf67..cc2fe6f424c75 100644 --- a/upup/pkg/fi/cloudup/awstasks/ebsvolume.go +++ b/upup/pkg/fi/cloudup/awstasks/ebsvolume.go @@ -227,12 +227,18 @@ type terraformVolume struct { } func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EBSVolume) error { + // TODO: Remove when Terraform gets support for "throughput" with "aws_ebs_volume" + // https://github.com/hashicorp/terraform-provider-aws/pull/16517 + throughput := e.VolumeThroughput + if fi.Int64Value(e.VolumeThroughput) <= 125 { + throughput = nil + } tf := &terraformVolume{ AvailabilityZone: e.AvailabilityZone, Size: e.SizeGB, Type: e.VolumeType, Iops: e.VolumeIops, - Throughput: e.VolumeThroughput, + Throughput: throughput, KmsKeyId: e.KmsKeyId, Encrypted: e.Encrypted, Tags: e.Tags,