Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

Commit

Permalink
Revert "Revert "Cherry pick: check network to k8s api server (#2965)""
Browse files Browse the repository at this point in the history 
This reverts commit b4cd0bc.
  • Loading branch information
Tony Xu committed Mar 27, 2020
1 parent d0cfc27 commit df57f0f
Show file tree
Hide file tree
Showing 9 changed files with 80 additions and 15 deletions.
1 change: 1 addition & 0 deletions parts/k8s/cloud-init/artifacts/cse_helpers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 {{/* Failed to add repo pkg file */}}
ERR_SYSTEMD_INSTALL_FAIL=48 {{/* Unable to install required systemd version */}}
ERR_MODPROBE_FAIL=49 {{/* Unable to load a kernel module using modprobe */}}
ERR_OUTBOUND_CONN_FAIL=50 {{/* Unable to establish outbound connection */}}
ERR_K8S_API_SERVER_CONN_FAIL=51 {{/* Unable to establish connection to k8s api server*/}}
ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60 {{/* Timeout waiting to download kata repo key */}}
ERR_KATA_APT_KEY_TIMEOUT=61 {{/* Timeout waiting for kata apt-key */}}
ERR_KATA_INSTALL_TIMEOUT=62 {{/* Timeout waiting for kata install */}}
Expand Down
4 changes: 4 additions & 0 deletions parts/k8s/cloud-init/artifacts/cse_main.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,10 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
fi
{{end}}

{{- if IsHostedMaster }}
retrycmd_if_failure 50 1 3 nc -vz ${API_SERVER_IP} 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL
{{end}}

if $REBOOTREQUIRED; then
echo 'reboot required, rebooting node in 1 minute'
/bin/bash -c "shutdown -r 1 &"
Expand Down
10 changes: 5 additions & 5 deletions pkg/engine/armresources_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func TestGenerateARMResourcesWithVMSSAgentPool(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -556,7 +556,7 @@ func TestGenerateARMResourcesWithVMSSAgentPool(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -856,7 +856,7 @@ func TestGenerateARMResourceWithVMASAgents(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: &map[string]interface{}{},
ProtectedSettings: &map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`},
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`},
},
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), copyIndex(variables('agentpool1Offset')),'/cse', '-agent-', copyIndex(variables('agentpool1Offset')))]"),
Type: to.StringPtr("Microsoft.Compute/virtualMachines/extensions"),
Expand Down Expand Up @@ -1354,7 +1354,7 @@ func TestGenerateARMResourcesWithVMSSAgentPoolAndSLB(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -1979,7 +1979,7 @@ func TestGenerateARMResourcesWithVMSSAgentPoolAndSLB(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down
49 changes: 48 additions & 1 deletion pkg/engine/armvariables_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,30 @@ func TestK8sVars(t *testing.T) {
if err != nil {
t.Fatal(err)
}
expectedMap["cloudInitFiles"] = map[string]interface{}{
"provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, cs),
"provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, cs),
"provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, cs),
"provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, cs),
"provisionCIS": getBase64EncodedGzippedCustomScript(kubernetesCISScript, cs),
"healthMonitorScript": getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript, cs),
"customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, cs),
"generateProxyCertsScript": getBase64EncodedGzippedCustomScript(kubernetesMasterGenerateProxyCertsScript, cs),
"mountEtcdScript": getBase64EncodedGzippedCustomScript(kubernetesMountEtcd, cs),
"kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, cs),
"kmsSystemdService": getBase64EncodedGzippedCustomScript(kmsSystemdService, cs),
"kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService, cs),
"dockerMonitorSystemdTimer": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdTimer, cs),
"dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs),
"labelNodesScript": getBase64EncodedGzippedCustomScript(labelNodesScript, cs),
"labelNodesSystemdService": getBase64EncodedGzippedCustomScript(labelNodesSystemdService, cs),
"aptPreferences": getBase64EncodedGzippedCustomScript(aptPreferences, cs),
"dockerClearMountPropagationFlags": getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags, cs),
"auditdRules": getBase64EncodedGzippedCustomScript(auditdRules, cs),
"etcdSystemdService": getBase64EncodedGzippedCustomScript(etcdSystemdService, cs),
"dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, cs),
"dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, cs),
}
expectedMap["agentNamePrefix"] = "[concat(parameters('orchestratorName'), '-agentpool-', parameters('nameSuffix'), '-')]"
expectedMap["agentpool1AccountName"] = "[concat(variables('storageAccountBaseName'), 'agnt0')]"
expectedMap["agentpool1StorageAccountOffset"] = "[mul(variables('maxStorageAccountsPerAgent'),variables('agentpool1Index'))]"
Expand Down Expand Up @@ -432,7 +456,30 @@ func TestK8sVars(t *testing.T) {
if err != nil {
t.Fatal(err)
}

expectedMap["cloudInitFiles"] = map[string]interface{}{
"provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, cs),
"provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, cs),
"provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, cs),
"provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, cs),
"provisionCIS": getBase64EncodedGzippedCustomScript(kubernetesCISScript, cs),
"healthMonitorScript": getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript, cs),
"customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, cs),
"generateProxyCertsScript": getBase64EncodedGzippedCustomScript(kubernetesMasterGenerateProxyCertsScript, cs),
"mountEtcdScript": getBase64EncodedGzippedCustomScript(kubernetesMountEtcd, cs),
"kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, cs),
"kmsSystemdService": getBase64EncodedGzippedCustomScript(kmsSystemdService, cs),
"kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService, cs),
"dockerMonitorSystemdTimer": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdTimer, cs),
"dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs),
"labelNodesScript": getBase64EncodedGzippedCustomScript(labelNodesScript, cs),
"labelNodesSystemdService": getBase64EncodedGzippedCustomScript(labelNodesSystemdService, cs),
"aptPreferences": getBase64EncodedGzippedCustomScript(aptPreferences, cs),
"dockerClearMountPropagationFlags": getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags, cs),
"auditdRules": getBase64EncodedGzippedCustomScript(auditdRules, cs),
"etcdSystemdService": getBase64EncodedGzippedCustomScript(etcdSystemdService, cs),
"dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, cs),
"dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, cs),
}
expectedMap["useManagedIdentityExtension"] = "true"
expectedMap["provisionScriptParametersCommon"] = "[concat('" + cs.GetProvisionScriptParametersCommon(api.ProvisionScriptParametersInput{Location: common.WrapAsARMVariable("location"), ResourceGroup: common.WrapAsARMVariable("resourceGroup"), TenantID: common.WrapAsARMVariable("tenantID"), SubscriptionID: common.WrapAsARMVariable("subscriptionId"), ClientID: common.WrapAsARMVariable("servicePrincipalClientId"), ClientSecret: common.WrapAsARMVariable("singleQuote") + common.WrapAsARMVariable("servicePrincipalClientSecret") + common.WrapAsARMVariable("singleQuote"), APIServerCertificate: common.WrapAsParameter("apiServerCertificate"), KubeletPrivateKey: common.WrapAsParameter("clientPrivateKey"), ClusterKeyVaultName: common.WrapAsARMVariable("clusterKeyVaultName")}) + "')]"

Expand Down
5 changes: 5 additions & 0 deletions pkg/engine/templates_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pkg/engine/virtualmachinescalesets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -743,7 +743,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
auditDEnabled := strconv.FormatBool(to.Bool(profile.AuditDEnabled))
isVHD := strconv.FormatBool(profile.IsVHDDistro())

commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground)
commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground)
vmssCSE = compute.VirtualMachineScaleSetExtension{
Name: to.StringPtr("vmssCSE"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Expand Down
Loading

0 comments on commit df57f0f

Please sign in to comment.