Private Commit for Azure Console Shell

Remove SPN secrets from agent node
Remove the Kube Dashboard and Heapster Addons
Add agentpool label on the agent nodes
Use static IP address for system and agentpool1

parts/kubernetesagentcustomdata.yml

@@ -121,7 +121,7 @@ write_files:
   encoding: gzip
   owner: "root"
   content: !!binary |
-    KUBELET_SERVICE_B64_GZIP_STR
+    KUBELET_SERVICE_AGENT_B64_GZIP_STR
 
 - path: "/opt/azure/containers/kubelet.sh"
   permissions: "0755"
@@ -136,7 +136,7 @@ write_files:
   encoding: gzip
   owner: "root"
   content: !!binary |
-    {{WrapAsVariable "provisionScript"}}
+    {{WrapAsVariable "agentProvisionScript"}}
 
 runcmd:
 - apt-get update
@@ -153,5 +153,4 @@ runcmd:
 - apt-get install -y docker-engine
 - systemctl restart docker
 - mkdir -p /etc/kubernetes/manifests
-- usermod -aG docker {{WrapAsVariable "username"}}
-
+- usermod -aG docker {{WrapAsVariable "username"}}

parts/kubernetesagentcustomscript.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+###########################################################
+# START SECRET DATA - ECHO DISABLED
+###########################################################
+
+# Fields for `azure.json`
+KUBELET_PRIVATE_KEY="${1}"
+NETWORK_POLICY="${2}"
+
+KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key"
+touch "${KUBELET_PRIVATE_KEY_PATH}"
+chmod 0644 "${KUBELET_PRIVATE_KEY_PATH}"
+chown root:root "${KUBELET_PRIVATE_KEY_PATH}"
+echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"
+
+###########################################################
+# END OF SECRET DATA
+###########################################################
+
+set -x
+
+function ensureDocker() {
+    systemctl enable docker
+    systemctl restart docker
+    dockerStarted=1
+    for i in {1..600}; do
+        if ! /usr/bin/docker info; then
+            echo "status $?"
+            /bin/systemctl restart docker
+        else
+            echo "docker started"
+            dockerStarted=0
+            break
+        fi
+        sleep 1
+    done
+    if [ $dockerStarted -ne 0 ]
+    then
+        echo "docker did not start"
+        exit 1
+    fi
+}
+
+function setAgentPool() {
+    AGENTPOOL=`hostname | cut -d- -f2`
+    sed -i "s/^KUBELET_NODE_LABELS=.*/KUBELET_NODE_LABELS=role=agent,agentpool=${AGENTPOOL}/" /etc/default/kubelet
+}
+
+function ensureKubelet() {
+    systemctl enable kubelet
+    systemctl restart kubelet
+}
+
+function setNetworkPlugin () { 
+    sed -i "s/^KUBELET_NETWORK_PLUGIN=.*/KUBELET_NETWORK_PLUGIN=${1}/" /etc/default/kubelet 
+}
+
+function setDockerOpts () { 
+    sed -i "s#^DOCKER_OPTS=.*#DOCKER_OPTS=${1}#" /etc/default/kubelet 
+} 
+
+function configNetworkPolicy() { 
+    if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then 
+        # on masters 
+        ADDONS="calico-configmap.yaml calico-daemonset.yaml" 
+        ADDONS_PATH=/etc/kubernetes/addons 
+        CALICO_URL="https://github.com/simonswine/calico/raw/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager" 
+        if [[ "${NETWORK_POLICY}" = "calico" ]]; then 
+            # download calico yamls 
+            for addon in ${ADDONS}; do 
+                curl -o "${ADDONS_PATH}/${addon}" -sSL --retry 12 --retry-delay 10 "${CALICO_URL}/${addon}" 
+            done 
+        else 
+            # make sure calico yaml are removed 
+            for addon in ${ADDONS}; do 
+                rm -f "${ADDONS_PATH}/${addon}" 
+            done 
+        fi 
+    else 
+        # on agents 
+        if [[ "${NETWORK_POLICY}" = "calico" ]]; then 
+            setNetworkPlugin cni 
+            setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro" 
+        else 
+            setNetworkPlugin kubenet 
+            setDockerOpts "" 
+        fi 
+    fi 
+} 
+
+ensureDocker
+configNetworkPolicy
+setAgentPool
+ensureKubelet
+
+echo "Install complete successfully"
+

parts/kubernetesagentkubelet.service

@@ -0,0 +1,51 @@
+[Unit]
+Description=Kubelet
+Requires=docker.service
+After=docker.service
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/default/kubelet
+SuccessExitStatus=143
+ExecStartPre=/bin/bash /opt/azure/containers/kubelet.sh
+ExecStartPre=/bin/mkdir -p /var/lib/kubelet
+ExecStartPre=/bin/bash -c "if [ $(mount | grep \"/var/lib/kubelet\" | wc -l) -le 0 ] ; then /bin/mount --bind /var/lib/kubelet /var/lib/kubelet ; fi"
+ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
+ExecStartPre=-/sbin/ebtables -t nat --list
+ExecStartPre=-/sbin/iptables -t nat --list
+ExecStart=/usr/bin/docker run \
+  --net=host \
+  --pid=host \
+  --privileged \
+  --rm \
+  --volume=/dev:/dev \
+  --volume=/sys:/sys:ro \
+  --volume=/var/run:/var/run:rw \
+  --volume=/var/lib/docker/:/var/lib/docker:rw \
+  --volume=/var/lib/kubelet/:/var/lib/kubelet:shared \
+  --volume=/var/log:/var/log:rw \
+  --volume=/etc/kubernetes/:/etc/kubernetes:ro \
+  --volume=/srv/kubernetes/:/srv/kubernetes:ro $DOCKER_OPTS \
+    ${KUBELET_IMAGE} \
+      /hyperkube kubelet \
+        --kubeconfig=/var/lib/kubelet/kubeconfig \
+        --require-kubeconfig \
+        --pod-infra-container-image="${KUBELET_POD_INFRA_CONTAINER_IMAGE}" \
+        --address=0.0.0.0 \
+        --allow-privileged=true \
+        --enable-server \
+        --enable-debugging-handlers \
+        --pod-manifest-path=/etc/kubernetes/manifests \
+        --cluster-dns=${KUBELET_CLUSTER_DNS} \
+        --cluster-domain=cluster.local \
+        --register-schedulable=${KUBELET_REGISTER_SCHEDULABLE} \
+        --node-labels=${KUBELET_NODE_LABELS} \
+        --cloud-provider= \
+        --cloud-config= \
+        --azure-container-registry-config= \
+        --hairpin-mode=promiscuous-bridge \
+        --network-plugin=${KUBELET_NETWORK_PLUGIN} \
+        --v=2
+
+[Install]
+WantedBy=multi-user.target

parts/kubernetesagentresourcesvmas.t

@@ -27,7 +27,15 @@
               {{if eq $seq 1}}
               "primary": true,
               {{end}}
+              {{if eq $.Name "system"}}
+              "privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(50, int(variables('masterFirstAddrOctet4')))))]",
+              "privateIPAllocationMethod": "Static",
+              {{else if eq $.Name "agentpool1"}}
+              "privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(100, int(variables('masterFirstAddrOctet4')))))]",
+              "privateIPAllocationMethod": "Static",
+              {{else}}
               "privateIPAllocationMethod": "Dynamic",
+              {{end}}
               "subnet": {
                 "id": "[variables('{{$.Name}}VnetSubnetID')]"
               }
@@ -205,7 +213,7 @@
         "autoUpgradeMinorVersion": true,
         "settings": {},
         "protectedSettings": {
-          "commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('tenantID'),' ',variables('subscriptionId'),' ',variables('resourceGroup'),' ',variables('location'),' ',variables('subnetName'),' ',variables('nsgName'),' ',variables('virtualNetworkName'),' ',variables('routeTableName'),' ',variables('primaryAvailablitySetName'),' ',variables('servicePrincipalClientId'),' ',variables('servicePrincipalClientSecret'),' ',variables('clientPrivateKey'),' ',variables('targetEnvironment'),' ',variables('networkPolicy'),' >> /var/log/azure/cluster-provision.log 2>&1 &\" &')]"
+          "commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('clientPrivateKey'),' ',variables('networkPolicy'),'>> /var/log/azure/agent-provision.log 2>&1 &\" &')]"
         }
       }
     }

parts/kubernetesagentvars.t

@@ -18,3 +18,4 @@
     "{{.Name}}VnetSubnetID": "[variables('vnetSubnetID')]",
     "{{.Name}}SubnetName": "[variables('subnetName')]",
 {{end}}
+    "agentProvisionScript": "{{GetKubernetesAgentB64Provision}}",

parts/kubernetesmasteraddons-kube-dns-deployment.yaml

@@ -100,4 +100,5 @@ spec:
             memory: 50Mi
       dnsPolicy: Default
       nodeSelector:
-        beta.kubernetes.io/os: linux
+        beta.kubernetes.io/os: linux
+        agentpool: system

parts/kubernetesmastercustomdata.yml

@@ -152,34 +152,6 @@ write_files:
   content: !!binary |
     MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR
 
-- path: /etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml
-  permissions: "0644"
-  encoding: gzip
-  owner: "root"
-  content: !!binary |
-    MASTER_ADDON_KUBERNETES_DASHBOARD_DEPLOYMENT_B64_GZIP_STR
-
-- path: /etc/kubernetes/addons/kubernetes-dashboard-service.yaml
-  permissions: "0644"
-  encoding: gzip
-  owner: "root"
-  content: !!binary |
-    MASTER_ADDON_KUBERNETES_DASHBOARD_SERVICE_B64_GZIP_STR
-
-- path: /etc/kubernetes/addons/kube-heapster-service.yaml
-  permissions: "0644"
-  encoding: gzip
-  owner: "root"
-  content: !!binary |
-    MASTER_ADDON_HEAPSTER_SERVICE_B64_GZIP_STR
-
-- path: /etc/kubernetes/addons/kube-heapster-deployment.yaml
-  permissions: "0644"
-  encoding: gzip
-  owner: "root"
-  content: !!binary |
-    MASTER_ADDON_HEAPSTER_DEPLOYMENT_B64_GZIP_STR
-
 - path: /etc/kubernetes/addons/default-storage-class.yaml
   permissions: "0644"
   encoding: gzip
@@ -248,8 +220,6 @@ write_files:
     sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
     sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
     sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsVariable "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsVariable "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsVariable "kubernetesExecHealthzSpec"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
-    sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsVariable "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsVariable "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
-    sed -i "s|<kubernetesDashboardSpec>|{{WrapAsVariable "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
 
     echo $(curl -f --retry 5 http://169.254.169.254/metadata/v1/InstanceInfo 2>/dev/null | jq -r .FD) > /etc/kubernetes/fd
     echo "{{WrapAsVariable "masterVMSize"}}" > /etc/kubernetes/vmsize

parts/kubernetesmastercustomscript.sh

@@ -331,6 +331,5 @@ if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then
     ensureApiserver
 fi
 
-# If APISERVER_PRIVATE_KEY is empty, then we are not on the master
 echo "Install complete successfully"
 

parts/kubernetesmastervars.t

@@ -91,7 +91,7 @@
 {{end}}
     "nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]",
     "nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]",
-    "primaryAvailablitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',variables('nameSuffix'))]",
+    "primaryAvailablitySetName": "[concat('{{ (index .AgentPoolProfiles 1).Name }}-availabilitySet-',variables('nameSuffix'))]",
     "masterPublicIPAddressName": "[concat(variables('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', variables('nameSuffix'))]",
     "masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]", 
     "masterLbIPConfigID": "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]", 

pkg/acsengine/azureconst.go

@@ -1110,4 +1110,4 @@ func GetClassicSizeMap() string {
       }
     }	
 `
-}
+}