Merge pull request #1388 from jsturtevant/windows-custom-binaries

Inject Windows custom binaries for use in PRs and running against Kubernetes CI

Makefile

@@ -49,7 +49,7 @@ GO_INSTALL = ./scripts/go_install.sh
 E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data
 KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
 KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml)
-KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml)
+KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/)
 AZURE_TEMPLATES := $(E2E_DATA_DIR)/infrastructure-azure
 
 # set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev
@@ -130,7 +130,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
 SKIP_CREATE_MGMT_CLUSTER ?= false
-WIN_REPO_LIST ?= https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list
+WIN_REPO_URL ?=
 
 # Build time versioning details.
 LDFLAGS := $(shell hack/version.sh)
@@ -202,8 +202,10 @@ test-conformance-fast: ## Run conformance test on workload cluster using a subse
 
 .PHONY: test-windows-upstream
 test-windows-upstream: ## Run windows upstream tests on workload cluster.
-	curl --retry $(CURL_RETRIES) $(WIN_REPO_LIST) -o $(KUBETEST_REPO_LIST_PATH)
-	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
+ifneq ($(WIN_REPO_URL), )
+	curl --retry $(CURL_RETRIES) $(WIN_REPO_URL) -o $(KUBETEST_REPO_LIST_PATH)/custom-repo-list.yaml
+endif
+	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-path=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
 
 $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools

scripts/ci-build-kubernetes.sh

@@ -38,6 +38,7 @@ source "${REPO_ROOT}/hack/parse-prow-creds.sh"
 : "${JOB_NAME:?Environment variable empty or not defined.}"
 
 declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
+declare -a WINDOWS_BINARIES=("kubeadm" "kubectl" "kubelet" "kube-proxy")
 declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
 
 setup() {
@@ -100,7 +101,19 @@ main() {
         for BINARY in "${BINARIES[@]}"; do
             az storage blob upload --container-name "${JOB_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/${BINARY}" --name "${KUBE_GIT_VERSION}/bin/linux/amd64/${BINARY}"
         done
-    fi
+
+        if [[ "${WINDOWS:-}" == "true" ]]; then
+            echo "Building Kubernetes Windows binaries"
+
+            for BINARY in "${WINDOWS_BINARIES[@]}"; do
+                "${KUBE_ROOT}"/build/run.sh make WHAT=cmd/"${BINARY}" KUBE_BUILD_PLATFORMS=windows/amd64 KUBE_VERBOSE=0
+            done
+
+            for BINARY in "${WINDOWS_BINARIES[@]}"; do
+                az storage blob upload --container-name "${JOB_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/windows/amd64/${BINARY}.exe" --name "${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe"
+            done
+        fi
+    fi 
 }
 
 # can_reuse_artifacts returns true if there exists Kubernetes artifacts built from a PR that we can reuse
@@ -117,6 +130,14 @@ can_reuse_artifacts() {
         fi
     done
 
+    if [[ "${WINDOWS:-}" == "true" ]]; then
+        for BINARY in "${WINDOWS_BINARIES[@]}"; do
+            if [[ "$(az storage blob exists --container-name "${JOB_NAME}" --name "${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe" --query exists)" == "false" ]]; then
+                echo "false" && return
+            fi
+        done
+    fi
+
     echo "true"
 }
 

templates/addons/windows/flannel.yaml

@@ -47,7 +47,7 @@ spec:
     rule: 'RunAsAny'
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 rules:
@@ -76,7 +76,7 @@ rules:
       - patch
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 roleRef:

templates/addons/windows/kube-proxy-windows.yaml

@@ -5,6 +5,12 @@ data:
     mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
     mkdir -force /host/k/kube-proxy
 
+    $$CI_VERSION="${CI_VERSION:-}"
+    if($$CI_VERSION -ne "" -And (Test-Path -Path "/host/k/kube-proxy.exe"))
+    {
+      cp -force /host/k/kube-proxy.exe /k/kube-proxy/kube-proxy.exe
+    }
+
     cp -force /k/kube-proxy/* /host/k/kube-proxy
     cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
@@ -57,7 +63,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-        image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver
+        image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-nanoserver
         name: kube-proxy
         volumeMounts:
         - name: wins

templates/cluster-template-aad.yaml

@@ -115,6 +115,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
     useExperimentalRetryJoin: true
   machineTemplate:
     infrastructureRef:

templates/cluster-template-azure-bastion.yaml

@@ -112,6 +112,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -199,6 +201,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

templates/cluster-template-ephemeral.yaml

@@ -110,6 +110,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -203,6 +205,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

templates/cluster-template-external-cloud-provider.yaml

@@ -111,6 +111,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -198,6 +200,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: external
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

templates/cluster-template-ipv6.yaml

@@ -151,6 +151,7 @@ spec:
     - mv /etc/resolv.conf /etc/resolv.conf.OLD && ln -s /run/systemd/resolve/resolv.conf
       /etc/resolv.conf
     - systemctl restart systemd-resolved
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-machinepool-multiple-subnets.yaml

@@ -117,6 +117,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-machinepool-system-assigned-identity.yaml

@@ -106,6 +106,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-machinepool-user-assigned-identity.yaml

@@ -106,6 +106,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-machinepool-windows.yaml

@@ -127,6 +127,7 @@ spec:
     - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
     - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
     - netplan apply
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-machinepool.yaml

@@ -110,6 +110,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-nat-gateway.yaml

@@ -123,6 +123,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -210,6 +212,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

templates/cluster-template-nvidia-gpu.yaml

@@ -111,6 +111,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4

templates/cluster-template-private.yaml

@@ -213,6 +213,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

templates/cluster-template-system-assigned-identity.yaml

@@ -106,6 +106,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -195,3 +197,4 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []

templates/cluster-template-user-assigned-identity.yaml

@@ -106,6 +106,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -199,3 +201,4 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []

templates/cluster-template-windows.yaml

@@ -127,6 +127,7 @@ spec:
     - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
     - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
     - netplan apply
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -160,7 +161,7 @@ metadata:
   namespace: default
 spec:
   clusterName: ${CLUSTER_NAME}
-  replicas: ${WORKER_MACHINE_COUNT}
+  replicas: ${LINUX_WORKER_MACHINE_COUNT:-1}
   selector:
     matchLabels: null
   template:
@@ -232,6 +233,7 @@ spec:
       - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
       - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
       - netplan apply
+      preKubeadmCommands: []
       useExperimentalRetryJoin: true
 ---
 apiVersion: cluster.x-k8s.io/v1alpha4

templates/cluster-template.yaml

@@ -110,6 +110,8 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands: []
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -197,6 +199,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

templates/flavors/base/cluster-template.yaml

@@ -109,6 +109,8 @@ spec:
     mounts:
       - - LABEL=etcd_disk
         - /var/lib/etcddisk
+    preKubeadmCommands: []
+    postKubeadmCommands: []
   version: "${KUBERNETES_VERSION}"
 ---
 kind: AzureMachineTemplate

templates/flavors/default/machine-deployment.yaml

@@ -42,6 +42,7 @@ metadata:
 spec:
   template:
     spec:
+      preKubeadmCommands: []
       joinConfiguration:
         nodeRegistration:
           name: '{{ ds.meta_data["local_hostname"] }}'

templates/flavors/windows/machine-deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   name: "${CLUSTER_NAME}-md-0"
 spec:
   clusterName: "${CLUSTER_NAME}"
-  replicas: ${WORKER_MACHINE_COUNT}
+  replicas: ${LINUX_WORKER_MACHINE_COUNT:-1}
   selector:
     matchLabels:
   template:
@@ -44,6 +44,7 @@ metadata:
 spec:
   template:
     spec:
+      preKubeadmCommands: []
       postKubeadmCommands:
         # Azures vnet MTU is 1400. 
         # When using Flannel VXLAN to avoid packet fragmentation