diff --git a/Makefile b/Makefile index cab5fa4bed6..989f331fdc3 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,8 @@ EXP_DIR := exp GO_INSTALL = ./scripts/go_install.sh E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml) +KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml) +KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml) # set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev ifneq ($(abspath $(ROOT_DIR)),$(GOPATH)/src/sigs.k8s.io/cluster-api-provider-azure) @@ -127,6 +129,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml SKIP_CLEANUP ?= false SKIP_CREATE_MGMT_CLUSTER ?= false +WIN_REPO_LIST ?= https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list # Build time versioning details. LDFLAGS := $(shell hack/version.sh) @@ -196,6 +199,11 @@ test-conformance: ## Run conformance test on workload cluster. test-conformance-fast: ## Run conformance test on workload cluster using a subset of the conformance suite in parallel. $(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_FAST_CONF_PATH) -kubetest.ginkgo-nodes=5 $(E2E_ARGS)" +.PHONY: test-windows-upstream +test-windows-upstream: ## Run windows upstream tests on workload cluster. + curl --retry $(CURL_RETRIES) $(WIN_REPO_LIST) -o $(KUBETEST_REPO_LIST_PATH) + $(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)" + $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd source ./scripts/fetch_ext_bins.sh && fetch_tools diff --git a/docs/book/src/developers/development.md b/docs/book/src/developers/development.md index 2fdcc91b130..780ae4f0f97 100644 --- a/docs/book/src/developers/development.md +++ b/docs/book/src/developers/development.md @@ -416,23 +416,31 @@ To run the Kubernetes Conformance test suite locally, you can run ./scripts/ci-conformance.sh ``` -With the following environment variables defined, you can build a CAPZ cluster from the HEAD of Kubernetes main branch or release branch, and run the Conformance test suite against it: +Optional settings are: -| Environment Variable | Value | -|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `E2E_ARGS` | `-kubetest.use-ci-artifacts` | +| Environment Variable | Default Value | Description | +|----------------------|-----------------|-------------| +| `WINDOWS` | `false` | Run conformance against Windows nodes | +| `CONFORMANCE_NODES` | `1` |Number of parallel ginkgo nodes to run | + +With the following environment variables defined, you can build a CAPZ cluster from the HEAD of Kubernetes main branch or release branch, and run the Conformance test suite against it. This is not enabled for Windows currently. + +| Environment Variable | Value | +|----------------------|--------| +| `E2E_ARGS` | `-kubetest.use-ci-artifacts` | | `KUBERNETES_VERSION` | `latest` - extract Kubernetes version from https://dl.k8s.io/ci/latest.txt (main's HEAD)
`latest-1.21` - extract Kubernetes version from https://dl.k8s.io/ci/latest-1.21.txt (release branch's HEAD) | + With the following environment variables defined, CAPZ runs `./scripts/ci-build-kubernetes.sh` as part of `./scripts/ci-conformance.sh`, which allows developers to build Kubernetes from source and run the Kubernetes Conformance test suite against a CAPZ cluster based on the custom build: -| Environment Variable | Value | -|-------------------------|-------------------------------------------------------------------------| -| `AZURE_STORAGE_ACCOUNT` | Your Azure storage account name | -| `AZURE_STORAGE_KEY` | Your Azure storage key | +| Environment Variable | Value | +|-------------------------|------------| +| `AZURE_STORAGE_ACCOUNT` | Your Azure storage account name | +| `AZURE_STORAGE_KEY` | Your Azure storage key | | `JOB_NAME` | `test` (an enviroment variable used by CI, can be any non-empty string) | -| `LOCAL_ONLY` | `false` | -| `REGISTRY` | Your Registry | -| `TEST_K8S` | `true` | +| `LOCAL_ONLY` | `false` | +| `REGISTRY` | Your Registry | +| `TEST_K8S` | `true` | #### Running custom test suites on CAPZ clusters diff --git a/scripts/ci-conformance.sh b/scripts/ci-conformance.sh index 2d877e7ffe2..a3a92c6e10b 100755 --- a/scripts/ci-conformance.sh +++ b/scripts/ci-conformance.sh @@ -76,6 +76,7 @@ export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr - export AZURE_LOCATION="${AZURE_LOCATION:-$(get_random_region)}" export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:-"Standard_D2s_v3"}" export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:-"Standard_D2s_v3"}" +export WINDOWS="${WINDOWS:-false}" # Generate SSH key. AZURE_SSH_PUBLIC_KEY_FILE=${AZURE_SSH_PUBLIC_KEY_FILE:-""} @@ -93,4 +94,8 @@ cleanup() { trap cleanup EXIT -make test-conformance +if [[ "${WINDOWS}" == "true" ]]; then + make test-windows-upstream +else + make test-conformance +fi diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml index 2b5534ba28f..3670fcd1c41 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml @@ -290,6 +290,11 @@ spec: New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) path: C:/create-external-network.ps1 permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: kubeletExtraArgs: @@ -302,6 +307,7 @@ spec: - nssm set kubelet start SERVICE_AUTO_START preKubeadmCommands: - powershell c:/create-external-network.ps1 + - powershell C:/create-temp-folder.ps1 users: - groups: Administrators name: capi diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml index 10a8a17c0c8..950689a3dd9 100644 --- a/templates/test/ci/cluster-template-prow-windows.yaml +++ b/templates/test/ci/cluster-template-prow-windows.yaml @@ -298,6 +298,11 @@ spec: New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) path: C:/create-external-network.ps1 permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: kubeletExtraArgs: @@ -310,6 +315,7 @@ spec: - nssm set kubelet start SERVICE_AUTO_START preKubeadmCommands: - powershell c:/create-external-network.ps1 + - powershell C:/create-temp-folder.ps1 users: - groups: Administrators name: capi diff --git a/templates/test/ci/prow-machine-pool-windows/kustomization.yaml b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml index 93d30b90f12..b419a26b501 100644 --- a/templates/test/ci/prow-machine-pool-windows/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml @@ -8,6 +8,14 @@ patchesStrategicMerge: - ../patches/tags.yaml - ../patches/cluster-cni-windows.yaml - ../patches/controller-manager.yaml +patchesJson6902: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + namespace: default + path: patches/windows-tmp-folder.yaml configMapGenerator: - name: cni-${CLUSTER_NAME}-flannel files: diff --git a/templates/test/ci/prow-machine-pool-windows/patches/windows-tmp-folder.yaml b/templates/test/ci/prow-machine-pool-windows/patches/windows-tmp-folder.yaml new file mode 100644 index 00000000000..3a1e144a4d2 --- /dev/null +++ b/templates/test/ci/prow-machine-pool-windows/patches/windows-tmp-folder.yaml @@ -0,0 +1,12 @@ +- op: add + path: /spec/files/- + value: + content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/create-temp-folder.ps1 diff --git a/templates/test/ci/prow-windows/kustomization.yaml b/templates/test/ci/prow-windows/kustomization.yaml index 6093305e9cd..cb85ba22760 100644 --- a/templates/test/ci/prow-windows/kustomization.yaml +++ b/templates/test/ci/prow-windows/kustomization.yaml @@ -8,6 +8,14 @@ patchesStrategicMerge: - ../patches/tags.yaml - ../patches/cluster-cni-windows.yaml - ../patches/controller-manager.yaml +patchesJson6902: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1alpha4 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + namespace: default + path: patches/windows-tmp-folder.yaml configMapGenerator: - name: cni-${CLUSTER_NAME}-flannel files: diff --git a/templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml b/templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml new file mode 100644 index 00000000000..38ec14d1d15 --- /dev/null +++ b/templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml @@ -0,0 +1,12 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/create-temp-folder.ps1 diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index af63752b415..40bebeb74a7 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -24,6 +24,11 @@ import ( "os" "path/filepath" "strconv" + "strings" + + "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/node" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -75,18 +80,28 @@ var _ = Describe("Conformance Tests", func() { kubernetesVersion := e2eConfig.GetVariable(capi_e2e.KubernetesVersion) flavor := clusterctl.DefaultFlavor + if isWindows(kubetestConfigFilePath) { + flavor = "windows" + } + // clusters with CI artifacts or PR artifacts are based on a known CI version // PR artifacts will replace the CI artifacts during kubeadm init if useCIArtifacts || usePRArtifacts { kubernetesVersion, err = resolveCIVersion(kubernetesVersion) Expect(err).NotTo(HaveOccurred()) Expect(os.Setenv("CI_VERSION", kubernetesVersion)).To(Succeed()) + + if useCIArtifacts { + flavor = "conformance-ci-artifacts" + } else if usePRArtifacts { + flavor = "conformance-presubmit-artifacts" + } + + if isWindows(kubetestConfigFilePath) { + flavor = flavor + "-windows" + } } - if useCIArtifacts { - flavor = "conformance-ci-artifacts" - } else if usePRArtifacts { - flavor = "conformance-presubmit-artifacts" - } + workerMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_WORKER_MACHINE_COUNT"), 10, 64) Expect(err).NotTo(HaveOccurred()) controlPlaneMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT"), 10, 64) @@ -118,19 +133,37 @@ var _ = Describe("Conformance Tests", func() { b.RecordValue("cluster creation", runtime.Seconds()) workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName) + // Windows requires a taint on control nodes nodes since not all conformance tests have ability to run + if isWindows(kubetestConfigFilePath) { + options := v1.ListOptions{ + LabelSelector: "kubernetes.io/os=linux", + } + + noScheduleTaint := &corev1.Taint{ + Key: "node-role.kubernetes.io/master", + Value: "", + Effect: "NoSchedule", + } + + err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint) + Expect(err).NotTo(HaveOccurred()) + } + ginkgoNodes, err := strconv.Atoi(e2eConfig.GetVariable("CONFORMANCE_NODES")) Expect(err).NotTo(HaveOccurred()) runtime = b.Time("conformance suite", func() { - kubetest.Run(context.Background(), + err := kubetest.Run(context.Background(), kubetest.RunInput{ - ClusterProxy: workloadProxy, - NumberOfNodes: int(workerMachineCount), - ConfigFilePath: kubetestConfigFilePath, - ConformanceImage: e2eConfig.GetVariable("CONFORMANCE_IMAGE"), - GinkgoNodes: ginkgoNodes, + ClusterProxy: workloadProxy, + NumberOfNodes: int(workerMachineCount), + ConfigFilePath: kubetestConfigFilePath, + KubeTestRepoListPath: kubetestRepoListPath, + ConformanceImage: e2eConfig.GetVariable("CONFORMANCE_IMAGE"), + GinkgoNodes: ginkgoNodes, }, ) + Expect(err).NotTo(HaveOccurred()) }) b.RecordValue("conformance suite run time", runtime.Seconds()) }, 1) @@ -144,3 +177,7 @@ var _ = Describe("Conformance Tests", func() { }) }) + +func isWindows(kubetestConfigFilePath string) bool { + return strings.Contains(kubetestConfigFilePath, "windows") +} diff --git a/test/e2e/data/kubetest/repo-list.yaml b/test/e2e/data/kubetest/repo-list.yaml new file mode 100644 index 00000000000..8f84224db48 --- /dev/null +++ b/test/e2e/data/kubetest/repo-list.yaml @@ -0,0 +1,3 @@ +gcAuthenticatedRegistry: e2eprivate +gcEtcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images +privateRegistry: e2eteam \ No newline at end of file diff --git a/test/e2e/data/kubetest/upstream-windows.yaml b/test/e2e/data/kubetest/upstream-windows.yaml new file mode 100644 index 00000000000..3ffda2ca23d --- /dev/null +++ b/test/e2e/data/kubetest/upstream-windows.yaml @@ -0,0 +1,9 @@ +ginkgo.focus: \[Conformance\]|\[NodeConformance\]|\[sig-windows\]|\[sig-apps\].CronJob|\[sig-api-machinery\].ResourceQuota|\[sig-scheduling\].SchedulerPreemption +ginkgo.skip: \[LinuxOnly\]|\[Serial\]|Guestbook.application.should.create.and.stop.a.working.application|device.plugin.for.Windows|Container.Lifecycle.Hook.when.create.a.pod.with.lifecycle.hook.should.execute(.*)http.hook.properly +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 0 +ginkgo.trace: true +ginkgo.v: true +node-os-distro: windows diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 3776ff7dae9..8faec5e8a5c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -92,6 +92,9 @@ var ( // kubetestConfigFilePath is the path to the kubetest configuration file kubetestConfigFilePath string + // kubetestRepoListPath + kubetestRepoListPath string + // useCIArtifacts specifies whether or not to use the latest build from the main branch of the Kubernetes repository useCIArtifacts bool @@ -242,7 +245,7 @@ func init() { flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file") - + flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-file", "", "path to the kubetest repo-list file") } func TestE2E(t *testing.T) { diff --git a/test/e2e/kubernetes/node/node.go b/test/e2e/kubernetes/node/node.go index 1da6ad639ec..757e6d376f7 100644 --- a/test/e2e/kubernetes/node/node.go +++ b/test/e2e/kubernetes/node/node.go @@ -20,10 +20,15 @@ package node import ( "context" + "encoding/json" "fmt" "strings" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes" "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/windows" ) @@ -54,3 +59,82 @@ func GetWindowsVersion(ctx context.Context, clientset *kubernetes.Clientset) (wi return windows.LTSC2019, nil } } + +func TaintNode(clientset *kubernetes.Clientset, options v1.ListOptions, taint *corev1.Taint) error { + result, err := clientset.CoreV1().Nodes().List(context.Background(), options) + if err != nil { + return err + } + + if len(result.Items) == 0 { + return fmt.Errorf("No Nodes found.") + } + + for _, n := range result.Items { + newNode, needsUpdate := addOrUpdateTaint(&n, taint) + if !needsUpdate { + continue + } + + err = PatchNodeTaints(clientset, newNode.Name, &n, newNode) + if err != nil { + return err + } + } + + return nil +} + +// https://github.com/kubernetes/kubernetes/blob/v1.21.1/staging/src/k8s.io/cloud-provider/node/helpers/taints.go#L91 +func PatchNodeTaints(clientset *kubernetes.Clientset, nodeName string, oldNode *corev1.Node, newNode *corev1.Node) error { + oldData, err := json.Marshal(oldNode) + if err != nil { + return fmt.Errorf("failed to marshal old node %#v for node %q: %v", oldNode, nodeName, err) + } + + newTaints := newNode.Spec.Taints + newNodeClone := oldNode.DeepCopy() + newNodeClone.Spec.Taints = newTaints + newData, err := json.Marshal(newNodeClone) + if err != nil { + return fmt.Errorf("failed to marshal new node %#v for node %q: %v", newNodeClone, nodeName, err) + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, corev1.Node{}) + if err != nil { + return fmt.Errorf("failed to create patch for node %q: %v", nodeName, err) + } + + _, err = clientset.CoreV1().Nodes().Patch(context.Background(), nodeName, types.StrategicMergePatchType, patchBytes, v1.PatchOptions{}) + return err +} + +// From https://github.com/kubernetes/kubernetes/blob/v1.21.1/staging/src/k8s.io/cloud-provider/node/helpers/taints.go#L116 +// addOrUpdateTaint tries to add a taint to annotations list. Returns a new copy of updated Node and true if something was updated +// false otherwise. +func addOrUpdateTaint(node *corev1.Node, taint *corev1.Taint) (*corev1.Node, bool) { + newNode := node.DeepCopy() + nodeTaints := newNode.Spec.Taints + + var newTaints []corev1.Taint + updated := false + for i := range nodeTaints { + if taint.MatchTaint(&nodeTaints[i]) { + if equality.Semantic.DeepEqual(*taint, nodeTaints[i]) { + return newNode, false + } + newTaints = append(newTaints, *taint) + updated = true + continue + } + + newTaints = append(newTaints, nodeTaints[i]) + } + + if !updated { + newTaints = append(newTaints, *taint) + } + + newNode.Spec.Taints = newTaints + return newNode, true +}