Merge pull request #39 from benmoss/testing

Add script for deploying/testing a cluster on GCE

kubeadm/hack/checkin_account.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import httplib
+import urllib
+import os
+
+BOSKOS_HOST=os.environ.get("BOSKOS_HOST", "boskos")
+BOSKOS_RESOURCE_NAME=os.environ['BOSKOS_RESOURCE_NAME']
+
+USER = "kubeadm-windows"
+
+if __name__ == "__main__":
+    conn = httplib.HTTPConnection(BOSKOS_HOST)
+    conn.request("POST", "/release?%s" % urllib.urlencode({
+        'name': BOSKOS_RESOURCE_NAME,
+        'dest': 'dirty',
+        'owner': USER,
+    }))
+
+    resp = conn.getresponse()
+    if resp.status != 200:
+        sys.exit("Got invalid response %d" % resp.status)
+    conn.close()

kubeadm/hack/checkout_account.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Checks out a gcp account from E2E"""
+
+import urllib
+import httplib
+import os
+import sys
+import json
+
+BOSKOS_HOST=os.environ.get("BOSKOS_HOST", "boskos")
+
+RESOURCE_TYPE = "gce-project"
+USER = "kubeadm-windows"
+
+def post_request(host, input_state = "clean"):
+    conn = httplib.HTTPConnection(host)
+    conn.request("POST", "/acquire?%s" % urllib.urlencode({
+        'type': RESOURCE_TYPE,
+        'owner': USER,
+        'state': input_state,
+        'dest': 'busy',
+    })
+
+    )
+    resp = conn.getresponse()
+    status = resp.status
+    reason = resp.reason
+    body = resp.read()
+    conn.close()
+    return status, reason, body
+
+if __name__ == "__main__":
+    status, reason, result = post_request(BOSKOS_HOST)
+    #  we're working around an issue with the data in boskos.
+    #  We'll remove the code that tries both free and clean once all the data is good.
+    #  Afterwards we should just check for free
+    if status == 404:
+        status, reason, result = post_request(BOSKOS_HOST, "free")
+
+    if status != 200:
+        sys.exit("Got invalid response %d: %s" % (status, reason))
+
+    body = json.loads(result)
+    print 'export BOSKOS_RESOURCE_NAME="%s";' % body['name']
+    print 'export GCP_PROJECT="%s";' % body['name']

kubeadm/hack/ci-e2e.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+################################################################################
+# usage: e2e.sh
+#  This program runs the e2e tests.
+################################################################################
+
+set -o nounset
+set -o pipefail
+
+BOSKOS_HOST=${BOSKOS_HOST:-"boskos.test-pods.svc.cluster.local."}
+ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+
+# our exit handler (trap)
+cleanup() {
+  # stop boskos heartbeat
+  [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+}
+
+trap cleanup EXIT
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}" || exit 1
+
+echo "using boskos host to checkout project: ${BOSKOS_HOST}"
+
+# If BOSKOS_HOST is set then acquire an GCP account from Boskos.
+# Check out the account from Boskos and store the produced environment
+# variables in a temporary file.
+account_env_var_file="$(mktemp)"
+python hack/checkout_account.py 1>"${account_env_var_file}"
+checkout_account_status="${?}"
+
+# If the checkout process was a success then load the account's
+# environment variables into this process.
+# shellcheck disable=SC1090
+[ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+
+# Always remove the account environment variable file. It contains
+# sensitive information.
+rm -f "${account_env_var_file}"
+
+if [ ! "${checkout_account_status}" = "0" ]; then
+  echo "error getting account from boskos" 1>&2
+  exit "${checkout_account_status}"
+fi
+
+# run the heart beat process to tell boskos that we are still
+# using the checked out account periodically
+ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+mkdir -p "$ARTIFACTS/logs/"
+python -u hack/heartbeat_account.py >> "$ARTIFACTS/logs/boskos.log" 2>&1 &
+HEART_BEAT_PID=$(echo $!)
+
+hack/e2e-cluster-gcp.sh $*
+test_status="${?}"
+
+# If Boskos is being used then release the GCP project back to Boskos.
+[ -z "${BOSKOS_HOST:-}" ] || hack/checkin_account.py >> $ARTIFACTS/logs/boskos.log 2>&1
+
+exit "${test_status}"

kubeadm/hack/e2e-cluster-gcp.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODE_COUNT=${NODE_COUNT:-2}
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+CLUSTER_NAME=${CLUSTER_NAME:-"test-$(date +%s)"}
+ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+scratchDir=$(mktemp -d -t $CLUSTER_NAME)
+export KUBECONFIG="$scratchDir/kubeconfig.yml"
+
+export ARTIFACTS
+mkdir -p "${ARTIFACTS}/logs"
+
+# disable gcloud prompts
+export CLOUDSDK_CORE_DISABLE_PROMPTS=1
+
+cleanup() {
+	gcloud compute firewall-rules list --filter network="$CLUSTER_NAME" --format "value(selfLink.basename())" | \
+		xargs gcloud compute firewall-rules delete
+	gcloud compute instances list --filter labels.cluster="$CLUSTER_NAME" --format "value(selfLink.basename())" | \
+		xargs gcloud compute instances delete
+	gcloud compute networks delete "$CLUSTER_NAME"
+	rm -rf $scratchDir
+}
+SKIP_CLEANUP=${SKIP_CLEANUP:-""}
+if [[ -z "${SKIP_CLEANUP}" ]]; then
+	trap cleanup EXIT
+fi
+
+gcloud compute networks create "$CLUSTER_NAME"
+gcloud compute firewall-rules create "$CLUSTER_NAME-default-internal" --network "$CLUSTER_NAME" --allow tcp,udp,icmp --source-ranges "10.0.0.0/8"
+gcloud compute firewall-rules create "$CLUSTER_NAME-default-external" --network "$CLUSTER_NAME" --allow "tcp:22,tcp:3389,tcp:6443" --source-ranges "0.0.0.0/0"
+
+ctrlPlaneNodeName="$CLUSTER_NAME-control-plane"
+controlPlane=$(gcloud compute instances create "$ctrlPlaneNodeName" \
+	--metadata-from-file startup-script="$REPO_ROOT/kubeadm/hack/startup/controlplane.sh" \
+	--image-family ubuntu-1804-lts \
+	--image-project ubuntu-os-cloud \
+	--machine-type n1-standard-2 \
+	--network "$CLUSTER_NAME" \
+	--tags kube-control-plane \
+	--labels=cluster="$CLUSTER_NAME" \
+	--format "json(networkInterfaces[0].networkIP, networkInterfaces[0].accessConfigs[0].natIP)")
+
+# wait for kubeadm init
+SECONDS=0
+while true; do
+	set +o errexit
+	timeout 30 gcloud compute ssh "root@${ctrlPlaneNodeName}" --command "KUBECONFIG=/etc/kubernetes/admin.conf kubectl version"
+	if [ $? -eq 0 ]; then
+		gcloud compute scp "root@${ctrlPlaneNodeName}:/etc/kubernetes/admin.conf" "$KUBECONFIG"
+		break
+	else
+		if [ $SECONDS -ge 600 ]; then
+			echo "Failed waiting for control plane to become ready"
+			exit 1
+		fi
+		sleep 10
+	fi
+done
+set -o errexit
+
+privateIP=$(echo $controlPlane | jq -r '.[] | .networkInterfaces[0].networkIP')
+externalIP=$(echo $controlPlane | jq -r '.[] | .networkInterfaces[0].accessConfigs[0].natIP')
+sed -i "s/${privateIP}/${externalIP}/g" $KUBECONFIG
+
+k8sVersion=$(kubectl version -ojson | jq -r .serverVersion.gitVersion)
+sed "s/VERSION/${k8sVersion}/" "$REPO_ROOT/kubeadm/hack/startup/windows.ps1" > $scratchDir/windows.ps1
+joinCmd="$(gcloud compute ssh root@${ctrlPlaneNodeName} --command "kubeadm token create --print-join-command") --ignore-preflight-errors=IsPrivilegedUser"
+
+for i in $(seq 1 $NODE_COUNT); do
+	gcloud compute instances create "$CLUSTER_NAME-windows-node-$i" \
+		--metadata-from-file windows-startup-script-ps1="$scratchDir/windows.ps1" \
+		--metadata join_cmd="$joinCmd" \
+		--image-project windows-cloud \
+		--network "$CLUSTER_NAME" \
+		--image-family windows-2019-core-for-containers \
+		--labels=cluster="$CLUSTER_NAME" \
+		--machine-type n1-standard-4
+done
+
+# build kubectl and e2e.test
+pushd $(go env GOPATH)/src/k8s.io/kubernetes
+git checkout "$k8sVersion"
+make all WHAT="test/e2e/e2e.test cmd/kubectl"
+export PATH="$PWD/_output/bin:$PATH"
+
+# install flannel overlay & kube-proxy
+curl -sL https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml | \
+	sed 's/"Type": "vxlan"/"Type": "vxlan", "VNI": 4096, "Port": 4789/' | kubectl apply -f-
+
+curl -sL https://github.com/kubernetes-sigs/sig-windows-tools/releases/latest/download/kube-proxy.yml | \
+	sed "s/VERSION/${k8sVersion}/g" | kubectl apply -f -
+
+kubectl apply -f https://github.com/kubernetes-sigs/sig-windows-tools/releases/latest/download/flannel-overlay.yml
+
+# wait for all the nodes to appear
+SECONDS=0
+while true; do
+	nodeCount=$(kubectl get nodes -l kubernetes.io/os=windows --no-headers | wc -l)
+	if [ $nodeCount -eq $NODE_COUNT ]; then
+		break
+	else
+		if [ $SECONDS -ge 600 ]; then
+			echo "Failed waiting for Windows nodes to join the cluster"
+			exit 1
+		fi
+		sleep 10
+	fi
+done
+
+# wait for all kube-system pods to become ready
+kubectl -n kube-system wait --for=condition=Ready pods --all --timeout=15m
+
+# setting this env prevents ginkgo e2e from trying to run provider setup
+export KUBERNETES_CONFORMANCE_TEST="y"
+
+SKIP="${SKIP:-}"
+FOCUS="${FOCUS:-"\\[Conformance\\]"}"
+# if we set PARALLEL=true, skip serial tests set --ginkgo-parallel
+if [ "${PARALLEL:-false}" = "true" ]; then
+	export GINKGO_PARALLEL=y
+	if [ -z "${SKIP}" ]; then
+		SKIP="\\[Serial\\]"
+	else
+		SKIP="\\[Serial\\]|${SKIP}"
+	fi
+fi
+$(go env GOPATH)/src/sigs.k8s.io/windows-testing/gce/run-e2e.sh \
+	--provider=skeleton --num-nodes="$NODE_COUNT" \
+	--ginkgo.focus="$FOCUS" --ginkgo.skip="$SKIP" \
+	--node-os-distro=windows --report-dir="$ARTIFACTS" \
+	--disable-log-dump=true

kubeadm/hack/startup/controlplane.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Install Docker CE
+## Set up the repository:
+### Install packages to allow apt to use a repository over HTTPS
+apt-get update && apt-get install -y \
+  apt-transport-https ca-certificates curl software-properties-common gnupg2
+
+### Add Docker’s official GPG key
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+### Add Docker apt repository.
+add-apt-repository \
+  "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) \
+  stable"
+
+## Install Docker CE.
+apt-get update && apt-get install -y \
+  containerd.io=1.2.10-3 \
+  docker-ce=5:19.03.4~3-0~ubuntu-$(lsb_release -cs) \
+  docker-ce-cli=5:19.03.4~3-0~ubuntu-$(lsb_release -cs)
+
+# Setup daemon.
+cat > /etc/docker/daemon.json <<EOF
+{
+  "exec-opts": ["native.cgroupdriver=systemd"],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2"
+}
+EOF
+
+mkdir -p /etc/systemd/system/docker.service.d
+
+# Restart docker.
+systemctl daemon-reload
+systemctl restart docker
+
+apt-get update && apt-get install -y apt-transport-https curl
+curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+cat <<EOF | tee /etc/apt/sources.list.d/kubernetes.list
+deb https://apt.kubernetes.io/ kubernetes-xenial main
+EOF
+apt-get update
+apt-get install -y kubelet kubeadm kubectl
+apt-mark hold kubelet kubeadm kubectl
+
+externalIP=$(curl -H "Metadata-Flavor:Google" 169.254.169.254/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+kubeadm init --pod-network-cidr=10.244.0.0/16 --apiserver-cert-extra-sans "$externalIP"