From aa3d8b188b19086a23d6cce721dd9b9d203453a5 Mon Sep 17 00:00:00 2001 From: bzub Date: Sun, 27 Aug 2017 11:40:15 -0500 Subject: [PATCH 1/4] Support building images with a provided k8s version Other enhancements: - Pre-determine if sudo is needed for docker commands - Copy kubernetes/cluster into image to support more non-conformance e2e tests (tested with E2E_FOCUS="Feature:NetworkPolicy") So now make {container,push} KUBE_VERSION=1.X, etc will get the latest 1.X.Y version of k8s in that minor release range and tag images with "latest", "v1.X", and "v1.X.Y". --- .gitignore | 1 + Dockerfile | 2 ++ Makefile | 42 +++++++++++++++++++++++++++--------------- run_e2e.sh | 4 ++-- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index d653dd1..1b8cc95 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /_cache /e2e.test /kubectl +/cluster diff --git a/Dockerfile b/Dockerfile index 44d8f0e..e31db1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,8 @@ RUN apt-get update && apt-get -y --no-install-recommends install \ COPY e2e.test /usr/local/bin/ COPY kubectl /usr/local/bin/ COPY run_e2e.sh /run_e2e.sh +COPY cluster /kubernetes/cluster +WORKDIR /usr/local/bin ENV E2E_FOCUS="Conformance" # NOTE: kubectl tests are temporarily disabled due to the fact that they do not use the in-cluster diff --git a/Makefile b/Makefile index 4141eca..b2595ec 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,13 @@ TARGET = kube-conformance GOTARGET = github.com/heptio/$(TARGET) REGISTRY ?= gcr.io/heptio-images -KVER = v1.7.3 +KUBE_VERSION ?= 1.7 +kube_version = $(subst v,,$(KUBE_VERSION)) +kube_version_full = $(shell curl https://storage.googleapis.com/kubernetes-release/release/stable-$(kube_version).txt) IMAGE = $(REGISTRY)/$(BIN) -DOCKER ?= docker +in_docker_group=$(filter docker,$(shell groups)) +is_root=$(filter 0,$(shell id -u)) +DOCKER?=$(if $(or $(in_docker_group),$(is_root)),docker,sudo docker) DIR := ${CURDIR} .PHONY: all container getbins clean @@ -31,25 +35,33 @@ all: container e2e.test: getbins kubectl: getbins -getbins: | _cache/.getbins.$(KVER).timestamp +getbins: | _cache/.getbins.$(kube_version_full).timestamp -_cache/.getbins.$(KVER).timestamp: - mkdir -p _cache/$(KVER) - curl -L -o _cache/$(KVER)/kubernetes.tar.gz http://gcsweb.k8s.io/gcs/kubernetes-release/release/$(KVER)/kubernetes.tar.gz - tar -C _cache/$(KVER) -xzf _cache/$(KVER)/kubernetes.tar.gz - cd _cache/$(KVER) && KUBERNETES_DOWNLOAD_TESTS=true KUBERNETES_SKIP_CONFIRM=true ./kubernetes/cluster/get-kube-binaries.sh - mv _cache/$(KVER)/kubernetes/platforms/linux/amd64/e2e.test ./ - mv _cache/$(KVER)/kubernetes/platforms/linux/amd64/kubectl ./ - rm -rf _cache/$(KVER) +_cache/.getbins.$(kube_version_full).timestamp: clean + mkdir -p _cache/$(kube_version_full) + curl -L -o _cache/$(kube_version_full)/kubernetes.tar.gz http://gcsweb.k8s.io/gcs/kubernetes-release/release/$(kube_version_full)/kubernetes.tar.gz + tar -C _cache/$(kube_version_full) -xzf _cache/$(kube_version_full)/kubernetes.tar.gz + cd _cache/$(kube_version_full) && KUBE_VERSION="${kube_version_full}" \ + KUBERNETES_DOWNLOAD_TESTS=true \ + KUBERNETES_SKIP_CONFIRM=true ./kubernetes/cluster/get-kube-binaries.sh + mv _cache/$(kube_version_full)/kubernetes/cluster ./ + mv _cache/$(kube_version_full)/kubernetes/platforms/linux/amd64/e2e.test ./ + mv _cache/$(kube_version_full)/kubernetes/platforms/linux/amd64/kubectl ./ + rm -rf _cache/$(kube_version_full) touch $@ container: e2e.test kubectl - $(DOCKER) build -t $(REGISTRY)/$(TARGET):latest -t $(REGISTRY)/$(TARGET):$(KVER) . + $(DOCKER) build -t $(REGISTRY)/$(TARGET):latest \ + -t $(REGISTRY)/$(TARGET):v$(kube_version) \ + -t $(REGISTRY)/$(TARGET):v$(kube_version_full) . push: $(DOCKER) push $(REGISTRY)/$(TARGET):latest - $(DOCKER) push $(REGISTRY)/$(TARGET):$(KVER) + $(DOCKER) push $(REGISTRY)/$(TARGET):v$(kube_version) + $(DOCKER) push $(REGISTRY)/$(TARGET):v$(kube_version_full) clean: - rm -rf _cache e2e.test kubectl - $(DOCKER) rmi $(REGISTRY)/$(TARGET):latest $(REGISTRY)/$(TARGET):$(KVER) || true + rm -rf _cache e2e.test kubectl cluster + $(DOCKER) rmi $(REGISTRY)/$(TARGET):latest \ + $(REGISTRY)/$(TARGET):v$(kube_version) \ + $(REGISTRY)/$(TARGET):v$(kube_version_full) || true diff --git a/run_e2e.sh b/run_e2e.sh index 32a45fe..4b059ff 100755 --- a/run_e2e.sh +++ b/run_e2e.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo "/usr/local/bin/e2e.test --ginkgo.skip=\"${E2E_SKIP}\" --ginkgo.focus=\"${E2E_FOCUS}\" --provider=\"${E2E_PROVIDER}\" --report-dir=\"${RESULTS_DIR}\" --ginkgo.noColor=true" -/usr/local/bin/e2e.test --ginkgo.skip="${E2E_SKIP}" --ginkgo.focus="${E2E_FOCUS}" --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --ginkgo.noColor=true | tee ${RESULTS_DIR}/e2e.log +echo "/usr/local/bin/e2e.test --repo-root=/kubernetes --ginkgo.skip=\"${E2E_SKIP}\" --ginkgo.focus=\"${E2E_FOCUS}\" --provider=\"${E2E_PROVIDER}\" --report-dir=\"${RESULTS_DIR}\" --ginkgo.noColor=true" +/usr/local/bin/e2e.test --repo-root=/kubernetes --ginkgo.skip="${E2E_SKIP}" --ginkgo.focus="${E2E_FOCUS}" --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --ginkgo.noColor=true | tee ${RESULTS_DIR}/e2e.log # tar up the results for transmission back cd ${RESULTS_DIR} tar -czf e2e.tar.gz * From 34a0bb124706c14742975e54bf53d05acdbeb334 Mon Sep 17 00:00:00 2001 From: bzub Date: Sun, 27 Aug 2017 14:10:10 -0500 Subject: [PATCH 2/4] Fix extra "v" in image tag k8s version --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b2595ec..4a15b63 100644 --- a/Makefile +++ b/Makefile @@ -53,15 +53,15 @@ _cache/.getbins.$(kube_version_full).timestamp: clean container: e2e.test kubectl $(DOCKER) build -t $(REGISTRY)/$(TARGET):latest \ -t $(REGISTRY)/$(TARGET):v$(kube_version) \ - -t $(REGISTRY)/$(TARGET):v$(kube_version_full) . + -t $(REGISTRY)/$(TARGET):$(kube_version_full) . push: $(DOCKER) push $(REGISTRY)/$(TARGET):latest $(DOCKER) push $(REGISTRY)/$(TARGET):v$(kube_version) - $(DOCKER) push $(REGISTRY)/$(TARGET):v$(kube_version_full) + $(DOCKER) push $(REGISTRY)/$(TARGET):$(kube_version_full) clean: rm -rf _cache e2e.test kubectl cluster $(DOCKER) rmi $(REGISTRY)/$(TARGET):latest \ $(REGISTRY)/$(TARGET):v$(kube_version) \ - $(REGISTRY)/$(TARGET):v$(kube_version_full) || true + $(REGISTRY)/$(TARGET):$(kube_version_full) || true From fd0a44ba92bcb90b0a7c23ee3121699cae2d3380 Mon Sep 17 00:00:00 2001 From: bzub Date: Sun, 27 Aug 2017 14:21:58 -0500 Subject: [PATCH 3/4] Silence curl except for errors --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4a15b63..2b98a37 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ GOTARGET = github.com/heptio/$(TARGET) REGISTRY ?= gcr.io/heptio-images KUBE_VERSION ?= 1.7 kube_version = $(subst v,,$(KUBE_VERSION)) -kube_version_full = $(shell curl https://storage.googleapis.com/kubernetes-release/release/stable-$(kube_version).txt) +kube_version_full = $(shell curl -Ss https://storage.googleapis.com/kubernetes-release/release/stable-$(kube_version).txt) IMAGE = $(REGISTRY)/$(BIN) in_docker_group=$(filter docker,$(shell groups)) is_root=$(filter 0,$(shell id -u)) @@ -39,7 +39,7 @@ getbins: | _cache/.getbins.$(kube_version_full).timestamp _cache/.getbins.$(kube_version_full).timestamp: clean mkdir -p _cache/$(kube_version_full) - curl -L -o _cache/$(kube_version_full)/kubernetes.tar.gz http://gcsweb.k8s.io/gcs/kubernetes-release/release/$(kube_version_full)/kubernetes.tar.gz + curl -SsL -o _cache/$(kube_version_full)/kubernetes.tar.gz http://gcsweb.k8s.io/gcs/kubernetes-release/release/$(kube_version_full)/kubernetes.tar.gz tar -C _cache/$(kube_version_full) -xzf _cache/$(kube_version_full)/kubernetes.tar.gz cd _cache/$(kube_version_full) && KUBE_VERSION="${kube_version_full}" \ KUBERNETES_DOWNLOAD_TESTS=true \ From e133ff310adb06f4b445ea48f8bc7f88828e314c Mon Sep 17 00:00:00 2001 From: bzub Date: Sun, 27 Aug 2017 14:24:22 -0500 Subject: [PATCH 4/4] Only give "latest" image tag to the latest k8s release For example, right now v1.7 minor version. When v1.8 is considered stable, change "latest_stable = 1.8" --- Makefile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2b98a37..6fa69a9 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ TARGET = kube-conformance GOTARGET = github.com/heptio/$(TARGET) REGISTRY ?= gcr.io/heptio-images -KUBE_VERSION ?= 1.7 +latest_stable = 1.7 +KUBE_VERSION ?= $(latest_stable) kube_version = $(subst v,,$(KUBE_VERSION)) kube_version_full = $(shell curl -Ss https://storage.googleapis.com/kubernetes-release/release/stable-$(kube_version).txt) IMAGE = $(REGISTRY)/$(BIN) @@ -51,14 +52,18 @@ _cache/.getbins.$(kube_version_full).timestamp: clean touch $@ container: e2e.test kubectl - $(DOCKER) build -t $(REGISTRY)/$(TARGET):latest \ - -t $(REGISTRY)/$(TARGET):v$(kube_version) \ + $(DOCKER) build -t $(REGISTRY)/$(TARGET):v$(kube_version) \ -t $(REGISTRY)/$(TARGET):$(kube_version_full) . + if [ "$(kube_version)" = "$(latest_stable)" ]; then \ + $(DOCKER) tag $(REGISTRY)/$(TARGET):v$(kube_version) $(REGISTRY)/$(TARGET):latest; \ + fi push: - $(DOCKER) push $(REGISTRY)/$(TARGET):latest $(DOCKER) push $(REGISTRY)/$(TARGET):v$(kube_version) $(DOCKER) push $(REGISTRY)/$(TARGET):$(kube_version_full) + if [ "$(kube_version)" = "$(latest_stable)" ]; then \ + $(DOCKER) push $(REGISTRY)/$(TARGET):latest; \ + fi clean: rm -rf _cache e2e.test kubectl cluster