From e949f7812e9599a822d1344d396036c77487b731 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 16 May 2019 19:38:02 +0200 Subject: [PATCH 01/26] Refactor e2e test * Run sequentially after build and tests in CircleCI to reuse artifacts * Support running it locally (both in Linux and Darwin systems) through `make e2e`. An existing Kubernetes cluster will be used if available, otherwise a cluster will be created with Kind. --- .circleci/config.yml | 18 +-- .gitignore | 1 + Makefile | 7 +- test/e2e/e2e-flux-build.sh | 13 -- test/e2e/e2e-flux-chart.sh | 113 -------------- test/e2e/e2e-git.sh | 17 --- test/e2e/e2e-golang.sh | 19 --- test/e2e/e2e-helm.sh | 15 -- test/e2e/e2e-kind.sh | 26 ---- test/e2e/{git-dep.yaml => gitsrv.yaml} | 2 - test/e2e/run.sh | 199 +++++++++++++++++++++++++ 11 files changed, 207 insertions(+), 223 deletions(-) delete mode 100755 test/e2e/e2e-flux-build.sh delete mode 100755 test/e2e/e2e-flux-chart.sh delete mode 100755 test/e2e/e2e-git.sh delete mode 100755 test/e2e/e2e-golang.sh delete mode 100755 test/e2e/e2e-helm.sh delete mode 100755 test/e2e/e2e-kind.sh rename test/e2e/{git-dep.yaml => gitsrv.yaml} (96%) create mode 100755 test/e2e/run.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d47399a96f..a59c97c5e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: - run: make check-generated - run: make test TEST_FLAGS="-race -tags integration -timeout 60s" - run: make all - + - run: make e2e - deploy: name: Maybe push prerelease images command: | @@ -34,7 +34,6 @@ jobs: docker tag "docker.io/weaveworks/helm-operator:$(docker/image-tag)" "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)" docker push "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)" fi - - deploy: name: Maybe push release image and upload binaries command: | @@ -50,17 +49,6 @@ jobs: RELEASE_TAG=$(echo "$CIRCLE_TAG" | cut -c 6-) docker push "docker.io/weaveworks/helm-operator:${RELEASE_TAG}" fi - e2e-testing: - machine: true - working_directory: ~/go/src/github.com/weaveworks/flux - steps: - - checkout - - run: test/e2e/e2e-golang.sh - - run: test/e2e/e2e-flux-build.sh - - run: test/e2e/e2e-kind.sh - - run: test/e2e/e2e-helm.sh - - run: test/e2e/e2e-git.sh - - run: test/e2e/e2e-flux-chart.sh workflows: version: 2 @@ -70,6 +58,4 @@ workflows: filters: tags: only: /(helm-)?[0-9]+(\.[0-9]+)*(-[a-z]+)?/ - - e2e-testing: - requires: - - build + diff --git a/.gitignore b/.gitignore index 7e72e279e3..f0927ceda6 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ docker/fluxy-dumbconf.priv test/profiles test/bin/kubectl test/bin/helm +test/bin/kind diff --git a/Makefile b/Makefile index 1df81e4bb6..9e16730860 100644 --- a/Makefile +++ b/Makefile @@ -47,14 +47,17 @@ release-bins: clean: go clean rm -rf ./build - rm -f test/bin/kubectl test/bin/helm + rm -f test/bin/kubectl test/bin/helm test/bin/kind realclean: clean rm -rf ./cache -test: test/bin/helm test/bin/kubectl +test: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/weaveworks/flux/vendor" | sort -u) +e2e: test/bin/helm test/bin/kubectl + PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh + build/.%.done: docker/Dockerfile.% mkdir -p ./build/docker/$* cp $^ ./build/docker/$*/ diff --git a/test/e2e/e2e-flux-build.sh b/test/e2e/e2e-flux-build.sh deleted file mode 100755 index 709500e438..0000000000 --- a/test/e2e/e2e-flux-build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - -echo ">>> Installing go dep" -curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh -dep ensure -vendor-only - -echo ">>> Building docker images" -make all diff --git a/test/e2e/e2e-flux-chart.sh b/test/e2e/e2e-flux-chart.sh deleted file mode 100755 index 42713e3403..0000000000 --- a/test/e2e/e2e-flux-chart.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -REPO_ROOT=$(git rev-parse --show-toplevel) -KNOWN_HOSTS=$(cat ${REPO_ROOT}/test/e2e/known_hosts) -GITCONFIG=$(cat ${REPO_ROOT}/test/e2e/gitconfig) - -echo ">>> Loading $(docker/image-tag) into the cluster" -kind load docker-image "docker.io/weaveworks/flux:$(docker/image-tag)" -kind load docker-image "docker.io/weaveworks/helm-operator:$(docker/image-tag)" - -echo ">>> Installing Flux with Helm" -helm install --name flux --wait \ ---namespace flux \ ---set image.tag=$(docker/image-tag) \ ---set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \ ---set git.secretName=ssh-git \ ---set git.pollInterval=30s \ ---set git.config.secretName=gitconfig \ ---set git.config.enabled=true \ ---set-string git.config.data="${GITCONFIG}" \ ---set helmOperator.tag=$(docker/image-tag) \ ---set helmOperator.create=true \ ---set helmOperator.createCRD=true \ ---set helmOperator.git.secretName=ssh-git \ ---set registry.excludeImage=* \ ---set-string ssh.known_hosts="${KNOWN_HOSTS}" \ -${REPO_ROOT}/chart/flux - -echo '>>> Waiting for gitconfig secret' -retries=12 -count=0 -ok=false -until ${ok}; do - actual=$(kubectl get secrets -n flux gitconfig -ojsonpath={..data.gitconfig} | base64 -d) - if [ "${actual}" == "${GITCONFIG}" ]; then - echo -e "Expected Git configuration deployed\n" - kubectl get secrets -n flux gitconfig && echo - ok=true - else - ok=false - sleep 10 - fi - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux get secrets - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for namespace demo' -retries=12 -count=1 -ok=false -until ${ok}; do - kubectl describe ns/demo && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for workload podinfo' -retries=12 -count=0 -ok=false -until ${ok}; do - kubectl -n demo describe deployment/podinfo && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for Helm release mongodb' -retries=12 -count=0 -ok=false -until ${ok}; do - kubectl -n demo describe deployment/mongodb && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - kubectl -n flux logs deployment/flux-helm-operator - echo "No more retries left" - exit 1 - fi -done - -echo ">>> Flux logs" -kubectl -n flux logs deployment/flux - -echo ">>> Helm Operator logs" -kubectl -n flux logs deployment/flux-helm-operator - -echo ">>> List pods" -kubectl -n demo get pods - -echo ">>> Check workload" -kubectl -n demo rollout status deployment/podinfo - -echo ">>> Check Helm release" -kubectl -n demo rollout status deployment/mongodb diff --git a/test/e2e/e2e-git.sh b/test/e2e/e2e-git.sh deleted file mode 100755 index 725c50742c..0000000000 --- a/test/e2e/e2e-git.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -SCRIPT_DIR="${REPO_ROOT}/test/e2e" -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" - -echo ">>> Installing git" -kubectl create namespace flux -ssh-keygen -t rsa -N "" -f "${SCRIPT_DIR}/id_rsa" -kubectl create secret generic ssh-git --namespace=flux --from-file="${SCRIPT_DIR}/known_hosts" --from-file="${SCRIPT_DIR}/id_rsa" --from-file=identity="${SCRIPT_DIR}/id_rsa" --from-file="${SCRIPT_DIR}/id_rsa.pub" -rm "${SCRIPT_DIR}/id_rsa" "${SCRIPT_DIR}/id_rsa.pub" -kubectl apply -f "${SCRIPT_DIR}/git-dep.yaml" - -kubectl -n flux rollout status deployment/gitsrv - diff --git a/test/e2e/e2e-golang.sh b/test/e2e/e2e-golang.sh deleted file mode 100755 index dad56278a0..0000000000 --- a/test/e2e/e2e-golang.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -GO_VERSION=1.12.5 - -echo ">>> Installing go ${GO_VERSION}" -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz -tar -xf go${GO_VERSION}.linux-amd64.tar.gz -sudo rm -rf /usr/local/go -sudo mv go /usr/local - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - -mkdir -p $HOME/go/bin -mkdir -p $HOME/go/src - -go version diff --git a/test/e2e/e2e-helm.sh b/test/e2e/e2e-helm.sh deleted file mode 100755 index 85f12a898d..0000000000 --- a/test/e2e/e2e-helm.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" - -echo ">>> Installing Helm" -curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - -echo '>>> Installing Tiller' -kubectl --namespace kube-system create sa tiller -kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller -helm init --service-account tiller --upgrade --wait - diff --git a/test/e2e/e2e-kind.sh b/test/e2e/e2e-kind.sh deleted file mode 100755 index 41ba82362e..0000000000 --- a/test/e2e/e2e-kind.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo ">>> Installing kubectl" -curl -sLO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ -chmod +x kubectl && \ -sudo mv kubectl /usr/local/bin/ - -echo ">>> Building sigs.k8s.io/kind" -# Hairy way to clone and build version 0.2.1 of Kind since it doesn't support Go Modules: -mkdir -p $GOPATH/src/sigs.k8s.io -git clone https://github.com/kubernetes-sigs/kind.git $GOPATH/src/sigs.k8s.io/kind -git -C $GOPATH/src/sigs.k8s.io/kind checkout tags/0.2.1 -go install sigs.k8s.io/kind - -echo ">>> Installing kind" -sudo cp $GOPATH/bin/kind /usr/local/bin/ -kind create cluster --wait 5m - -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -kubectl get pods --all-namespaces diff --git a/test/e2e/git-dep.yaml b/test/e2e/gitsrv.yaml similarity index 96% rename from test/e2e/git-dep.yaml rename to test/e2e/gitsrv.yaml index 0b0da2041f..812bd4721e 100644 --- a/test/e2e/git-dep.yaml +++ b/test/e2e/gitsrv.yaml @@ -5,7 +5,6 @@ metadata: labels: name: gitsrv name: gitsrv - namespace: flux spec: replicas: 1 selector: @@ -46,7 +45,6 @@ metadata: labels: name: gitsrv name: gitsrv - namespace: flux spec: ports: - name: ssh diff --git a/test/e2e/run.sh b/test/e2e/run.sh new file mode 100755 index 0000000000..33a11c2d7e --- /dev/null +++ b/test/e2e/run.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +set -o errexit + +declare -a on_exit_items + +function on_exit() { + if [ "${#on_exit_items[@]}" -gt 0 ]; then + echo -e '\nRunning deferred items, please do not interrupt until they are done:' + fi + for I in "${on_exit_items[@]}"; do + echo "deferred: ${I}" + eval "${I}" + done +} + +# Cleaning up only makes sense in a local environment +# it just wastes time in CircleCI +if [ "${CI}" != 'true' ]; then + trap on_exit EXIT +fi + +function defer() { + on_exit_items=("$*" "${on_exit_items[@]}") +} + +REPO_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_DIR="${REPO_ROOT}/test/e2e" +KIND_VERSION=0.2.1 +CACHE_DIR="${REPO_ROOT}/cache/$CURRENT_OS_ARCH" +KIND_CACHE_PATH="${CACHE_DIR}/kind-$KIND_VERSION" +KIND_CLUSTER=flux-e2e +USING_KIND=false +FLUX_NAMESPACE=flux-e2e +DEMO_NAMESPACE=demo + + +# Check if there is a kubernetes cluster running, otherwise use Kind +if ! kubectl version > /dev/null 2>&1 ; then + if [ ! -f "${KIND_CACHE_PATH}" ]; then + echo '>>> Downloading Kind' + mkdir -p "${CACHE_DIR}" + curl -sL "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${CURRENT_OS_ARCH}" -o "${KIND_CACHE_PATH}" + fi + echo '>>> Creating Kind Kubernetes cluster' + cp "${KIND_CACHE_PATH}" "${REPO_ROOT}/test/bin/kind" + chmod +x "${REPO_ROOT}/test/bin/kind" + kind create cluster --name "${KIND_CLUSTER}" --wait 5m + defer kind --name "${KIND_CLUSTER}" delete cluster + export KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)" + USING_KIND=true + kubectl get pods --all-namespaces +fi + + +if ! helm version > /dev/null 2>&1; then + echo '>>> Installing Tiller' + kubectl --namespace kube-system create sa tiller + defer kubectl --namespace kube-system delete sa tiller + kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + defer kubectl delete clusterrolebinding tiller-cluster-rule + helm init --service-account tiller --upgrade --wait + defer helm reset --force +fi + +kubectl create namespace "$FLUX_NAMESPACE" +defer kubectl delete namespace "$FLUX_NAMESPACE" + +echo '>>> Installing mock git server' +ssh-keygen -t rsa -N "" -f "${SCRIPT_DIR}/id_rsa" +defer rm -f "${SCRIPT_DIR}/id_rsa" "${SCRIPT_DIR}/id_rsa.pub" +kubectl create secret generic ssh-git --namespace="${FLUX_NAMESPACE}" --from-file="${SCRIPT_DIR}/known_hosts" --from-file="${SCRIPT_DIR}/id_rsa" --from-file=identity="${SCRIPT_DIR}/id_rsa" --from-file="${SCRIPT_DIR}/id_rsa.pub" +kubectl apply -n "${FLUX_NAMESPACE}" -f "${SCRIPT_DIR}/gitsrv.yaml" +kubectl -n "${FLUX_NAMESPACE}" rollout status deployment/gitsrv + + +if [ "${USING_KIND}" = 'true' ]; then + echo '>>> Loading images into the Kind cluster' + kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/weaveworks/flux:latest' + kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/weaveworks/helm-operator:latest' +fi + +echo '>>> Installing Flux with Helm' + +KNOWN_HOSTS=$(cat "${REPO_ROOT}/test/e2e/known_hosts") +GITCONFIG=$(cat "${REPO_ROOT}/test/e2e/gitconfig") + +helm install --name flux --wait \ +--namespace "${FLUX_NAMESPACE}" \ +--set image.tag=latest \ +--set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \ +--set git.secretName=ssh-git \ +--set git.pollInterval=30s \ +--set git.config.secretName=gitconfig \ +--set git.config.enabled=true \ +--set-string git.config.data="$GITCONFIG" \ +--set helmOperator.tag=latest \ +--set helmOperator.create=true \ +--set helmOperator.createCRD=true \ +--set helmOperator.git.secretName=ssh-git \ +--set registry.excludeImage=* \ +--set-string ssh.known_hosts="$KNOWN_HOSTS" \ +"${REPO_ROOT}/chart/flux" + +defer helm delete --purge flux +# These CRDs are have a keep policy and need to be deleted manually +defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works + +echo -n '>>> Waiting for gitconfig secret ' +retries=24 +count=0 +ok=false +until ${ok}; do + actual=$(kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig -ojsonpath={..data.gitconfig} | base64 --decode) + if [ "${actual}" = "${GITCONFIG}" ]; then + echo ' Expected Git configuration deployed' + kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig && echo + ok=true + else + echo -n '.' + ok=false + sleep 5 + fi + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo ' No more retries left' + kubectl -n "${FLUX_NAMESPACE}" get secrets + exit 1 + fi +done + +echo -n ">>> Waiting for namespace ${DEMO_NAMESPACE} " +retries=24 +count=1 +ok=false +until ${ok}; do + kubectl describe "ns/${DEMO_NAMESPACE}" && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo -n '>>> Waiting for workload podinfo ' +retries=24 +count=0 +ok=false +until ${ok}; do + kubectl -n "${DEMO_NAMESPACE}" describe deployment/podinfo && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo -n '>>> Waiting for Helm release mongodb ' +retries=24 +count=0 +ok=false +until ${ok}; do + kubectl -n $DEMO_NAMESPACE describe deployment/mongodb && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo '>>> Flux logs' +kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + +echo '>>> Helm Operator logs' +kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator + +echo '>>> List pods' +kubectl -n "${DEMO_NAMESPACE}" get pods + +echo '>>> Check workload' +kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/podinfo + +echo '>>> Check Helm release' +kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/mongodb + +echo -e '\nEnd to end test was successful!!\n' \ No newline at end of file From cbf901db76e7e6c2d571877cd33a56b85c41032e Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 17 May 2019 02:46:45 +0200 Subject: [PATCH 02/26] Use machine builder --- .circleci/config.yml | 34 ++++++++++++++++++++++++---------- gpg/gpgtest/gpg.go | 2 +- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a59c97c5e1..3b083ad986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,27 +1,41 @@ version: 2 jobs: build: - working_directory: /go/src/github.com/weaveworks/flux - docker: - - image: circleci/golang:1.10 - - image: memcached + working_directory: ~/go/src/github.com/weaveworks/flux + machine: true + environment: + GO_VERSION: 1.12.5 + GOPATH: /home/circleci/go + PATH: /bin:/usr/bin:/usr/local/go/bin:/home/circleci/go/bin steps: - checkout - - setup_remote_docker - - run: - # Ensure latest version of git + name: Install Golang + command: | + curl -OL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz + tar -xf go${GO_VERSION}.linux-amd64.tar.gz + sudo rm -rf /usr/local/go + sudo mv go /usr/local + mkdir -p "$HOME/go/bin" + go version + - run: + name: Update packages and Start Memcached command: | - echo "deb http://deb.debian.org/debian stretch-backports main" | sudo tee -a /etc/apt/sources.list.d/stretch-backports.list + # These repos fail and we don't need them: + sudo rm /etc/apt/sources.list.d/circleci_trusty.list /etc/apt/sources.list.d/google-chrome.list sudo apt-get update - sudo apt-get install -t stretch-backports -y --only-upgrade git + sudo apt-get install -y git rng-tools docker-ce memcached git version - - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + docker version + - run: + name: Install Dep + command: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - run: dep ensure -vendor-only - run: make check-generated - run: make test TEST_FLAGS="-race -tags integration -timeout 60s" - run: make all - run: make e2e + - deploy: name: Maybe push prerelease images command: | diff --git a/gpg/gpgtest/gpg.go b/gpg/gpgtest/gpg.go index aecbb2ac3f..d1b574ed35 100644 --- a/gpg/gpgtest/gpg.go +++ b/gpg/gpgtest/gpg.go @@ -39,7 +39,7 @@ func GPGKey(t *testing.T) (string, string, func()) { t.Fatal(err) } - gpgCmd := exec.Command("gpg", "--homedir", newDir, "--list-keys", "--with-colons") + gpgCmd := exec.Command("gpg", "--homedir", newDir, "--list-keys", "--with-colons", "--with-fingerprint") grepCmd := exec.Command("grep", "^fpr") cutCmd := exec.Command("cut", "-d:", "-f10") From dd18bda89b0c0f4e429f27cb665011c364a30c09 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 17 May 2019 19:17:26 +0200 Subject: [PATCH 03/26] Adjust cleanup scheduling and handle existing CRDs --- test/e2e/run.sh | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 33a11c2d7e..1d3feb629f 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -45,8 +45,8 @@ if ! kubectl version > /dev/null 2>&1 ; then echo '>>> Creating Kind Kubernetes cluster' cp "${KIND_CACHE_PATH}" "${REPO_ROOT}/test/bin/kind" chmod +x "${REPO_ROOT}/test/bin/kind" + defer kind --name "${KIND_CLUSTER}" delete cluster > /dev/null 2>&1 kind create cluster --name "${KIND_CLUSTER}" --wait 5m - defer kind --name "${KIND_CLUSTER}" delete cluster export KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)" USING_KIND=true kubectl get pods --all-namespaces @@ -82,9 +82,22 @@ fi echo '>>> Installing Flux with Helm' +CREATE_CRDS='true' +if kubectl get crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works > /dev/null 2>&1; then + # CRDs existed, don't try to create them + echo 'CRDs existed, setting helmOperator.createCRD=false' + CREATE_CRDS='false' +else + # Schedule CRD deletion before calling helm, since it may fail and create them anyways + defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works > /dev/null 2>&1 +fi + KNOWN_HOSTS=$(cat "${REPO_ROOT}/test/e2e/known_hosts") GITCONFIG=$(cat "${REPO_ROOT}/test/e2e/gitconfig") + +defer helm delete --purge flux > /dev/null 2>&1 + helm install --name flux --wait \ --namespace "${FLUX_NAMESPACE}" \ --set image.tag=latest \ @@ -93,18 +106,18 @@ helm install --name flux --wait \ --set git.pollInterval=30s \ --set git.config.secretName=gitconfig \ --set git.config.enabled=true \ ---set-string git.config.data="$GITCONFIG" \ +--set-string git.config.data="${GITCONFIG}" \ --set helmOperator.tag=latest \ --set helmOperator.create=true \ --set helmOperator.createCRD=true \ --set helmOperator.git.secretName=ssh-git \ --set registry.excludeImage=* \ ---set-string ssh.known_hosts="$KNOWN_HOSTS" \ +--set-string ssh.known_hosts="${KNOWN_HOSTS}" \ +--set helmOperator.createCRD="${CREATE_CRDS}" \ "${REPO_ROOT}/chart/flux" -defer helm delete --purge flux -# These CRDs are have a keep policy and need to be deleted manually -defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works + + echo -n '>>> Waiting for gitconfig secret ' retries=24 From d074318aa5db1fec44905796a824726202feabfc Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Mon, 20 May 2019 14:02:09 +0200 Subject: [PATCH 04/26] Add container dependencies where they belong --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9e16730860..8b64600f21 100644 --- a/Makefile +++ b/Makefile @@ -52,10 +52,10 @@ clean: realclean: clean rm -rf ./cache -test: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done +test: test/bin/helm test/bin/kubectl PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/weaveworks/flux/vendor" | sort -u) -e2e: test/bin/helm test/bin/kubectl +e2e: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh build/.%.done: docker/Dockerfile.% From 015bfd540bbe84c53df47643d406f42782549e77 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Mon, 20 May 2019 14:02:59 +0200 Subject: [PATCH 05/26] Add EOL --- test/e2e/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 1d3feb629f..b5f400a3bc 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -209,4 +209,4 @@ kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/podinfo echo '>>> Check Helm release' kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/mongodb -echo -e '\nEnd to end test was successful!!\n' \ No newline at end of file +echo -e '\nEnd to end test was successful!!\n' From 55a724a2b4f919b89aed08955d15a29e370f677b Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 9 May 2019 11:10:02 +0200 Subject: [PATCH 06/26] Build of sigs.k8s.io/kind now requires gomodules https://github.com/kubernetes-sigs/kind/issues/509 --- test/e2e/e2e-golang.sh | 4 ++-- test/e2e/e2e-kind.sh | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/e2e-golang.sh b/test/e2e/e2e-golang.sh index 3443d25cfd..dad56278a0 100755 --- a/test/e2e/e2e-golang.sh +++ b/test/e2e/e2e-golang.sh @@ -2,11 +2,11 @@ set -o errexit -GO_VERSION=1.11.4 +GO_VERSION=1.12.5 echo ">>> Installing go ${GO_VERSION}" curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz -tar -xf go1.11.4.linux-amd64.tar.gz +tar -xf go${GO_VERSION}.linux-amd64.tar.gz sudo rm -rf /usr/local/go sudo mv go /usr/local diff --git a/test/e2e/e2e-kind.sh b/test/e2e/e2e-kind.sh index 6f8f0183b6..99b845f84b 100755 --- a/test/e2e/e2e-kind.sh +++ b/test/e2e/e2e-kind.sh @@ -12,7 +12,9 @@ chmod +x kubectl && \ sudo mv kubectl /usr/local/bin/ echo ">>> Building sigs.k8s.io/kind" -go get -u sigs.k8s.io/kind +cd $HOME +GO111MODULE="on" go get -u sigs.k8s.io/kind@master +cd $REPO_ROOT echo ">>> Installing kind" sudo cp $GOPATH/bin/kind /usr/local/bin/ From 8b5e28ee2d8336208139740fd1e1f9354a7375e1 Mon Sep 17 00:00:00 2001 From: Rian Finnegan Date: Fri, 3 May 2019 12:52:06 +1000 Subject: [PATCH 07/26] Updated FAQ.md Adding an explicit warning to users that Flux ignores JSON files. --- site/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/faq.md b/site/faq.md index a19f26b67c..e7e8e83934 100644 --- a/site/faq.md +++ b/site/faq.md @@ -120,7 +120,7 @@ built). ### Is there any special directory layout I need in my git repo? Nope. Flux doesn't place any significance on the directory structure, -and will descend into subdirectories in search of YAMLs. It avoids +and will descend into subdirectories in search of YAMLs. Although [kubectl works with JSON files](https://kubernetes.io/docs/concepts/configuration/overview/#using-kubectl), Flux will ignore JSON. It avoids directories that look like Helm charts. If you have YAML files in the repo that _aren't_ for applying to From 7020a84a40d286dede7a6eff04ddc3b1d0093334 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Wed, 8 May 2019 16:00:38 +0200 Subject: [PATCH 08/26] Show unknown tag for workload in list-images If we are unable to match the image of the workload to a tag from our cache, we still list the name of the tag and a question mark where we would normally list the created date, to indicate we have no record of it in our cache. --- cmd/fluxctl/list_images_cmd.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/fluxctl/list_images_cmd.go b/cmd/fluxctl/list_images_cmd.go index 416bab7595..3ac8f8d581 100644 --- a/cmd/fluxctl/list_images_cmd.go +++ b/cmd/fluxctl/list_images_cmd.go @@ -136,6 +136,14 @@ func (opts *imageListOpts) RunE(cmd *cobra.Command, args []string) error { fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, tag, createdAt) } } + if !foundRunning { + running := "'->" + if currentTag == "" { + currentTag = "(untagged)" + } + fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, currentTag, "?") + + } workloadName = "" } } From 3a598f76696a877df78a4c1f53cd39ea5600f74d Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 9 May 2019 18:54:13 +0200 Subject: [PATCH 09/26] Always list the status of a workload in fluxctl Before this change the `fluxctl list-workloads` command would only list the status of a workload if it had containers. But because of how we integrated HelmReleases into the Flux daemon, a HelmRelease only has 'containers' if we are able to detect images in the .spec.values of the resource. After this change it will always list the status of the workload (and any policies), regardless of the amount of containers it (does not) have. --- cmd/fluxctl/list_workloads_cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/fluxctl/list_workloads_cmd.go b/cmd/fluxctl/list_workloads_cmd.go index 8e8af38ef4..715439531b 100644 --- a/cmd/fluxctl/list_workloads_cmd.go +++ b/cmd/fluxctl/list_workloads_cmd.go @@ -63,7 +63,7 @@ func (opts *workloadListOpts) RunE(cmd *cobra.Command, args []string) error { fmt.Fprintf(w, "\t%s\t%s\t\t\n", c.Name, c.Current.ID) } } else { - fmt.Fprintf(w, "%s\t\t\t\t\n", workload.ID) + fmt.Fprintf(w, "%s\t\t\t%s\t%s\n", workload.ID, workload.Status, policies(workload)) } } w.Flush() From eba139ac91a1aa9ca31ab652879d17fc0e6e7093 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Thu, 9 May 2019 17:35:19 +0100 Subject: [PATCH 10/26] Log warning re not applying resource by namespace We log when a resource in a manifest is dropped from sync because it's marked as ignored; would also help troubleshooting if we log when a resource is dropped because it's not in the allowed namespaces. --- cluster/kubernetes/sync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index e6f7f67a78..1bf5bf3ce4 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -64,10 +64,11 @@ func (c *Cluster) Sync(syncSet cluster.SyncSet) error { var errs cluster.SyncError for _, res := range syncSet.Resources { resID := res.ResourceID() + id := resID.String() if !c.IsAllowedResource(resID) { + logger.Log("warning", "not applying resource; excluded by namespace constraints", "resource", id, "source", res.Source()) continue } - id := resID.String() // make a record of the checksum, whether we stage it to // be applied or not, so that we don't delete it later. csum := sha1.Sum(res.Bytes()) From 49f8ae12ca879e6d8857cd5b83115ce3a21940f4 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 13 May 2019 10:26:22 +0100 Subject: [PATCH 11/26] Log once for excluded resources Logging all excluded resources may be quite noisy. Instead, collect the IDs of excluded resources and log once (or no times, if there aren't any). --- cluster/kubernetes/sync.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster/kubernetes/sync.go b/cluster/kubernetes/sync.go index 1bf5bf3ce4..364cd6feac 100644 --- a/cluster/kubernetes/sync.go +++ b/cluster/kubernetes/sync.go @@ -62,11 +62,12 @@ func (c *Cluster) Sync(syncSet cluster.SyncSet) error { cs := makeChangeSet() var errs cluster.SyncError + var excluded []string for _, res := range syncSet.Resources { resID := res.ResourceID() id := resID.String() if !c.IsAllowedResource(resID) { - logger.Log("warning", "not applying resource; excluded by namespace constraints", "resource", id, "source", res.Source()) + excluded = append(excluded, id) continue } // make a record of the checksum, whether we stage it to @@ -94,6 +95,10 @@ func (c *Cluster) Sync(syncSet cluster.SyncSet) error { } } + if len(excluded) > 0 { + logger.Log("warning", "not applying resources; excluded by namespace constraints", "resources", strings.Join(excluded, ",")) + } + c.mu.Lock() defer c.mu.Unlock() c.muSyncErrors.RLock() From 2c832467d1aeb6d4a8b65573a7fb6e2783b79a94 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 4 Apr 2019 15:06:09 +0200 Subject: [PATCH 12/26] Ensure Git >=2.12 in both images and CI --- .circleci/config.yml | 7 +++++++ docker/Dockerfile.flux | 2 +- docker/Dockerfile.helm-operator | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b675a0d8eb..d47399a96f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,13 @@ jobs: - checkout - setup_remote_docker + - run: + # Ensure latest version of git + command: | + echo "deb http://deb.debian.org/debian stretch-backports main" | sudo tee -a /etc/apt/sources.list.d/stretch-backports.list + sudo apt-get update + sudo apt-get install -t stretch-backports -y --only-upgrade git + git version - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - run: dep ensure -vendor-only - run: make check-generated diff --git a/docker/Dockerfile.flux b/docker/Dockerfile.flux index 6c4d5bb4eb..4bf186daa6 100644 --- a/docker/Dockerfile.flux +++ b/docker/Dockerfile.flux @@ -2,7 +2,7 @@ FROM alpine:3.9 WORKDIR /home/flux -RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' gnupg +RUN apk add --no-cache openssh ca-certificates tini 'git>=2.12.0' gnupg # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh diff --git a/docker/Dockerfile.helm-operator b/docker/Dockerfile.helm-operator index fca75fc0ac..acf6165a12 100644 --- a/docker/Dockerfile.helm-operator +++ b/docker/Dockerfile.helm-operator @@ -2,7 +2,7 @@ FROM alpine:3.9 WORKDIR /home/flux -RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' +RUN apk add --no-cache openssh ca-certificates tini 'git>=2.12.0' # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh From 6466e32fa3647d012d7b239f01b0d29116657001 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 13 May 2019 16:29:34 +0200 Subject: [PATCH 13/26] Ensure gnutls >=3.6.7 Should resolve the trigger of several CVE warnings (which did not pose a threat): CVE-2019-3829, CVE-2019-3836, CVE-2018-1000654 --- docker/Dockerfile.flux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.flux b/docker/Dockerfile.flux index 4bf186daa6..4ee0c8f24c 100644 --- a/docker/Dockerfile.flux +++ b/docker/Dockerfile.flux @@ -2,7 +2,7 @@ FROM alpine:3.9 WORKDIR /home/flux -RUN apk add --no-cache openssh ca-certificates tini 'git>=2.12.0' gnupg +RUN apk add --no-cache openssh ca-certificates tini 'git>=2.12.0' 'gnutls>=3.6.7' gnupg # Add git hosts to known hosts file so we can use # StrickHostKeyChecking with git+ssh From 1f8a99003608eb4819a41f561d71bd67ff7171d8 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Tue, 14 May 2019 12:00:09 +0200 Subject: [PATCH 14/26] Rename controller to workload in `fluxctl release` Leftover from #1777 --- cmd/fluxctl/release_cmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/fluxctl/release_cmd.go b/cmd/fluxctl/release_cmd.go index 05b565b595..4be5907470 100644 --- a/cmd/fluxctl/release_cmd.go +++ b/cmd/fluxctl/release_cmd.go @@ -42,7 +42,7 @@ func newWorkloadRelease(parent *rootOpts) *workloadReleaseOpts { func (opts *workloadReleaseOpts) Command() *cobra.Command { cmd := &cobra.Command{ Use: "release", - Short: "Release a new version of a controller.", + Short: "Release a new version of a workload.", Example: makeExample( "fluxctl release -n default --workload=deployment/foo --update-image=library/hello:v2", "fluxctl release --all --update-image=library/hello:v2", @@ -56,10 +56,10 @@ func (opts *workloadReleaseOpts) Command() *cobra.Command { cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "default", "Workload namespace") // Note: we cannot define a shorthand for --workload since it clashes with the shorthand of --watch cmd.Flags().StringSliceVarP(&opts.workloads, "workload", "", []string{}, "List of workloads to release :/") - cmd.Flags().BoolVar(&opts.allWorkloads, "all", false, "Release all controllers") + cmd.Flags().BoolVar(&opts.allWorkloads, "all", false, "Release all workloads") cmd.Flags().StringVarP(&opts.image, "update-image", "i", "", "Update a specific image") cmd.Flags().BoolVar(&opts.allImages, "update-all-images", false, "Update all images to latest versions") - cmd.Flags().StringSliceVar(&opts.exclude, "exclude", []string{}, "List of controllers to exclude") + cmd.Flags().StringSliceVar(&opts.exclude, "exclude", []string{}, "List of workloads to exclude") cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Do not release anything; just report back what would have been done") cmd.Flags().BoolVar(&opts.interactive, "interactive", false, "Select interactively which containers to update") cmd.Flags().BoolVarP(&opts.force, "force", "f", false, "Disregard locks and container image filters (has no effect when used with --all or --update-all-images)") From c06d2c3cfc86e40bf437ac27cf29570adb8e158d Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 16 May 2019 15:23:17 +0200 Subject: [PATCH 15/26] Give full output of git command on errors As a side effect, the tracing of commands also prints the full output (instead of separating stderr and stdout), which I think is more useful. --- git/operations.go | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/git/operations.go b/git/operations.go index 8f841377d8..dd4b91ac20 100644 --- a/git/operations.go +++ b/git/operations.go @@ -3,16 +3,14 @@ package git import ( "bufio" "bytes" + "context" "encoding/json" "fmt" "io" - "io/ioutil" "os" "os/exec" "strings" - "context" - "github.com/pkg/errors" ) @@ -290,7 +288,7 @@ func changed(ctx context.Context, workingDir, ref string, subPaths []string) ([] } // traceGitCommand returns a log line that can be useful when debugging and developing git activity -func traceGitCommand(args []string, config gitCmdConfig, stdout string, stderr string) string { +func traceGitCommand(args []string, config gitCmdConfig, stdOutAndStdErr string) string { for _, exemptedCommand := range exemptedTraceCommands { if exemptedCommand == args[0] { return "" @@ -305,14 +303,12 @@ func traceGitCommand(args []string, config gitCmdConfig, stdout string, stderr s } command := `git ` + strings.Join(args, " ") - out := prepare(stdout) - err := prepare(stderr) + out := prepare(stdOutAndStdErr) return fmt.Sprintf( - "TRACE: command=%q out=%q err=%q dir=%q env=%q", + "TRACE: command=%q out=%q dir=%q env=%q", command, out, - err, config.dir, strings.Join(config.env, ","), ) @@ -326,30 +322,27 @@ func execGitCmd(ctx context.Context, args []string, config gitCmdConfig) error { c.Dir = config.dir } c.Env = append(env(), config.env...) - c.Stdout = ioutil.Discard - if config.out != nil { - c.Stdout = config.out - } - errOut := &bytes.Buffer{} - c.Stderr = errOut - traceStdout := &bytes.Buffer{} - traceStderr := &bytes.Buffer{} - if trace { - c.Stdout = io.MultiWriter(c.Stdout, traceStdout) - c.Stderr = io.MultiWriter(c.Stderr, traceStderr) + stdOutAndStdErr := &bytes.Buffer{} + c.Stdout = stdOutAndStdErr + c.Stderr = stdOutAndStdErr + if config.out != nil { + c.Stdout = io.MultiWriter(c.Stdout, config.out) } err := c.Run() if err != nil { - msg := findErrorMessage(errOut) - if msg != "" { - err = errors.New(msg) + if len(stdOutAndStdErr.Bytes()) > 0 { + err = errors.New(stdOutAndStdErr.String()) + msg := findErrorMessage(stdOutAndStdErr) + if msg != "" { + err = fmt.Errorf("%s, full output:\n %s", msg, err.Error()) + } } } if trace { - if traceCommand := traceGitCommand(args, config, traceStdout.String(), traceStderr.String()); traceCommand != "" { + if traceCommand := traceGitCommand(args, config, stdOutAndStdErr.String()); traceCommand != "" { println(traceCommand) } } From 6d220fd8d840a5d525a26200ad451bf9f8ad3a13 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 16 May 2019 15:54:54 +0200 Subject: [PATCH 16/26] Fix race condition and tests --- git/operations.go | 21 +++++++++++++++++++-- git/operations_test.go | 9 ++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/git/operations.go b/git/operations.go index dd4b91ac20..43c5509248 100644 --- a/git/operations.go +++ b/git/operations.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "strings" + "sync" "github.com/pkg/errors" ) @@ -314,6 +315,23 @@ func traceGitCommand(args []string, config gitCmdConfig, stdOutAndStdErr string) ) } +type threadSafeBuffer struct { + bytes.Buffer + sync.Mutex +} + +func (b *threadSafeBuffer) Write(p []byte) (n int, err error) { + b.Lock() + defer b.Unlock() + return b.Buffer.Write(p) +} + +func (b *threadSafeBuffer) ReadFrom(r io.Reader) (n int64, err error) { + b.Lock() + defer b.Unlock() + return b.Buffer.ReadFrom(r) +} + // execGitCmd runs a `git` command with the supplied arguments. func execGitCmd(ctx context.Context, args []string, config gitCmdConfig) error { c := exec.CommandContext(ctx, "git", args...) @@ -322,8 +340,7 @@ func execGitCmd(ctx context.Context, args []string, config gitCmdConfig) error { c.Dir = config.dir } c.Env = append(env(), config.env...) - - stdOutAndStdErr := &bytes.Buffer{} + stdOutAndStdErr := &threadSafeBuffer{} c.Stdout = stdOutAndStdErr c.Stderr = stdOutAndStdErr if config.out != nil { diff --git a/git/operations_test.go b/git/operations_test.go index 74fcedae75..113af66f4d 100644 --- a/git/operations_test.go +++ b/git/operations_test.go @@ -316,7 +316,7 @@ func TestTraceGitCommand(t *testing.T) { dir: "/tmp/flux-working628880789", }, }, - expected: `TRACE: command="git clone --branch master /tmp/flux-gitclone239583443 /tmp/flux-working628880789" out="" err="" dir="/tmp/flux-working628880789" env=""`, + expected: `TRACE: command="git clone --branch master /tmp/flux-gitclone239583443 /tmp/flux-working628880789" out="" dir="/tmp/flux-working628880789" env=""`, }, { name: "git rev-list", @@ -333,7 +333,7 @@ func TestTraceGitCommand(t *testing.T) { dir: "/tmp/flux-gitclone239583443", }, }, - expected: `TRACE: command="git rev-list --max-count 1 flux-sync --" out="b9d6a543acf8085ff6bed23fac17f8dc71bfcb66" err="" dir="/tmp/flux-gitclone239583443" env=""`, + expected: `TRACE: command="git rev-list --max-count 1 flux-sync --" out="b9d6a543acf8085ff6bed23fac17f8dc71bfcb66" dir="/tmp/flux-gitclone239583443" env=""`, }, { name: "git config email", @@ -347,7 +347,7 @@ func TestTraceGitCommand(t *testing.T) { dir: "/tmp/flux-working056923691", }, }, - expected: `TRACE: command="git config user.email support@weave.works" out="" err="" dir="/tmp/flux-working056923691" env=""`, + expected: `TRACE: command="git config user.email support@weave.works" out="" dir="/tmp/flux-working056923691" env=""`, }, { name: "git notes", @@ -363,7 +363,7 @@ func TestTraceGitCommand(t *testing.T) { }, out: "refs/notes/flux", }, - expected: `TRACE: command="git notes --ref flux get-ref" out="refs/notes/flux" err="" dir="/tmp/flux-working647148942" env=""`, + expected: `TRACE: command="git notes --ref flux get-ref" out="refs/notes/flux" dir="/tmp/flux-working647148942" env=""`, }, } for _, example := range examples { @@ -371,7 +371,6 @@ func TestTraceGitCommand(t *testing.T) { example.input.args, example.input.config, example.input.out, - example.input.err, ) assert.Equal(t, example.expected, actual) } From 1586b296997b524847aa4abf4493136dd9e9a2ad Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 15 May 2019 20:43:12 +0200 Subject: [PATCH 17/26] Bump gitsrv's version In order for the initialization to retry if there is a netwroking glitch --- test/e2e/git-dep.yaml | 2 +- test/e2e/known_hosts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/e2e/git-dep.yaml b/test/e2e/git-dep.yaml index 0a263f68af..0b0da2041f 100644 --- a/test/e2e/git-dep.yaml +++ b/test/e2e/git-dep.yaml @@ -17,7 +17,7 @@ spec: name: gitsrv spec: containers: - - image: stefanprodan/gitsrv:0.0.5 + - image: stefanprodan/gitsrv:0.0.12 name: git env: - name: REPO diff --git a/test/e2e/known_hosts b/test/e2e/known_hosts index 48c28ec974..27da08378c 100644 --- a/test/e2e/known_hosts +++ b/test/e2e/known_hosts @@ -1,3 +1,4 @@ -gitsrv ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDt65M0saFA6Ewsjc8mQ405bI406BsFSJrbGz9GFrbWBn0Rs3Loi4S6ADWgTWlcspHvbfmf7vXW9oiTjxeke8oHPCfFYrHFdPK26Big2J1kE2DzRONizZWcdw8dpqRhvxlt2+/EJuuay8CGXz3VLCV8NgJc5AYmAwNPknUhWKxaAzjwvRd/0cyXr4vvcTDcmwR63oiWcRPkHCZ5LClFuZE065Ulm2o6CgItl+e6M5ouDSJWZDpYWWmmJJJv1DQoRruNbaf5f4bgWVkKjrQ/0cA4iWWlkwJMlAWQgp9saD0G2F83hrf2XapM/clWTFybkzPqPqaw2BEwXP6vWp6A1iUR -gitsrv ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM6WyUxQRPtZhYLxjG6YRQOHD/SX/+uI4XBo44UTSu21uqmf/lG8cLWTdMVzDlUDY9/Dx4tFz96LT97kUC1pLJs= -gitsrv ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAMv1z2YnDug5M4Klp3FMbBvw9NdFrx7Om5uEKFQs07t +# generated with "ssh-keyscan gitsrv" +gitsrv ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2WoJ2k+WA54pdxw5EGhg9CQBHKDVjHzNNlgRfTGrQBpgQT3/HEBi6BGi2ZmS6o6W9EJfzYzl3PvC+JY6BqcdM8XqbDazC1rkGtlycHd+dFT/TmWvBqJ2Oh+oJNL7IgpjBPJJMdAEc9nzUTTYa7V2A9SeaAyQJKGaftZhHEXTxkxxbWP2an7bzyw9QNCiF/ogQ79DPsp7ly4v4KgeGLSm9AoT/HO5+kJwXX3yQ1hKrFZyhzhaYiwzdApc3iUJtUEz1lKVX+63+WN6qhkbCUjlhfOGyT3qk18sMU6raqKt8uuQeR9f4/xkMXGWQuULhjGwOkju+8Dma8GvnhKKwHf5V +gitsrv ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFhuyD3SzMaTye/OX51Jb3fgZDxhGnXgJQ6oFvSSwqDGDm4fcueHE979xEPolNe9hn6jGg/2DS3xkU8boPKv8mo= +gitsrv ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAbLc9veRHa/l/kK6hmRWMA+QoWd8vLtLHbm4v6wj8XU From 6ab058e07ebbb6a71eb2d7b927b2291d4a6e124f Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 16 May 2019 11:46:08 +0200 Subject: [PATCH 18/26] Pin Kind's version to 0.2.1 Using master broke connectivity with the external world (at least when used CircleCI) --- test/e2e/e2e-kind.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/e2e-kind.sh b/test/e2e/e2e-kind.sh index 99b845f84b..41ba82362e 100755 --- a/test/e2e/e2e-kind.sh +++ b/test/e2e/e2e-kind.sh @@ -12,9 +12,11 @@ chmod +x kubectl && \ sudo mv kubectl /usr/local/bin/ echo ">>> Building sigs.k8s.io/kind" -cd $HOME -GO111MODULE="on" go get -u sigs.k8s.io/kind@master -cd $REPO_ROOT +# Hairy way to clone and build version 0.2.1 of Kind since it doesn't support Go Modules: +mkdir -p $GOPATH/src/sigs.k8s.io +git clone https://github.com/kubernetes-sigs/kind.git $GOPATH/src/sigs.k8s.io/kind +git -C $GOPATH/src/sigs.k8s.io/kind checkout tags/0.2.1 +go install sigs.k8s.io/kind echo ">>> Installing kind" sudo cp $GOPATH/bin/kind /usr/local/bin/ From bc9b0c6ae7ef7e4abd471ee3df18db1d01b430ef Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 16 May 2019 19:38:02 +0200 Subject: [PATCH 19/26] Refactor e2e test * Run sequentially after build and tests in CircleCI to reuse artifacts * Support running it locally (both in Linux and Darwin systems) through `make e2e`. An existing Kubernetes cluster will be used if available, otherwise a cluster will be created with Kind. --- .circleci/config.yml | 18 +-- .gitignore | 1 + Makefile | 7 +- test/e2e/e2e-flux-build.sh | 13 -- test/e2e/e2e-flux-chart.sh | 113 -------------- test/e2e/e2e-git.sh | 17 --- test/e2e/e2e-golang.sh | 19 --- test/e2e/e2e-helm.sh | 15 -- test/e2e/e2e-kind.sh | 26 ---- test/e2e/{git-dep.yaml => gitsrv.yaml} | 2 - test/e2e/run.sh | 199 +++++++++++++++++++++++++ 11 files changed, 207 insertions(+), 223 deletions(-) delete mode 100755 test/e2e/e2e-flux-build.sh delete mode 100755 test/e2e/e2e-flux-chart.sh delete mode 100755 test/e2e/e2e-git.sh delete mode 100755 test/e2e/e2e-golang.sh delete mode 100755 test/e2e/e2e-helm.sh delete mode 100755 test/e2e/e2e-kind.sh rename test/e2e/{git-dep.yaml => gitsrv.yaml} (96%) create mode 100755 test/e2e/run.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d47399a96f..a59c97c5e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: - run: make check-generated - run: make test TEST_FLAGS="-race -tags integration -timeout 60s" - run: make all - + - run: make e2e - deploy: name: Maybe push prerelease images command: | @@ -34,7 +34,6 @@ jobs: docker tag "docker.io/weaveworks/helm-operator:$(docker/image-tag)" "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)" docker push "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)" fi - - deploy: name: Maybe push release image and upload binaries command: | @@ -50,17 +49,6 @@ jobs: RELEASE_TAG=$(echo "$CIRCLE_TAG" | cut -c 6-) docker push "docker.io/weaveworks/helm-operator:${RELEASE_TAG}" fi - e2e-testing: - machine: true - working_directory: ~/go/src/github.com/weaveworks/flux - steps: - - checkout - - run: test/e2e/e2e-golang.sh - - run: test/e2e/e2e-flux-build.sh - - run: test/e2e/e2e-kind.sh - - run: test/e2e/e2e-helm.sh - - run: test/e2e/e2e-git.sh - - run: test/e2e/e2e-flux-chart.sh workflows: version: 2 @@ -70,6 +58,4 @@ workflows: filters: tags: only: /(helm-)?[0-9]+(\.[0-9]+)*(-[a-z]+)?/ - - e2e-testing: - requires: - - build + diff --git a/.gitignore b/.gitignore index 7e72e279e3..f0927ceda6 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ docker/fluxy-dumbconf.priv test/profiles test/bin/kubectl test/bin/helm +test/bin/kind diff --git a/Makefile b/Makefile index 1df81e4bb6..9e16730860 100644 --- a/Makefile +++ b/Makefile @@ -47,14 +47,17 @@ release-bins: clean: go clean rm -rf ./build - rm -f test/bin/kubectl test/bin/helm + rm -f test/bin/kubectl test/bin/helm test/bin/kind realclean: clean rm -rf ./cache -test: test/bin/helm test/bin/kubectl +test: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/weaveworks/flux/vendor" | sort -u) +e2e: test/bin/helm test/bin/kubectl + PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh + build/.%.done: docker/Dockerfile.% mkdir -p ./build/docker/$* cp $^ ./build/docker/$*/ diff --git a/test/e2e/e2e-flux-build.sh b/test/e2e/e2e-flux-build.sh deleted file mode 100755 index 709500e438..0000000000 --- a/test/e2e/e2e-flux-build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - -echo ">>> Installing go dep" -curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh -dep ensure -vendor-only - -echo ">>> Building docker images" -make all diff --git a/test/e2e/e2e-flux-chart.sh b/test/e2e/e2e-flux-chart.sh deleted file mode 100755 index 42713e3403..0000000000 --- a/test/e2e/e2e-flux-chart.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -REPO_ROOT=$(git rev-parse --show-toplevel) -KNOWN_HOSTS=$(cat ${REPO_ROOT}/test/e2e/known_hosts) -GITCONFIG=$(cat ${REPO_ROOT}/test/e2e/gitconfig) - -echo ">>> Loading $(docker/image-tag) into the cluster" -kind load docker-image "docker.io/weaveworks/flux:$(docker/image-tag)" -kind load docker-image "docker.io/weaveworks/helm-operator:$(docker/image-tag)" - -echo ">>> Installing Flux with Helm" -helm install --name flux --wait \ ---namespace flux \ ---set image.tag=$(docker/image-tag) \ ---set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \ ---set git.secretName=ssh-git \ ---set git.pollInterval=30s \ ---set git.config.secretName=gitconfig \ ---set git.config.enabled=true \ ---set-string git.config.data="${GITCONFIG}" \ ---set helmOperator.tag=$(docker/image-tag) \ ---set helmOperator.create=true \ ---set helmOperator.createCRD=true \ ---set helmOperator.git.secretName=ssh-git \ ---set registry.excludeImage=* \ ---set-string ssh.known_hosts="${KNOWN_HOSTS}" \ -${REPO_ROOT}/chart/flux - -echo '>>> Waiting for gitconfig secret' -retries=12 -count=0 -ok=false -until ${ok}; do - actual=$(kubectl get secrets -n flux gitconfig -ojsonpath={..data.gitconfig} | base64 -d) - if [ "${actual}" == "${GITCONFIG}" ]; then - echo -e "Expected Git configuration deployed\n" - kubectl get secrets -n flux gitconfig && echo - ok=true - else - ok=false - sleep 10 - fi - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux get secrets - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for namespace demo' -retries=12 -count=1 -ok=false -until ${ok}; do - kubectl describe ns/demo && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for workload podinfo' -retries=12 -count=0 -ok=false -until ${ok}; do - kubectl -n demo describe deployment/podinfo && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for Helm release mongodb' -retries=12 -count=0 -ok=false -until ${ok}; do - kubectl -n demo describe deployment/mongodb && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - kubectl -n flux logs deployment/flux-helm-operator - echo "No more retries left" - exit 1 - fi -done - -echo ">>> Flux logs" -kubectl -n flux logs deployment/flux - -echo ">>> Helm Operator logs" -kubectl -n flux logs deployment/flux-helm-operator - -echo ">>> List pods" -kubectl -n demo get pods - -echo ">>> Check workload" -kubectl -n demo rollout status deployment/podinfo - -echo ">>> Check Helm release" -kubectl -n demo rollout status deployment/mongodb diff --git a/test/e2e/e2e-git.sh b/test/e2e/e2e-git.sh deleted file mode 100755 index 725c50742c..0000000000 --- a/test/e2e/e2e-git.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -SCRIPT_DIR="${REPO_ROOT}/test/e2e" -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" - -echo ">>> Installing git" -kubectl create namespace flux -ssh-keygen -t rsa -N "" -f "${SCRIPT_DIR}/id_rsa" -kubectl create secret generic ssh-git --namespace=flux --from-file="${SCRIPT_DIR}/known_hosts" --from-file="${SCRIPT_DIR}/id_rsa" --from-file=identity="${SCRIPT_DIR}/id_rsa" --from-file="${SCRIPT_DIR}/id_rsa.pub" -rm "${SCRIPT_DIR}/id_rsa" "${SCRIPT_DIR}/id_rsa.pub" -kubectl apply -f "${SCRIPT_DIR}/git-dep.yaml" - -kubectl -n flux rollout status deployment/gitsrv - diff --git a/test/e2e/e2e-golang.sh b/test/e2e/e2e-golang.sh deleted file mode 100755 index dad56278a0..0000000000 --- a/test/e2e/e2e-golang.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -GO_VERSION=1.12.5 - -echo ">>> Installing go ${GO_VERSION}" -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz -tar -xf go${GO_VERSION}.linux-amd64.tar.gz -sudo rm -rf /usr/local/go -sudo mv go /usr/local - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - -mkdir -p $HOME/go/bin -mkdir -p $HOME/go/src - -go version diff --git a/test/e2e/e2e-helm.sh b/test/e2e/e2e-helm.sh deleted file mode 100755 index 85f12a898d..0000000000 --- a/test/e2e/e2e-helm.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" - -echo ">>> Installing Helm" -curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - -echo '>>> Installing Tiller' -kubectl --namespace kube-system create sa tiller -kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller -helm init --service-account tiller --upgrade --wait - diff --git a/test/e2e/e2e-kind.sh b/test/e2e/e2e-kind.sh deleted file mode 100755 index 41ba82362e..0000000000 --- a/test/e2e/e2e-kind.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo ">>> Installing kubectl" -curl -sLO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ -chmod +x kubectl && \ -sudo mv kubectl /usr/local/bin/ - -echo ">>> Building sigs.k8s.io/kind" -# Hairy way to clone and build version 0.2.1 of Kind since it doesn't support Go Modules: -mkdir -p $GOPATH/src/sigs.k8s.io -git clone https://github.com/kubernetes-sigs/kind.git $GOPATH/src/sigs.k8s.io/kind -git -C $GOPATH/src/sigs.k8s.io/kind checkout tags/0.2.1 -go install sigs.k8s.io/kind - -echo ">>> Installing kind" -sudo cp $GOPATH/bin/kind /usr/local/bin/ -kind create cluster --wait 5m - -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -kubectl get pods --all-namespaces diff --git a/test/e2e/git-dep.yaml b/test/e2e/gitsrv.yaml similarity index 96% rename from test/e2e/git-dep.yaml rename to test/e2e/gitsrv.yaml index 0b0da2041f..812bd4721e 100644 --- a/test/e2e/git-dep.yaml +++ b/test/e2e/gitsrv.yaml @@ -5,7 +5,6 @@ metadata: labels: name: gitsrv name: gitsrv - namespace: flux spec: replicas: 1 selector: @@ -46,7 +45,6 @@ metadata: labels: name: gitsrv name: gitsrv - namespace: flux spec: ports: - name: ssh diff --git a/test/e2e/run.sh b/test/e2e/run.sh new file mode 100755 index 0000000000..33a11c2d7e --- /dev/null +++ b/test/e2e/run.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +set -o errexit + +declare -a on_exit_items + +function on_exit() { + if [ "${#on_exit_items[@]}" -gt 0 ]; then + echo -e '\nRunning deferred items, please do not interrupt until they are done:' + fi + for I in "${on_exit_items[@]}"; do + echo "deferred: ${I}" + eval "${I}" + done +} + +# Cleaning up only makes sense in a local environment +# it just wastes time in CircleCI +if [ "${CI}" != 'true' ]; then + trap on_exit EXIT +fi + +function defer() { + on_exit_items=("$*" "${on_exit_items[@]}") +} + +REPO_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_DIR="${REPO_ROOT}/test/e2e" +KIND_VERSION=0.2.1 +CACHE_DIR="${REPO_ROOT}/cache/$CURRENT_OS_ARCH" +KIND_CACHE_PATH="${CACHE_DIR}/kind-$KIND_VERSION" +KIND_CLUSTER=flux-e2e +USING_KIND=false +FLUX_NAMESPACE=flux-e2e +DEMO_NAMESPACE=demo + + +# Check if there is a kubernetes cluster running, otherwise use Kind +if ! kubectl version > /dev/null 2>&1 ; then + if [ ! -f "${KIND_CACHE_PATH}" ]; then + echo '>>> Downloading Kind' + mkdir -p "${CACHE_DIR}" + curl -sL "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${CURRENT_OS_ARCH}" -o "${KIND_CACHE_PATH}" + fi + echo '>>> Creating Kind Kubernetes cluster' + cp "${KIND_CACHE_PATH}" "${REPO_ROOT}/test/bin/kind" + chmod +x "${REPO_ROOT}/test/bin/kind" + kind create cluster --name "${KIND_CLUSTER}" --wait 5m + defer kind --name "${KIND_CLUSTER}" delete cluster + export KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)" + USING_KIND=true + kubectl get pods --all-namespaces +fi + + +if ! helm version > /dev/null 2>&1; then + echo '>>> Installing Tiller' + kubectl --namespace kube-system create sa tiller + defer kubectl --namespace kube-system delete sa tiller + kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + defer kubectl delete clusterrolebinding tiller-cluster-rule + helm init --service-account tiller --upgrade --wait + defer helm reset --force +fi + +kubectl create namespace "$FLUX_NAMESPACE" +defer kubectl delete namespace "$FLUX_NAMESPACE" + +echo '>>> Installing mock git server' +ssh-keygen -t rsa -N "" -f "${SCRIPT_DIR}/id_rsa" +defer rm -f "${SCRIPT_DIR}/id_rsa" "${SCRIPT_DIR}/id_rsa.pub" +kubectl create secret generic ssh-git --namespace="${FLUX_NAMESPACE}" --from-file="${SCRIPT_DIR}/known_hosts" --from-file="${SCRIPT_DIR}/id_rsa" --from-file=identity="${SCRIPT_DIR}/id_rsa" --from-file="${SCRIPT_DIR}/id_rsa.pub" +kubectl apply -n "${FLUX_NAMESPACE}" -f "${SCRIPT_DIR}/gitsrv.yaml" +kubectl -n "${FLUX_NAMESPACE}" rollout status deployment/gitsrv + + +if [ "${USING_KIND}" = 'true' ]; then + echo '>>> Loading images into the Kind cluster' + kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/weaveworks/flux:latest' + kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/weaveworks/helm-operator:latest' +fi + +echo '>>> Installing Flux with Helm' + +KNOWN_HOSTS=$(cat "${REPO_ROOT}/test/e2e/known_hosts") +GITCONFIG=$(cat "${REPO_ROOT}/test/e2e/gitconfig") + +helm install --name flux --wait \ +--namespace "${FLUX_NAMESPACE}" \ +--set image.tag=latest \ +--set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \ +--set git.secretName=ssh-git \ +--set git.pollInterval=30s \ +--set git.config.secretName=gitconfig \ +--set git.config.enabled=true \ +--set-string git.config.data="$GITCONFIG" \ +--set helmOperator.tag=latest \ +--set helmOperator.create=true \ +--set helmOperator.createCRD=true \ +--set helmOperator.git.secretName=ssh-git \ +--set registry.excludeImage=* \ +--set-string ssh.known_hosts="$KNOWN_HOSTS" \ +"${REPO_ROOT}/chart/flux" + +defer helm delete --purge flux +# These CRDs are have a keep policy and need to be deleted manually +defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works + +echo -n '>>> Waiting for gitconfig secret ' +retries=24 +count=0 +ok=false +until ${ok}; do + actual=$(kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig -ojsonpath={..data.gitconfig} | base64 --decode) + if [ "${actual}" = "${GITCONFIG}" ]; then + echo ' Expected Git configuration deployed' + kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig && echo + ok=true + else + echo -n '.' + ok=false + sleep 5 + fi + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo ' No more retries left' + kubectl -n "${FLUX_NAMESPACE}" get secrets + exit 1 + fi +done + +echo -n ">>> Waiting for namespace ${DEMO_NAMESPACE} " +retries=24 +count=1 +ok=false +until ${ok}; do + kubectl describe "ns/${DEMO_NAMESPACE}" && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo -n '>>> Waiting for workload podinfo ' +retries=24 +count=0 +ok=false +until ${ok}; do + kubectl -n "${DEMO_NAMESPACE}" describe deployment/podinfo && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo -n '>>> Waiting for Helm release mongodb ' +retries=24 +count=0 +ok=false +until ${ok}; do + kubectl -n $DEMO_NAMESPACE describe deployment/mongodb && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo '>>> Flux logs' +kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + +echo '>>> Helm Operator logs' +kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator + +echo '>>> List pods' +kubectl -n "${DEMO_NAMESPACE}" get pods + +echo '>>> Check workload' +kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/podinfo + +echo '>>> Check Helm release' +kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/mongodb + +echo -e '\nEnd to end test was successful!!\n' \ No newline at end of file From 0c53b073bd1f76b9565702a873392f0563bdab57 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 17 May 2019 02:46:45 +0200 Subject: [PATCH 20/26] Use machine builder --- .circleci/config.yml | 34 ++++++++++++++++++++++++---------- gpg/gpgtest/gpg.go | 2 +- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a59c97c5e1..3b083ad986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,27 +1,41 @@ version: 2 jobs: build: - working_directory: /go/src/github.com/weaveworks/flux - docker: - - image: circleci/golang:1.10 - - image: memcached + working_directory: ~/go/src/github.com/weaveworks/flux + machine: true + environment: + GO_VERSION: 1.12.5 + GOPATH: /home/circleci/go + PATH: /bin:/usr/bin:/usr/local/go/bin:/home/circleci/go/bin steps: - checkout - - setup_remote_docker - - run: - # Ensure latest version of git + name: Install Golang + command: | + curl -OL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz + tar -xf go${GO_VERSION}.linux-amd64.tar.gz + sudo rm -rf /usr/local/go + sudo mv go /usr/local + mkdir -p "$HOME/go/bin" + go version + - run: + name: Update packages and Start Memcached command: | - echo "deb http://deb.debian.org/debian stretch-backports main" | sudo tee -a /etc/apt/sources.list.d/stretch-backports.list + # These repos fail and we don't need them: + sudo rm /etc/apt/sources.list.d/circleci_trusty.list /etc/apt/sources.list.d/google-chrome.list sudo apt-get update - sudo apt-get install -t stretch-backports -y --only-upgrade git + sudo apt-get install -y git rng-tools docker-ce memcached git version - - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + docker version + - run: + name: Install Dep + command: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - run: dep ensure -vendor-only - run: make check-generated - run: make test TEST_FLAGS="-race -tags integration -timeout 60s" - run: make all - run: make e2e + - deploy: name: Maybe push prerelease images command: | diff --git a/gpg/gpgtest/gpg.go b/gpg/gpgtest/gpg.go index aecbb2ac3f..d1b574ed35 100644 --- a/gpg/gpgtest/gpg.go +++ b/gpg/gpgtest/gpg.go @@ -39,7 +39,7 @@ func GPGKey(t *testing.T) (string, string, func()) { t.Fatal(err) } - gpgCmd := exec.Command("gpg", "--homedir", newDir, "--list-keys", "--with-colons") + gpgCmd := exec.Command("gpg", "--homedir", newDir, "--list-keys", "--with-colons", "--with-fingerprint") grepCmd := exec.Command("grep", "^fpr") cutCmd := exec.Command("cut", "-d:", "-f10") From 352a3094bcec7984e78382407252a89acbdf4cd8 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 17 May 2019 19:17:26 +0200 Subject: [PATCH 21/26] Adjust cleanup scheduling and handle existing CRDs --- test/e2e/run.sh | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 33a11c2d7e..1d3feb629f 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -45,8 +45,8 @@ if ! kubectl version > /dev/null 2>&1 ; then echo '>>> Creating Kind Kubernetes cluster' cp "${KIND_CACHE_PATH}" "${REPO_ROOT}/test/bin/kind" chmod +x "${REPO_ROOT}/test/bin/kind" + defer kind --name "${KIND_CLUSTER}" delete cluster > /dev/null 2>&1 kind create cluster --name "${KIND_CLUSTER}" --wait 5m - defer kind --name "${KIND_CLUSTER}" delete cluster export KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)" USING_KIND=true kubectl get pods --all-namespaces @@ -82,9 +82,22 @@ fi echo '>>> Installing Flux with Helm' +CREATE_CRDS='true' +if kubectl get crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works > /dev/null 2>&1; then + # CRDs existed, don't try to create them + echo 'CRDs existed, setting helmOperator.createCRD=false' + CREATE_CRDS='false' +else + # Schedule CRD deletion before calling helm, since it may fail and create them anyways + defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works > /dev/null 2>&1 +fi + KNOWN_HOSTS=$(cat "${REPO_ROOT}/test/e2e/known_hosts") GITCONFIG=$(cat "${REPO_ROOT}/test/e2e/gitconfig") + +defer helm delete --purge flux > /dev/null 2>&1 + helm install --name flux --wait \ --namespace "${FLUX_NAMESPACE}" \ --set image.tag=latest \ @@ -93,18 +106,18 @@ helm install --name flux --wait \ --set git.pollInterval=30s \ --set git.config.secretName=gitconfig \ --set git.config.enabled=true \ ---set-string git.config.data="$GITCONFIG" \ +--set-string git.config.data="${GITCONFIG}" \ --set helmOperator.tag=latest \ --set helmOperator.create=true \ --set helmOperator.createCRD=true \ --set helmOperator.git.secretName=ssh-git \ --set registry.excludeImage=* \ ---set-string ssh.known_hosts="$KNOWN_HOSTS" \ +--set-string ssh.known_hosts="${KNOWN_HOSTS}" \ +--set helmOperator.createCRD="${CREATE_CRDS}" \ "${REPO_ROOT}/chart/flux" -defer helm delete --purge flux -# These CRDs are have a keep policy and need to be deleted manually -defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works + + echo -n '>>> Waiting for gitconfig secret ' retries=24 From 63aa3ef112b5f37fe5ffc972c666767e0a69918b Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Mon, 20 May 2019 14:02:09 +0200 Subject: [PATCH 22/26] Add container dependencies where they belong --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9e16730860..8b64600f21 100644 --- a/Makefile +++ b/Makefile @@ -52,10 +52,10 @@ clean: realclean: clean rm -rf ./cache -test: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done +test: test/bin/helm test/bin/kubectl PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/weaveworks/flux/vendor" | sort -u) -e2e: test/bin/helm test/bin/kubectl +e2e: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh build/.%.done: docker/Dockerfile.% From 844b447b3a003854d3172b95b8507305f9c65b2c Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Mon, 20 May 2019 14:02:59 +0200 Subject: [PATCH 23/26] Add EOL --- test/e2e/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 1d3feb629f..b5f400a3bc 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -209,4 +209,4 @@ kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/podinfo echo '>>> Check Helm release' kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/mongodb -echo -e '\nEnd to end test was successful!!\n' \ No newline at end of file +echo -e '\nEnd to end test was successful!!\n' From 0bd3587795a2ca02c68295712e6ea84980cf2f61 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 22 May 2019 13:03:01 +0200 Subject: [PATCH 24/26] Bump Flux deployment to 1.12.3 --- deploy/flux-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/flux-deployment.yaml b/deploy/flux-deployment.yaml index 7bfd7cdbbd..a502bd7aa8 100644 --- a/deploy/flux-deployment.yaml +++ b/deploy/flux-deployment.yaml @@ -54,7 +54,7 @@ spec: # There are no ":latest" images for flux. Find the most recent # release or image version at https://hub.docker.com/r/weaveworks/flux/tags # and replace the tag here. - image: docker.io/weaveworks/flux:1.12.2 + image: docker.io/weaveworks/flux:1.12.3 imagePullPolicy: IfNotPresent resources: requests: From 0c0215a43f6639bf222a40bf71547fd10091b738 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 22 May 2019 13:12:47 +0200 Subject: [PATCH 25/26] Add entry for 1.12.3 in CHANGELOG.md --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 259bc50970..dd26b65213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,46 @@ This is the changelog for the Flux daemon; the changelog for the Helm operator is in [./CHANGELOG-helmop.md](./CHANGELOG-helmop.md). +## 1.12.3 (2019-05-22) + +This is a patch release. + +### Fixes + +- Show tag image for workload in list-images + [weaveworks/flux#2024][] +- Log warning when not applying resource by namespace + [weaveworks/flux#2034][] +- Always list the status of a workload in `fluxctl` + [weaveworks/flux#2035][] +- Ensure Flux installs gnutls >=3.6.7, to resolve security scan issues + [weaveworks/flux#2001][] +- Rename controller to workload in `fluxctl release` + [weaveworks/flux#2048][] +- Give full output of git command on errors + [weaveworks/flux#2054][] + +### Maintenance and documentation + +- Warn about Flux only supporting YAML and not JSON + [weaveworks/flux#2010][] +- Fix and refactor end-to-end tests + [weaveworks/flux#2050][] [weaveworks/flux#2058][] + +### Thanks + +Thanks to @2opremio, @hiddeco, @squaremo and @xtellurian for contributions. + +[weaveworks/flux#2010]: https://github.com/weaveworks/flux/pull/2010 +[weaveworks/flux#2024]: https://github.com/weaveworks/flux/pull/2024 +[weaveworks/flux#2034]: https://github.com/weaveworks/flux/pull/2034 +[weaveworks/flux#2035]: https://github.com/weaveworks/flux/pull/2035 +[weaveworks/flux#2044]: https://github.com/weaveworks/flux/pull/2044 +[weaveworks/flux#2048]: https://github.com/weaveworks/flux/pull/2048 +[weaveworks/flux#2050]: https://github.com/weaveworks/flux/pull/2050 +[weaveworks/flux#2054]: https://github.com/weaveworks/flux/pull/2054 +[weaveworks/flux#2058]: https://github.com/weaveworks/flux/pull/2058 + ## 1.12.2 (2019-05-08) This is a patch release. From 5668c02251eb148b1cbc643946ce13e73b8dcb50 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 22 May 2019 14:49:06 +0200 Subject: [PATCH 26/26] Fix typo en CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd26b65213..f5fcea3e22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ This is a patch release. - Always list the status of a workload in `fluxctl` [weaveworks/flux#2035][] - Ensure Flux installs gnutls >=3.6.7, to resolve security scan issues - [weaveworks/flux#2001][] + [weaveworks/flux#2044][] - Rename controller to workload in `fluxctl release` [weaveworks/flux#2048][] - Give full output of git command on errors