ci: Run e2e tests on GCE

To get e2e tests to run on GCE, we do the following:
download gcloud, authenticate using secrets,
create cluster on GKE, push docker image to GCR,
and finally run the e2e test on the GKE cluster, using the image from GCR

Co-authored-by: Lorenzo Manacorda <lorenzo@kinvolk.io>

Signed-off-by: Kosy Anyanwu <kosy@kinvolk.io>

.circleci/config.yml

@@ -9,17 +9,17 @@ jobs:
           name: Check for changes in documentation and examples
           command: ./hack/check-skippable-changes.sh
       - run:
-          name: setup
+          name: Setup environment variables
+          command: |
+            echo 'export IMAGE_NAME=eu.gcr.io/$GCLOUD_PROJECT_ID/habitat/habitat-operator-$CIRCLE_BUILD_NUM' >>"${BASH_ENV}"
+            echo 'export IMAGE_TAG=testing' >>"${BASH_ENV}"
+            echo 'export IMAGE_FULL_NAME="${IMAGE_NAME}:${IMAGE_TAG}"' >>"${BASH_ENV}"
+      - run:
+          name: setup kubectl
           environment:
             K8S_VERSION: v1.10.0
-            MINIKUBE_VERSION: v0.26.1
-            CHANGE_MINIKUBE_NONE_USER: true
           command: |
             curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-            curl -Lo minikube https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-            sudo minikube config set WantReportErrorPrompt false
-            # TODO: remove the --bootstrapper flag once this issue is solved: https://github.com/kubernetes/minikube/issues/2704
-            sudo -E minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=${K8S_VERSION} --extra-config=apiserver.Authorization.Mode=RBAC
       - run:
           name: code-gen script
           environment:
@@ -33,26 +33,61 @@ jobs:
       - run:
           name: unit tests
           command: make test
+      - run:
+          name: install google cloud sdk
+          command: |
+            sudo apt-get install lsb-release
+            CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"
+            echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+            curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+            sudo apt-get update && sudo apt-get install google-cloud-sdk
+
+            # the GCLOUD_SERVICE_KEY environment variable is set in the web UI
+            echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
+
+            # setup SDK
+            gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+            gcloud config set project $GCLOUD_PROJECT_ID # GCLOUD_PROJECT_ID is set in Circle CI web UI
+            gcloud config set compute/zone europe-west1-b
+      - run:
+          name: Update gcloud components
+          command: sudo /opt/google-cloud-sdk/bin/gcloud components update --quiet
+      - run:
+          name: boot cluster on gke
+          environment:
+            GKE_CLUSTER_VERSION: 1.10.4-gke.2
+          command: gcloud container clusters create --cluster-version=$GKE_CLUSTER_VERSION --disk-size=20 operator-test-$CIRCLE_BUILD_NUM
       - run:
           name: create image
-          command: make TAG=testing image
+          command: make IMAGE=$IMAGE_NAME TAG=testing image
+      - run:
+          name: Configure docker to use gcloud to authenticate requests to Container Registry
+          command: gcloud auth configure-docker
+      - run:
+          name: Push the docker image
+          command: docker push $IMAGE_FULL_NAME
       - run:
           name: waiting for kubernetes to be ready
           command: |
             JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
             until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
               sleep 1
             done
+      - run:
+          name: Grant the user the ability to create authorization roles
+          command: kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)
       - run:
           name: e2e tests
-          command: make TESTIMAGE=habitat/habitat-operator:testing e2e
+          command: make TESTIMAGE=$IMAGE_FULL_NAME e2e
       - run:
           name: print habitat object logs
           command: kubectl logs -lhabitat-operator=true --tail=100
           when: on_fail
-      # NOTE: this relies on journalctl, which is not present on the version of
-      # Ubuntu currently run by CircleCI
-      # - run:
-          # - name: print minikube logs
-          # - command: minikube logs
-          # - when: on_fail
+      - run:
+          name: Delete image from Container Registry
+          command: gcloud container images delete $IMAGE_FULL_NAME --force-delete-tags --quiet
+          when: always
+      - run:
+          name: delete cluster on gke
+          command: gcloud container clusters delete operator-test-$CIRCLE_BUILD_NUM --quiet
+          when: always

test/e2e/v1beta1/framework/helpers.go

@@ -263,3 +263,21 @@ func QueryService(url string) (string, error) {
 
 	return string(bodyBytes), nil
 }
+
+// GetLoadBalancerIP waits for Load Balancer IP to become available and returns it
+func (f *Framework) GetLoadBalancerIP(serviceName string) (string, error) {
+	loadBalancerIP := ""
+	err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
+		service, err := f.KubeClient.Core().Services(TestNs).Get(serviceName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		if len(service.Status.LoadBalancer.Ingress) == 0 {
+			return false, nil
+		}
+		loadBalancerIP = service.Status.LoadBalancer.Ingress[0].IP
+		return true, nil
+	})
+	return loadBalancerIP, err
+}

test/e2e/v1beta1/main_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	operatorFramework "github.com/habitat-sh/habitat-operator/test/e2e/v1beta1/framework"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // Needed for GCP on Circle CI
 )
 
 var framework *operatorFramework.Framework

test/e2e/v1beta1/operator_test.go

@@ -102,8 +102,12 @@ func TestBind(t *testing.T) {
 
 	time.Sleep(serviceStartupWaitTime)
 
+	loadBalancerIP, err := framework.GetLoadBalancerIP(svc.ObjectMeta.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Get response from Habitat Service.
-	url := fmt.Sprintf("http://%s:30001/", framework.ExternalIP)
+	url := fmt.Sprintf("http://%s:5555/", loadBalancerIP)
 
 	body, err := utils.QueryService(url)
 	if err != nil {

test/e2e/v1beta1/resources/bind-config/service.yml

@@ -3,11 +3,10 @@ kind: Service
 metadata:
   name: test-go
 spec:
+  type: LoadBalancer
   selector:
     habitat-name: test-go
-  type: NodePort
   ports:
   - name: web
-    nodePort: 30001
     port: 5555
     protocol: TCP