ci: Run e2e tests on GCE

To get e2e tests to run on GCE, we do the following:
download gcloud, authenticate using secrets,
create cluster on GKE, push docker image to GCR,
and finally run the e2e test on the GKE cluster, using the image from GCR

Co-authored-by: Kosy Anyanwu <kosy@kinvolk.io>

Signed-off-by: Kosy Anyanwu <kosy@kinvolk.io>

.circleci/config.yml

@@ -3,6 +3,8 @@ jobs:
   build:
     machine: true
     working_directory: ~/.go_workspace/src/github.com/habitat-sh/habitat-operator
+    environment:
+      IMAGE_NAME: habitat/habitat-operator
     steps:
       - checkout
       - run:
@@ -12,14 +14,8 @@ jobs:
           name: setup
           environment:
             K8S_VERSION: v1.10.0
-            MINIKUBE_VERSION: v0.26.1
-            CHANGE_MINIKUBE_NONE_USER: true
           command: |
             curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-            curl -Lo minikube https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-            sudo minikube config set WantReportErrorPrompt false
-            # TODO: remove the --bootstrapper flag once this issue is solved: https://github.com/kubernetes/minikube/issues/2704
-            sudo -E minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=${K8S_VERSION} --extra-config=apiserver.Authorization.Mode=RBAC
       - run:
           name: code-gen script
           environment:
@@ -33,26 +29,66 @@ jobs:
       - run:
           name: unit tests
           command: make test
+      - run:
+          name: install google cloud sdk
+          command: |
+            sudo apt-get install lsb-release
+            CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"
+            echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+            curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+            sudo apt-get update && sudo apt-get install google-cloud-sdk
+
+            # the GCLOUD_SERVICE_KEY environment variable is set in the web UI
+            echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
+
+            # setup SDK
+            gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+            gcloud config set project $GCLOUD_PROJECT_ID # GCLOUD_PROJECT_ID is set in Circle CI web UI
+            gcloud config set compute/zone europe-west1-b
+      - run:
+          name: Update gcloud components
+          command: sudo /opt/google-cloud-sdk/bin/gcloud components update --quiet
+      - run:
+          name: boot cluster on gke
+          environment:
+            GKE_K8S_VERSION: 1.10.4-gke.2
+          command: gcloud container clusters create --cluster-version=$K8S_VERSION --disk-size=20 operator-test-$CIRCLE_BUILD_NUM
       - run:
           name: create image
           command: make TAG=testing image
+      - run:
+          name: Configure docker to use gcloud to authenticate requests to Container Registry
+          command: gcloud auth configure-docker
+      - run:
+          name: Tag the docker image
+          command: docker tag $IMAGE_NAME:testing eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing
+      - run:
+          name: Push the docker image
+          environment:
+            GCLOUD_COMPUTE_ZONE: europe-west1-b
+          command: docker push eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing
       - run:
           name: waiting for kubernetes to be ready
           command: |
             JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
             until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
               sleep 1
             done
+      - run:
+          name: Grant the user the ability to create authorization roles
+          command: kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)
       - run:
           name: e2e tests
-          command: make TESTIMAGE=habitat/habitat-operator:testing e2e
+          command: make TESTIMAGE=eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing e2e
       - run:
           name: print habitat object logs
           command: kubectl logs -lhabitat-operator=true --tail=100
           when: on_fail
-      # NOTE: this relies on journalctl, which is not present on the version of
-      # Ubuntu currently run by CircleCI
-      # - run:
-          # - name: print minikube logs
-          # - command: minikube logs
-          # - when: on_fail
+      - run:
+          name: Delete image from Container Registry
+          command: gcloud container images delete eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing --force-delete-tags --quiet
+          when: always
+      - run:
+          name: delete cluster on gke
+          command: gcloud container clusters delete operator-test-$CIRCLE_BUILD_NUM --quiet
+          when: always

test/e2e/v1beta1/framework/helpers.go

@@ -263,3 +263,33 @@ func QueryService(url string) (string, error) {
 
 	return string(bodyBytes), nil
 }
+
+// GetServiceIP gets the load balancer ip of a service if it exists or cluster ip
+func (f *Framework) GetServiceIP(serviceName string) (string, error) {
+	service, err := f.KubeClient.Core().Services(TestNs).Get(serviceName, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	if len(service.Status.LoadBalancer.Ingress) > 0 {
+		return service.Status.LoadBalancer.Ingress[0].IP, nil
+	}
+
+	return service.Spec.ClusterIP, nil
+}
+
+// WaitForLoadBalancerIP waits for Load Balancer IP to be available
+func (f *Framework) WaitForLoadBalancerIP(serviceName string) error {
+	return wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
+		service, err := f.KubeClient.Core().Services(TestNs).Get(serviceName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		if len(service.Status.LoadBalancer.Ingress) == 0 {
+			return false, nil
+		}
+
+		return true, nil
+	})
+}

test/e2e/v1beta1/main_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	operatorFramework "github.com/habitat-sh/habitat-operator/test/e2e/v1beta1/framework"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // Needed for GCP on Circle CI
 )
 
 var framework *operatorFramework.Framework

test/e2e/v1beta1/operator_test.go

@@ -100,10 +100,19 @@ func TestBind(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Wait until Load Balancer IP is ready
+	if err := framework.WaitForLoadBalancerIP(svc.ObjectMeta.Name); err != nil {
+		t.Fatal(err)
+	}
+
 	time.Sleep(serviceStartupWaitTime)
 
+	serviceIP, err := framework.GetServiceIP(svc.ObjectMeta.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Get response from Habitat Service.
-	url := fmt.Sprintf("http://%s:30001/", framework.ExternalIP)
+	url := fmt.Sprintf("http://%s:5555/", serviceIP)
 
 	body, err := utils.QueryService(url)
 	if err != nil {

test/e2e/v1beta1/resources/bind-config/service.yml

@@ -3,11 +3,10 @@ kind: Service
 metadata:
   name: test-go
 spec:
+  type: LoadBalancer
   selector:
     habitat-name: test-go
-  type: NodePort
   ports:
   - name: web
-    nodePort: 30001
     port: 5555
     protocol: TCP