ci: Run e2e tests on GCE

To get e2e tests to run on GCE, we do the following:
download gcloud, authenticate using secrets,
create cluster on GKE, push docker image to GCR,
and finally run the e2e test on the GKE cluster, using the image from GCR

Co-authored-by: Kosy Anyanwu <kosy@kinvolk.io>

Signed-off-by: Kosy Anyanwu <kosy@kinvolk.io>

.circleci/config.yml

@@ -3,6 +3,8 @@ jobs:
  build:
  machine: true
  working_directory: ~/.go_workspace/src/github.com/habitat-sh/habitat-operator
+ environment:
+ IMAGE_NAME: habitat/habitat-operator
  steps:
  - checkout
  - run:
@@ -12,14 +14,8 @@ jobs:
  name: setup
  environment:
  K8S_VERSION: v1.10.0
- MINIKUBE_VERSION: v0.26.1
- CHANGE_MINIKUBE_NONE_USER: true
  command: |
  curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
- curl -Lo minikube https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
- sudo minikube config set WantReportErrorPrompt false
- # TODO: remove the --bootstrapper flag once this issue is solved: https://github.com/kubernetes/minikube/issues/2704
- sudo -E minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=${K8S_VERSION} --extra-config=apiserver.Authorization.Mode=RBAC
  - run:
  name: code-gen script
  environment:
@@ -33,26 +29,66 @@ jobs:
  - run:
  name: unit tests
  command: make test
+ - run:
+ name: install google cloud sdk
+ command: |
+ sudo apt-get install lsb-release
+ CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"
+ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+ sudo apt-get update && sudo apt-get install google-cloud-sdk
+
+ # the GCLOUD_SERVICE_KEY environment variable is set in the web UI
+ echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
+
+ # setup SDK
+ gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+ gcloud config set project $GCLOUD_PROJECT_ID # GCLOUD_PROJECT_ID is set in Circle CI web UI
+ gcloud config set compute/zone europe-west1-b
+ - run:
+ name: Update gcloud components
+ command: sudo /opt/google-cloud-sdk/bin/gcloud components update --quiet
+ - run:
+ name: boot cluster on gke
+ environment:
+ GKE_K8S_VERSION: 1.10.4-gke.2
+ command: gcloud container clusters create --cluster-version=$K8S_VERSION --disk-size=20 operator-test-$CIRCLE_BUILD_NUM
  - run:
  name: create image
  command: make TAG=testing image
+ - run:
+ name: Configure docker to use gcloud to authenticate requests to Container Registry
+ command: gcloud auth configure-docker
+ - run:
+ name: Tag the docker image
+ command: docker tag $IMAGE_NAME:testing eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing
+ - run:
+ name: Push the docker image
+ environment:
+ GCLOUD_COMPUTE_ZONE: europe-west1-b
+ command: docker push eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing
  - run:
  name: waiting for kubernetes to be ready
  command: |
  JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
  until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
  sleep 1
  done
+ - run:
+ name: Grant the user the ability to create authorization roles
+ command: kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)
  - run:
  name: e2e tests
- command: make TESTIMAGE=habitat/habitat-operator:testing e2e
+ command: make TESTIMAGE=eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing e2e
  - run:
  name: print habitat object logs
  command: kubectl logs -lhabitat-operator=true --tail=100
  when: on_fail
- # NOTE: this relies on journalctl, which is not present on the version of
- # Ubuntu currently run by CircleCI
- # - run:
- # - name: print minikube logs
- # - command: minikube logs
- # - when: on_fail
+ - run:
+ name: Delete image from Container Registry
+ command: gcloud container images delete eu.gcr.io/$GCLOUD_PROJECT_ID/$IMAGE_NAME-$CIRCLE_BUILD_NUM:testing --force-delete-tags --quiet
+ when: always
+ - run:
+ name: delete cluster on gke
+ command: gcloud container clusters delete operator-test-$CIRCLE_BUILD_NUM --quiet
+ when: always

test/e2e/v1beta1/framework/helpers.go

@@ -263,3 +263,33 @@ func QueryService(url string) (string, error) {
 
  return string(bodyBytes), nil
 }
+
+// GetServiceIP gets the load balancer ip of a service if it exists or cluster ip
+func (f *Framework) GetServiceIP(serviceName string) (string, error) {
+ service, err := f.KubeClient.Core().Services(TestNs).Get(serviceName, metav1.GetOptions{})
+ if err != nil {
+ return "", err
+ }
+
+ if len(service.Status.LoadBalancer.Ingress) > 0 {
+ return service.Status.LoadBalancer.Ingress[0].IP, nil
+ }
+
+ return service.Spec.ClusterIP, nil
+}
+
+// WaitForLoadBalancerIP waits for Load Balancer IP to be available
+func (f *Framework) WaitForLoadBalancerIP(serviceName string) error {
+ return wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
+ service, err := f.KubeClient.Core().Services(TestNs).Get(serviceName, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+
+ if len(service.Status.LoadBalancer.Ingress) == 0 {
+ return false, nil
+ }
+
+ return true, nil
+ })
+}

test/e2e/v1beta1/main_test.go

@@ -21,6 +21,7 @@ import (
  "testing"
 
  operatorFramework "github.com/habitat-sh/habitat-operator/test/e2e/v1beta1/framework"
+ _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // Needed for GCP on Circle CI
 )
 
 var framework *operatorFramework.Framework

test/e2e/v1beta1/operator_test.go

@@ -100,10 +100,19 @@ func TestBind(t *testing.T) {
  t.Fatal(err)
  }
 
+ // Wait until Load Balancer IP is ready
+ if err := framework.WaitForLoadBalancerIP(svc.ObjectMeta.Name); err != nil {
+ t.Fatal(err)
+ }
+
  time.Sleep(serviceStartupWaitTime)
 
+ serviceIP, err := framework.GetServiceIP(svc.ObjectMeta.Name)
+ if err != nil {
+ t.Fatal(err)
+ }
  // Get response from Habitat Service.
- url := fmt.Sprintf("http://%s:30001/", framework.ExternalIP)
+ url := fmt.Sprintf("http://%s:5555/", serviceIP)
 
  body, err := utils.QueryService(url)
  if err != nil {

test/e2e/v1beta1/resources/bind-config/service.yml

@@ -3,11 +3,10 @@ kind: Service
 metadata:
  name: test-go
 spec:
+ type: LoadBalancer
  selector:
  habitat-name: test-go
- type: NodePort
  ports:
  - name: web
- nodePort: 30001
  port: 5555
  protocol: TCP