Run E2E tests on GCE

.circleci/config.yml

@@ -9,17 +9,17 @@ jobs:
  name: Check for changes in documentation and examples
  command: ./hack/check-skippable-changes.sh
  - run:
- name: setup
+ name: Setup environment variables
+ command: |
+ echo 'export IMAGE_NAME=eu.gcr.io/$GCLOUD_PROJECT_ID/habitat/habitat-operator-$CIRCLE_BUILD_NUM' >>"${BASH_ENV}"
+ echo 'export IMAGE_TAG=testing' >>"${BASH_ENV}"
+ echo 'export IMAGE_FULL_NAME="${IMAGE_NAME}:${IMAGE_TAG}"' >>"${BASH_ENV}"
+ - run:
+ name: setup kubectl
  environment:
  K8S_VERSION: v1.10.0
- MINIKUBE_VERSION: v0.26.1
- CHANGE_MINIKUBE_NONE_USER: true
  command: |
  curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
- curl -Lo minikube https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
- sudo minikube config set WantReportErrorPrompt false
- # TODO: remove the --bootstrapper flag once this issue is solved: https://github.com/kubernetes/minikube/issues/2704
- sudo -E minikube start --vm-driver=none --bootstrapper=localkube --kubernetes-version=${K8S_VERSION} --extra-config=apiserver.Authorization.Mode=RBAC
  - run:
  name: code-gen script
  environment:
@@ -33,26 +33,62 @@ jobs:
  - run:
  name: unit tests
  command: make test
+ - run:
+ name: install google cloud sdk
+ command: |
+ sudo apt-get install lsb-release
+ CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"
+ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+ sudo apt-get update && sudo apt-get install google-cloud-sdk
+
+ # the GCLOUD_SERVICE_KEY environment variable is set in the web UI
+ echo $GCLOUD_SERVICE_KEY > ${HOME}/gcloud-service-key.json
+
+ # setup SDK
+ gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json
+ # GCLOUD_PROJECT_ID is set in Circle CI web UI
+ gcloud config set project $GCLOUD_PROJECT_ID
+ gcloud config set compute/zone europe-west1-b
+ - run:
+ name: Update gcloud components
+ command: sudo /opt/google-cloud-sdk/bin/gcloud components update --quiet
+ - run:
+ name: boot cluster on gke
+ environment:
+ GKE_CLUSTER_VERSION: 1.10.4-gke.2
+ command: gcloud container clusters create --cluster-version=$GKE_CLUSTER_VERSION --disk-size=20 operator-test-$CIRCLE_BUILD_NUM
  - run:
  name: create image
- command: make TAG=testing image
+ command: make IMAGE=$IMAGE_NAME TAG=$IMAGE_TAG image
+ - run:
+ name: Configure docker to use gcloud to authenticate requests to Container Registry
+ command: gcloud auth configure-docker
+ - run:
+ name: Push the docker image
+ command: docker push $IMAGE_FULL_NAME
  - run:
  name: waiting for kubernetes to be ready
  command: |
  JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
  until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
  sleep 1
  done
+ - run:
+ name: Grant the user the ability to create authorization roles
+ command: kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)
  - run:
  name: e2e tests
- command: make TESTIMAGE=habitat/habitat-operator:testing e2e
+ command: make TESTIMAGE=$IMAGE_FULL_NAME e2e
  - run:
  name: print habitat object logs
  command: kubectl logs -lhabitat-operator=true --tail=100
  when: on_fail
- # NOTE: this relies on journalctl, which is not present on the version of
- # Ubuntu currently run by CircleCI
- # - run:
- # - name: print minikube logs
- # - command: minikube logs
- # - when: on_fail
+ - run:
+ name: Delete image from Container Registry
+ command: gcloud container images delete $IMAGE_FULL_NAME --force-delete-tags --quiet
+ when: always
+ - run:
+ name: delete cluster on gke
+ command: gcloud container clusters delete operator-test-$CIRCLE_BUILD_NUM --quiet
+ when: always

test/e2e/v1beta1/framework/helpers.go

@@ -263,3 +263,21 @@ func QueryService(url string) (string, error) {
 
  return string(bodyBytes), nil
 }
+
+// GetLoadBalancerIP waits for Load Balancer IP to become available and returns it
+func (f *Framework) GetLoadBalancerIP(serviceName string) (string, error) {
+ loadBalancerIP := ""
+ err := wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
+ service, err := f.KubeClient.Core().Services(TestNs).Get(serviceName, metav1.GetOptions{})
+ if err != nil {
+ return false, err
+ }
+
+ if len(service.Status.LoadBalancer.Ingress) == 0 {
+ return false, nil
+ }
+ loadBalancerIP = service.Status.LoadBalancer.Ingress[0].IP
+ return true, nil
+ })
+ return loadBalancerIP, err
+}

test/e2e/v1beta1/main_test.go

@@ -21,6 +21,7 @@ import (
  "testing"
 
  operatorFramework "github.com/habitat-sh/habitat-operator/test/e2e/v1beta1/framework"
+ _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // Needed for GCP on Circle CI
 )
 
 var framework *operatorFramework.Framework

test/e2e/v1beta1/operator_test.go

@@ -102,8 +102,12 @@ func TestBind(t *testing.T) {
 
  time.Sleep(serviceStartupWaitTime)
 
+ loadBalancerIP, err := framework.GetLoadBalancerIP(svc.ObjectMeta.Name)
+ if err != nil {
+ t.Fatal(err)
+ }
  // Get response from Habitat Service.
- url := fmt.Sprintf("http://%s:30001/", framework.ExternalIP)
+ url := fmt.Sprintf("http://%s:5555/", loadBalancerIP)
 
  body, err := utils.QueryService(url)
  if err != nil {

test/e2e/v1beta1/resources/bind-config/service.yml

@@ -3,11 +3,10 @@ kind: Service
 metadata:
  name: test-go
 spec:
+ type: LoadBalancer
  selector:
  habitat-name: test-go
- type: NodePort
  ports:
  - name: web
- nodePort: 30001
  port: 5555
  protocol: TCP