cluster-launch-installer-e2e.yaml

kind: Template
apiVersion: template.openshift.io/v1

parameters:
- name: JOB_NAME_SAFE
  required: true
- name: JOB_NAME_HASH
  required: true
- name: NAMESPACE
  required: true
- name: IMAGE_FORMAT
  required: true
- name: IMAGE_INSTALLER
  required: true
- name: IMAGE_TESTS
  required: true
- name: CLUSTER_TYPE
  required: true
# Ensures the release image is created and tested
- name: TEST_COMMAND
  required: true
- name: RELEASE_IMAGE_LATEST
  required: true

objects:

# We want the cluster to be able to access these images
- kind: RoleBinding
  apiVersion: authorization.openshift.io/v1
  metadata:
    name: ${JOB_NAME_SAFE}-image-puller
    namespace: ${NAMESPACE}
  roleRef:
    name: system:image-puller
  subjects:
  - kind: SystemGroup
    name: system:unauthenticated

# The e2e pod spins up a cluster, runs e2e tests, and then cleans up the cluster.
- kind: Pod
  apiVersion: v1
  metadata:
    name: ${JOB_NAME_SAFE}
    namespace: ${NAMESPACE}
    annotations:
      # we want to gather the teardown logs no matter what
      ci-operator.openshift.io/wait-for-container-artifacts: teardown
  spec:
    restartPolicy: Never
    activeDeadlineSeconds: 10800
    terminationGracePeriodSeconds: 900
    volumes:
    - name: artifacts
      emptyDir: {}
    - name: shared-tmp
      emptyDir: {}
    - name: cluster-profile
      secret:
        secretName: ${JOB_NAME_SAFE}-cluster-profile

    containers:

    # Once the cluster is up, executes shared tests
    - name: test
      image: ${IMAGE_TESTS}
      resources:
        requests:
          cpu: 1
          memory: 300Mi
        limits:
          cpu: 3
          memory: 2Gi
      volumeMounts:
      - name: shared-tmp
        mountPath: /tmp/shared
      - name: cluster-profile
        mountPath: /tmp/cluster
      - name: artifacts
        mountPath: /tmp/artifacts
      env:
      - name: HOME
        value: /tmp/home
      - name: KUBECONFIG
        value: /tmp/artifacts/installer/auth/kubeconfig
      command:
      - /bin/bash
      - -c
      - |
        #!/bin/bash
        set -euo pipefail

        export PATH=/usr/libexec/origin:$PATH

        trap 'touch /tmp/shared/exit' EXIT
        trap 'kill $(jobs -p); exit 0' TERM

        cp "$(which oc)" /tmp/shared/

        mkdir -p "${HOME}"

        # wait for the router namespace
        FOUND_KUBECONFIG=
        API_UP=
        ROUTER_NAMESPACE=
        while true; do
          if [[ -f /tmp/shared/exit ]]; then
            echo "Another process exited" 2>&1
            exit 1
          fi
          if [[ ! -f "${KUBECONFIG}" ]]; then
            sleep 15 & wait
            continue
          elif [[ -z "${FOUND_KUBECONFIG}" ]]; then
            echo "Found shared kubeconfig"
            FOUND_KUBECONFIG=1

            # don't let clients impact the global kubeconfig
            cp "${KUBECONFIG}" /tmp/admin.kubeconfig
            export KUBECONFIG=/tmp/admin.kubeconfig
          fi
          if ! oc get nodes 2>/dev/null; then
            echo "Waiting for API at $(oc whoami --show-server) to respond ..."
            sleep 15 & wait
            continue
          elif [[ -z "${API_UP}" ]]; then
            echo "API at $(oc whoami --show-server) has responded"
            API_UP=1
          fi
          if [[ -z "${ROUTER_NAMESPACE}" ]]; then
            # check multiple namespaces while we are transitioning to the new locations
            if oc get deploy/router -n tectonic-ingress 2>/dev/null; then
              ROUTER_NAMESPACE=tectonic-ingress
            elif oc get ds/router-default -n openshift-ingress 2>/dev/null; then
              ROUTER_NAMESPACE=openshift-ingress
            elif oc get deploy/router -n openshift-ingress 2>/dev/null; then
              ROUTER_NAMESPACE=openshift-ingress
            elif oc get deploy/router -n default 2>/dev/null; then
              ROUTER_NAMESPACE=default
            else
              echo "Waiting for router to be created ..."
              sleep 15 & wait
              continue
            fi
            echo "Found router in ${ROUTER_NAMESPACE}"
          fi
          break
        done

        i=0
        MAX_RETRIES=10
        until oc wait deploy/router -n "${ROUTER_NAMESPACE}" --for condition=available --timeout=10m || [ $i -eq $MAX_RETRIES ]; do
            i=$((i+1))
            [ $i -eq $MAX_RETRIES ] && echo "timeout waiting for router to be available" && exit 1
            echo "error deploy/router did not come up"
            sleep 60
        done

        # set up cloud-provider-specific env vars
        if [[ "${CLUSTER_TYPE}" == "gcp" ]]; then
          export GOOGLE_APPLICATION_CREDENTIALS="/tmp/cluster/gce.json"
          export KUBE_SSH_USER=cloud-user
          mkdir -p ~/.ssh
          cp /tmp/cluster/ssh-privatekey ~/.ssh/google_compute_engine || true
          export PROVIDER_ARGS='-provider=gce -gce-zone=us-east1-c -gce-project=openshift-gce-devel-ci'
        elif [[ "${CLUSTER_TYPE}" == "aws" ]]; then
          export PROVIDER_ARGS="-provider=aws -gce-zone=us-east-1"
          mkdir -p ~/.ssh
          cp /tmp/cluster/ssh-privatekey ~/.ssh/kube_aws_rsa || true
        fi

        mkdir -p /tmp/output
        cd /tmp/output

        # install telemetry client (TEMPORARY)
        set +e
        curl https://raw.githubusercontent.com/openshift/telemeter/master/deploy/default-rules > /tmp/rules
        oc create ns openshift-telemetry
        oc adm policy add-cluster-role-to-user cluster-reader -z default -n openshift-telemetry
        oc create secret generic telemeter-client --from-literal=to=https://telemeter.svc.ci.openshift.org --from-literal=id=${NAMESPACE}-${JOB_NAME_HASH} --from-file=token=/tmp/cluster/telemeter-token --from-literal=salt=random-salt --from-file=rules=/tmp/rules --dry-run -o yaml | oc apply -n openshift-telemetry -f -
        oc create -f https://raw.githubusercontent.com/openshift/telemeter/master/deploy/telemeter-client-openshift.yaml -n openshift-telemetry
        set -e

        # TODO: the test binary should really be a more structured command - most of these flags should be
        #       autodetected from the running cluster.
        # TODO: bump nodes up to 40 again
        function run-tests() {
          if [[ -n "${TEST_FOCUS:-}" ]]; then
            ginkgo -v -noColor -nodes="${TEST_PARALLELISM:-30}" $( which extended.test ) -- \
              -ginkgo.focus="${TEST_FOCUS}" -ginkgo.skip="${TEST_SKIP:-"\\[local\\]"}" \
              -e2e-output-dir /tmp/artifacts -report-dir /tmp/artifacts/junit \
              -test.timeout=2h ${PROVIDER_ARGS-} || rc=$?
          fi
          if [[ -n "${TEST_FOCUS_SERIAL:-}" ]]; then
            ginkgo -v -noColor -nodes=1 $( which extended.test ) -- \
              -ginkgo.focus="${TEST_FOCUS_SERIAL}" -ginkgo.skip="${TEST_SKIP_SERIAL:-"\\[local\\]"}" \
              -e2e-output-dir /tmp/artifacts -report-dir /tmp/artifacts/junit/serial \
              -test.timeout=2h ${PROVIDER_ARGS-} || rc=$?
          fi
          exit ${rc:-0}
        }

        ${TEST_COMMAND}

    # Runs an install
    - name: setup
      image: ${IMAGE_INSTALLER}
      volumeMounts:
      - name: shared-tmp
        mountPath: /tmp
      - name: cluster-profile
        mountPath: /etc/openshift-installer
      - name: artifacts
        mountPath: /tmp/artifacts
      env:
      - name: TYPE
        value: ${CLUSTER_TYPE}
      - name: AWS_SHARED_CREDENTIALS_FILE
        value: /etc/openshift-installer/.awscred
      - name: OPENSHIFT_INSTALL_CLUSTER_NAME
        value: ${NAMESPACE}-${JOB_NAME_HASH}
      - name: OPENSHIFT_INSTALL_BASE_DOMAIN
        value: origin-ci-int-aws.dev.rhcloud.com
      - name: OPENSHIFT_INSTALL_EMAIL_ADDRESS
        value: test@ci.openshift.io
      - name: OPENSHIFT_INSTALL_PASSWORD
        value: verysecure
      - name: OPENSHIFT_INSTALL_SSH_PUB_KEY_PATH
        value: /etc/openshift-installer/ssh-publickey
      - name: OPENSHIFT_INSTALL_PULL_SECRET_PATH
        value: /etc/openshift-installer/pull-secret
      - name: OPENSHIFT_INSTALL_PLATFORM
        value: aws
      - name: OPENSHIFT_INSTALL_AWS_REGION
        value: us-east-1
      - name: OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE
        value: ${RELEASE_IMAGE_LATEST}
      command:
      - /bin/sh
      - -c
      - |
        #!/bin/sh
        export _CI_ONLY_STAY_AWAY_OPENSHIFT_INSTALL_AWS_USER_TAGS="{\"expirationDate\": \"$(date -d '4 hours' --iso=minutes --utc)\"}"
        exec /bin/openshift-install --dir=/tmp/artifacts/installer --log-level=debug cluster

    # Performs cleanup of all created resources
    - name: teardown
      image: ${IMAGE_INSTALLER}
      volumeMounts:
      - name: shared-tmp
        mountPath: /tmp/shared
      - name: cluster-profile
        mountPath: /etc/openshift-installer
      - name: artifacts
        mountPath: /tmp/artifacts
      env:
      - name: INSTANCE_PREFIX
        value: ${NAMESPACE}-${JOB_NAME_HASH}
      - name: TYPE
        value: ${CLUSTER_TYPE}
      - name: KUBECONFIG
        value: /tmp/artifacts/installer/auth/kubeconfig
      command:
      - /bin/bash
      - -c
      - |
        #!/bin/bash
        function queue() {
          local TARGET="${1}"
          shift
          local LIVE="$(jobs | wc -l)"
          while [[ "${LIVE}" -ge 45 ]]; do
            sleep 1
            LIVE="$(jobs | wc -l)"
          done
          echo "${@}"
          if [[ -n "${FILTER}" ]]; then
            "${@}" | "${FILTER}" >"${TARGET}" &
          else
            "${@}" >"${TARGET}" &
          fi
        }

        function teardown() {
          set +e
          touch /tmp/shared/exit
          export PATH=$PATH:/tmp/shared

          echo "Gathering artifacts ..."
          mkdir -p /tmp/artifacts/pods /tmp/artifacts/nodes /tmp/artifacts/metrics

          oc --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.metadata.name}{"\n"}{end}' > /tmp/nodes
          oc --request-timeout=5s get pods --all-namespaces --template '{{ range .items }}{{ $name := .metadata.name }}{{ $ns := .metadata.namespace }}{{ range .spec.containers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ range .spec.initContainers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ end }}' > /tmp/containers
          oc --request-timeout=5s get pods -l openshift.io/component=api --all-namespaces --template '{{ range .items }}-n {{ .metadata.namespace }} {{ .metadata.name }}{{ "\n" }}{{ end }}' > /tmp/pods-api

          queue /tmp/artifacts/nodes.json oc --request-timeout=5s get nodes -o json
          queue /tmp/artifacts/events.json oc --request-timeout=5s get events --all-namespaces -o json

          # gather nodes first in parallel since they may contain the most relevant debugging info
          while IFS= read -r i; do
            mkdir -p /tmp/artifacts/nodes/$i
            FILTER=gzip queue /tmp/artifacts/nodes/$i/messages.gz oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/logs/messages
            oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/logs/journal | sed -e 's|.*href="\(.*\)".*|\1|;t;d' > /tmp/journals
            while IFS= read -r j; do
              FILTER=gzip queue /tmp/artifacts/nodes/$i/journal.gz oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/logs/journal/${j}system.journal
            done < /tmp/journals
            FILTER=gzip queue /tmp/artifacts/metrics/node-$i.gz oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/metrics
            queue /tmp/artifacts/nodes/$i/heap oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/debug/pprof/heap
            FILTER=gzip queue /tmp/artifacts/nodes/$i/secure.gz oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/logs/secure
            FILTER=gzip queue /tmp/artifacts/nodes/$i/audit.gz oc get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/logs/audit
          done < /tmp/nodes

          while IFS= read -r i; do
            file="$( echo "$i" | cut -d ' ' -f 3 | tr -s ' ' '_' )"
            queue /tmp/artifacts/metrics/${file}-heap oc exec $i -- /bin/bash -c 'oc get --raw /debug/pprof/heap --server "https://$( hostname ):8443" --config /etc/origin/master/admin.kubeconfig'
            FILTER=gzip queue /tmp/artifacts/metrics/${file}-api.gz oc exec $i -- /bin/bash -c 'oc get --raw /metrics --server "https://$( hostname ):8443" --config /etc/origin/master/admin.kubeconfig'
            queue /tmp/artifacts/metrics/${file}-controllers-heap oc exec $i -- /bin/bash -c 'oc get --raw /debug/pprof/heap --server "https://$( hostname ):8444" --config /etc/origin/master/admin.kubeconfig'
            FILTER=gzip queue /tmp/artifacts/metrics/${file}-controllers.gz oc exec $i -- /bin/bash -c 'oc get --raw /metrics --server "https://$( hostname ):8444" --config /etc/origin/master/admin.kubeconfig'
          done < /tmp/pods-api

          while IFS= read -r i; do
            file="$( echo "$i" | cut -d ' ' -f 2,3,5 | tr -s ' ' '_' )"
            FILTER=gzip queue /tmp/artifacts/pods/${file}.log.gz oc logs --request-timeout=20s $i
            FILTER=gzip queue /tmp/artifacts/pods/${file}_previous.log.gz oc logs --request-timeout=20s -p $i
          done < /tmp/containers

          echo "Waiting for logs ..."
          wait

          echo "Deprovisioning cluster ..."
          export AWS_SHARED_CREDENTIALS_FILE=/etc/openshift-installer/.awscred
          openshift-install --dir /tmp/artifacts/installer --log-level debug destroy-cluster
        }

        trap 'teardown' EXIT
        trap 'kill $(jobs -p); exit 0' TERM

        for i in `seq 1 120`; do
          if [[ -f /tmp/shared/exit ]]; then
            exit 0
          fi
          sleep 60 & wait
        done