Verify if pipeline works after upgrading from previous release to current release 🧗‍♂️

test/README.md

@@ -203,6 +203,18 @@ To run the YAML e2e tests, run the following command:
 ./test/e2e-tests-yaml.sh
 ```
 
+### Running upgrade tests
+
+There are two scenarios in upgrade tests. One is to install the previous release, upgrade to the current release, and
+validate whether the Tekton pipeline works. The other is to install the previous release, create the pipelines and tasks,
+upgrade to the current release, and validate whether the Tekton pipeline works.
+
+To run the upgrade tests, run the following command:
+
+```bash
+./test/e2e-tests-upgrade.sh
+```
+
 ### Adding integration tests
 
 In the [`test`](/test/) dir you will find several libraries in the `test`

test/e2e-common.sh

@@ -89,44 +89,82 @@ function check_results() {
  return ${failed}
 }
 
-function run_yaml_tests() {
- echo ">> Starting tests"
+function apply_resources() {
+ local resource=$1
+ echo ">> Applying the resource ${resource}"
 
- # Applying *taskruns
- for file in $(find ${REPO_ROOT_DIR}/examples/taskruns/ -name *.yaml | sort); do
+ # Applying the resources, either *taskruns or * *pipelineruns
+ for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name *.yaml | sort); do
  perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko apply -f - || return 1
  done
+}
 
- # Applying *pipelineruns
- for file in $(find ${REPO_ROOT_DIR}/examples/pipelineruns/ -name *.yaml | sort); do
- perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko apply -f - || return 1
- done
+function run_tests() {
+ local resource=$1
 
  # Wait for tests to finish.
- echo ">> Waiting for tests to finish for ${test}"
- if validate_run $1; then
+ echo ">> Waiting for tests to finish for ${resource}"
+ if validate_run $resource; then
  echo "ERROR: tests timed out"
  fi
 
  # Check that tests passed.
- echo ">> Checking test results for ${test}"
- if check_results $1; then
+ echo ">> Checking test results for ${resource}"
+ if check_results $resource; then
  echo ">> All YAML tests passed"
  return 0
  fi
-
  return 1
 }
 
+function run_yaml_tests() {
+ echo ">> Starting tests for the resource ${1}"
+ apply_resources $1
+ if ! run_tests ${1}; then
+ return 1
+ fi
+ return 0
+}
+
 function install_pipeline_crd() {
  echo ">> Deploying Tekton Pipelines"
  ko apply -f config/ || fail_test "Build pipeline installation failed"
+ verify_pipeline_installation
+}
+
+# Install the Tekton pipeline crd based on the release number
+function install_pipeline_crd_version() {
+ echo ">> Deploying Tekton Pipelines of Version $1"
+ kubectl apply -f "https://github.com/tektoncd/pipeline/releases/download/$1/release.yaml" || fail_test "Build pipeline installation failed of Version $1"
+ verify_pipeline_installation
+}
 
- # Make sure thateveything is cleaned up in the current namespace.
+function verify_pipeline_installation() {
+ # Make sure that everything is cleaned up in the current namespace.
  for res in conditions pipelineresources tasks pipelines taskruns pipelineruns; do
  kubectl delete --ignore-not-found=true ${res}.tekton.dev --all
  done
 
  # Wait for pods to be running in the namespaces we are deploying to
  wait_until_pods_running tekton-pipelines || fail_test "Tekton Pipeline did not come up"
 }
+
+function uninstall_pipeline_crd() {
+ echo ">> Uninstalling Tekton Pipelines"
+ ko delete --ignore-not-found=true -f config/
+
+ # Make sure that everything is cleaned up in the current namespace.
+ for res in conditions pipelineresources tasks pipelines taskruns pipelineruns; do
+ kubectl delete --ignore-not-found=true ${res}.tekton.dev --all
+ done
+}
+
+function uninstall_pipeline_crd_version() {
+ echo ">> Uninstalling Tekton Pipelines of version $1"
+ kubectl delete --ignore-not-found=true -f "https://github.com/tektoncd/pipeline/releases/download/$1/release.yaml"
+
+ # Make sure that everything is cleaned up in the current namespace.
+ for res in conditions pipelineresources tasks pipelines taskruns pipelineruns; do
+ kubectl delete --ignore-not-found=true ${res}.tekton.dev --all
+ done
+}

test/e2e-tests-upgrade.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script calls out to scripts in tektoncd/plumbing to setup a cluster
+# and deploy Tekton Pipelines to it for running upgrading tests. There are
+# two scenarios we need to cover in this script:
+
+# Scenario 1: install the previous release, upgrade to the current release, and
+# validate whether the Tekton pipeline works.
+
+# Scenario 2: install the previous release, create the pipelines and tasks, upgrade
+# to the current release, and validate whether the Tekton pipeline works.
+
+source $(dirname $0)/e2e-common.sh
+PREVIOUS_PIPELINE_VERSION=v0.5.2
+
+# Script entry point.
+
+initialize $@
+
+header "Setting up environment"
+
+# Handle failures ourselves, so we can dump useful info.
+set +o errexit
+set +o pipefail
+
+# First, we will verify if Scenario 1 works.
+# Install the previous release.
+header "Install the previous release of Tekton pipeline $PREVIOUS_PIPELINE_VERSION"
+install_pipeline_crd_version $PREVIOUS_PIPELINE_VERSION
+
+# Upgrade to the current release.
+header "Upgrade to the current release of Tekton pipeline"
+install_pipeline_crd
+
+# Run the integration tests.
+failed=0
+go_test_e2e -timeout=20m ./test || failed=1
+
+# Run the post-integration tests.
+for test in taskrun pipelinerun; do
+ header "Running YAML e2e tests for ${test}s"
+ if ! run_yaml_tests ${test}; then
+ echo "ERROR: one or more YAML tests failed"
+ output_yaml_test_results ${test}
+ output_pods_logs ${test}
+ failed=1
+ fi
+done
+
+# Remove all the pipeline CRDs, and clean up the environment for next Scenario.
+uninstall_pipeline_crd
+uninstall_pipeline_crd_version $PREVIOUS_PIPELINE_VERSION
+
+# Next, we will verify if Scenario 2 works.
+# Install the previous release.
+header "Install the previous release of Tekton pipeline $PREVIOUS_PIPELINE_VERSION"
+install_pipeline_crd_version $PREVIOUS_PIPELINE_VERSION
+
+# Create the resources of taskrun and pipelinerun, under the directories example/taskrun
+# and example/pipelinerun.
+for test in taskrun pipelinerun; do
+ header "Applying the resources ${test}s"
+ apply_resources ${test}
+done
+
+# Upgrade to the current release.
+header "Upgrade to the current release of Tekton pipeline"
+install_pipeline_crd
+
+# Run the integration tests.
+go_test_e2e -timeout=20m ./test || failed=1
+
+# Run the post-integration tests. We do not need to install the resources again, since
+# they are installed before the upgrade. We verify if they still work, after going through
+# the upgrade.
+for test in taskrun pipelinerun; do
+ header "Running YAML e2e tests for ${test}s"
+ if ! run_tests ${test}; then
+ echo "ERROR: one or more YAML tests failed"
+ output_yaml_test_results ${test}
+ output_pods_logs ${test}
+ failed=1
+ fi
+done
+
+(( failed )) && fail_test
+
+success