Merge pull request shipwright-io#40 from otaviof/e2e-bootstrap

Bootstrapping End-to-End Tests

.github/workflows/e2e.yaml

@@ -0,0 +1,59 @@
+---
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+name: End-to-End (E2E) Tests
+jobs:
+  e2e:
+    strategy:
+      fail-fast: true
+      matrix:
+        kubernetes:
+          - v1.20.7
+      max-parallel: 1
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Install Tools
+        run: sudo apt-get update && sudo apt-get install -y make gcc
+
+      - name: Setup BATS
+        uses: mig4/setup-bats@v1
+        with:
+          bats-version: 1.2.1
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: ${{ matrix.kubernetes }}
+
+      - name: Create KinD cluster
+        uses: helm/kind-action@v1.2.0
+        with:
+          version: v0.11.1
+          node_image: kindest/node:${{ matrix.kubernetes }}
+          cluster_name: kind
+          wait: 120s
+
+      - name: Verify KinD cluster
+        run: make verify-kind
+
+      - name: Installing Shipwright Build Controller
+        run: make install-shipwright
+
+      - name: Build Application (shp)
+        run: make build
+
+      - name: End-to-End Tests
+        run: make test-e2e

.github/workflows/unit.yaml

@@ -27,8 +27,8 @@ jobs:
     - name: Install Tools
       run: sudo apt-get update && sudo apt-get install -y make gcc
 
-    - name: Build Application
+    - name: Build Application (shp)
       run: make build
 
     - name: Run Unit Tests
-      run: make test
+      run: make test-unit

Makefile

@@ -12,6 +12,12 @@ GO_TEST_FLAGS ?= -race -cover
 
 ARGS ?=
 
+# Tekton and Shipwright Build Controller versions for CI
+TEKTON_VERSION ?= v0.25.0
+SHIPWRIGHT_VERSION ?= v0.5.1
+
+.EXPORT_ALL_VARIABLES:
+
 .PHONY: $(BIN)
 $(BIN):
 	go build $(GO_FLAGS) -o $(BIN) $(CMD)
@@ -36,12 +42,22 @@ clean:
 run:
 	go run $(GO_FLAGS) $(CMD) $(ARGS)
 
-test: test-unit
+# runs all tests, unit and end-to-end.
+test: test-unit test-e2e
 
 .PHONY: test-unit
 test-unit:
 	go test $(GO_FLAGS) $(GO_TEST_FLAGS) $(CMD) $(PKG) $(ARGS)
 
-travis:
-	./hack/install-kubectl.sh
-	./hack/install-kind.sh
+# runs all end-to-end tests using bats, it assumes bats is installed
+test-e2e:
+	./test/e2e/e2e.bats
+
+# wait for KinD cluster to be on ready state, so tests can be performed
+verify-kind:
+	./hack/verify-kind.sh
+
+# deploys Tekton and Shipwright Build Controller following the versions exported
+install-shipwright:
+	./hack/install-tekton.sh
+	./hack/install-shipwright.sh

README.md

@@ -59,11 +59,7 @@ make run ARGS='run build --help'
 
 ### Testing
 
-To execute all tests available in the project, run:
-
-```sh
-make test
-```
+Please consider specific testing [documentation here](docs/testing.md).
 
 ## Usage
 

docs/testing.md

@@ -0,0 +1,87 @@
+# Testing
+
+All testing tooling is managed by the `Makefile` on this project, therefore all you'll have to do is
+invoke one of those targets to test current changes.
+
+To run all projet tests, execute:
+
+```sh
+make test
+```
+
+## Unit-Testing
+
+To execute unit-testing present on this project, run:
+
+```sh
+make test-unit
+```
+
+## End-to-End (E2E)
+
+End-to-End tests aim to mimic the real world usage of this CLI against the
+[Shipwright Build Controller][shipwrightBuild]. To run end-to-end tests, make sure you have the
+latest changes compiled (`make build`), and then run the tests (`make test-e2e`), in short:
+
+```sh
+make build test-e2e
+```
+
+### Requirements
+
+The following componets are required to run end-to-end tests.
+
+#### Kubernetes
+
+Before testing a [KinD][kindSig] instance is created, running a predefined version of Kubernetes.
+Please consider [GitHub Action files](../.github/workflows/e2e.yaml) for more details. Additionally,
+`kubectl` is required to intereact with the Kubernetes Cluster, you need to install it before running
+scripts on the [`./hack` directory](../hack).
+
+#### Shipwright Build Controller
+
+The [Shipwright Build Controller][shipwrightBuild] must be up and running as well. To install the
+controller and its dependencies, run:
+
+```sh
+make install-shipwright
+```
+
+The install script waits for the Controller instance to be running.
+
+#### BATS
+
+[BATS][batsCore] is a testing framework for Bash. It's structured as a regular script with enhanced
+syntax to define test cases, collect results, and more.
+
+To run BATS based tests, make sure you have `bats` installed, and then execute:
+
+```sh
+make test-e2e
+```
+
+### Test Cases
+
+The usual [BATS][batsCore] test-cases can be defined as the following example:
+
+```bash
+@test "short test description, or test name" {
+	# prepare the test-case context running the necessary commands to do so.
+	kubectl ...
+
+	# then execute the actual test, usually by running "shp" command with arguments.
+	result=$(shp ...)
+
+	# at the end, assert the test result by inspecting the output, or maybe, execute probes to
+	# identify if the desired changes have been performed on the cluster, and such.
+	[ ! -z "${result}" ]
+}
+```
+
+Repetitive tasks can be defined as Bash `function`, the actual `shp` command employed during testing
+is overwritten to use executable compiled in the project folder.
+
+
+[kindSig]: https://kind.sigs.k8s.io/
+[batsCore]: https://github.com/bats-core/bats-core
+[shipwrightBuild]: https://github.com/shipwright-io/build

hack/install-shipwright.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Installs Shipwright Build Controller and Build-Strategies.
+#
+
+set -eu
+
+SHIPWRIGHT_HOST="github.com"
+SHIPWRIGHT_HOST_PATH="shipwright-io/build/releases/download"
+SHIPWRIGHT_VERSION="${SHIPWRIGHT_VERSION:-v0.5.1}"
+
+echo "# Deploying Shipwright Controller '${SHIPWRIGHT_VERSION}'"
+
+kubectl apply -f "https://${SHIPWRIGHT_HOST}/${SHIPWRIGHT_HOST_PATH}/${SHIPWRIGHT_VERSION}/release.yaml"
+
+echo "# Waiting for Build Controller rollout..."
+
+kubectl --namespace="shipwright-build" rollout status deployment shipwright-build-controller --timeout=1m
+
+echo "# Installing upstream Build-Strategies..."
+
+kubectl apply -f "https://${SHIPWRIGHT_HOST}/${SHIPWRIGHT_HOST_PATH}/nightly/default_strategies.yaml"

hack/install-tekton.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Installs Tekton Pipelines.
+#
+
+set -eu
+
+TEKTON_VERSION="${TEKTON_VERSION:-v0.25.0}"
+
+TEKTON_HOST="github.com"
+TEKTON_HOST_PATH="tektoncd/pipeline/releases/download"
+
+function rollout_status () {
+	kubectl --namespace="tekton-pipelines" rollout status deployment ${1} --timeout=1m
+}
+
+echo "# Deploying Tekton Pipelines '${TEKTON_VERSION}'"
+
+kubectl apply -f "https://${TEKTON_HOST}/${TEKTON_HOST_PATH}/${TEKTON_VERSION}/release.yaml"
+
+echo "# Waiting for Tekton components..."
+
+rollout_status "tekton-pipelines-controller"
+rollout_status "tekton-pipelines-webhook"

hack/verify-kind.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Make sure a KinD instance is up and running.
+#
+
+set -eu
+
+function node_status () {
+	echo $(kubectl get node kind-control-plane -o json | \
+		jq -r .'status.conditions[] | select(.type == "Ready") | .status')
+}
+
+echo "# Using KinD context..."
+kubectl config use-context "kind-kind"
+
+echo "# KinD nodes:"
+kubectl get nodes
+
+if [ "$(node_status)" == "True" ]; then
+	echo "# Kind is Ready!"
+else
+	echo "# Node is not ready:"
+	kubectl describe node kind-control-plane
+
+	echo "# Pods:"
+	kubectl get pod -A
+	echo "# Events:"
+	kubectl get events -A
+
+	exit 1
+fi

test/e2e/e2e.bats

@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+source test/e2e/helpers.sh
+
+@test "shp binary can be executed" {
+	result="$(shp --help)"
+	[ ! -z "$result" ]
+}

test/e2e/helpers.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eu
+
+BIN="${BIN:-./_output/shp}"
+
+function fail () {
+	echo $* >&2
+	exit 1
+}
+
+function shp () {
+	if [ ! -x "${BIN}" ] ; then
+		fail "Unable to find '${BIN}' executable"
+	fi
+
+	${BIN} ${*}
+}