tests: Add initial openshift testing

This change adds openshift installation into the CI process
and executes a simple test that verifies connectivity between
an openshift pod and the host.

Fixes: #220.

Depends-on: github.com/kata-containers/runtime#232

Signed-off-by: Salvador Fuentes <salvador.fuentes@intel.com>

.ci/install_openshift.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+cidir=$(dirname "$0")
+source /etc/os-release
+source "${cidir}/lib.sh"
+openshift_origin_version=$(get_version "externals.openshift.version")
+openshift_origin_commit=$(get_version "externals.openshift.commit")
+
+echo "Install Skopeo"
+skopeo_repo="github.com/projectatomic/skopeo"
+go get -d "$skopeo_repo" || true
+pushd "$GOPATH/src/$skopeo_repo"
+make binary-local
+sudo -E PATH=$PATH make install-binary
+popd
+
+echo "Install Openshift Origin"
+openshift_repo="github.com/openshift/origin"
+openshift_tarball="openshift-origin-server-${openshift_origin_version}-${openshift_origin_commit}-linux-64bit.tar.gz"
+openshift_dir="${openshift_tarball/.tar.gz/}"
+openshift_url="https://${openshift_repo}/releases/download/${openshift_origin_version}/${openshift_tarball}"
+
+curl -L -O "$openshift_url"
+tar -xf "$openshift_tarball"
+sudo install ${openshift_dir}/{openshift,oc,oadm} /usr/bin
+rm -rf "$openshift_dir" "${openshift_tarball}"
+
+echo "Openshift installed successfully"

.ci/setup.sh

@@ -58,6 +58,9 @@ bash -f ${cidir}/install_crio.sh
 echo "Install Kubernetes"
 bash -f ${cidir}/install_kubernetes.sh
 
+echo "Install Openshift"
+bash -f ${cidir}/install_openshift.sh
+
 echo "Drop caches"
 sync
 sudo -E PATH=$PATH bash -c "echo 3 > /proc/sys/vm/drop_caches"

Makefile

@@ -52,8 +52,24 @@ swarm:
 log-parser:
 	make -C cmd/log-parser
 
-test: functional integration crio docker-compose kubernetes swarm
+openshift:
+	bash -f .ci/install_bats.sh
+	bash -f integration/openshift/run_openshift_tests.sh
+
+test: functional integration crio docker-compose openshift kubernetes swarm
 
 check: checkcommits log-parser
 
-.PHONY: check checkcommits integration ginkgo log-parser test crio functional docker-compose kubernetes swarm
+.PHONY: \
+	check \
+	checkcommits \
+	crio \
+	docker-compose \
+	functional \
+	ginkgo \
+	integration \
+	kubernetes \
+	log-parser \
+	openshift \
+	swarm \
+	test

integration/openshift/data/hello-pod-kata.json

@@ -0,0 +1,52 @@
+{
+  "kind": "Pod",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "hello-openshift",
+    "creationTimestamp": null,
+    "labels": {
+      "name": "hello-openshift"
+    },
+	"annotations": {
+      "io.kubernetes.cri-o.TrustedSandbox": "false"
+    }
+  },
+  "spec": {
+    "containers": [
+      {
+        "name": "hello-openshift",
+        "image": "docker.io/openshift/hello-openshift",
+        "ports": [
+          {
+            "containerPort": 8080,
+            "protocol": "TCP"
+          }
+        ],
+        "resources": {},
+        "volumeMounts": [
+          {
+            "name":"tmp",
+            "mountPath":"/tmp"
+          }
+        ],
+        "terminationMessagePath": "/dev/termination-log",
+        "imagePullPolicy": "IfNotPresent",
+        "capabilities": {},
+        "securityContext": {
+          "capabilities": {},
+          "privileged": false
+        }
+      }
+    ],
+    "volumes": [
+      {
+        "name":"tmp",
+        "emptyDir": {}
+      }
+    ],
+    "restartPolicy": "Always",
+    "dnsPolicy": "ClusterFirst",
+    "serviceAccount": ""
+  },
+  "status": {}
+}

integration/openshift/hello_world.bats

@@ -0,0 +1,51 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/openshiftrc"
+load "${BATS_TEST_DIRNAME}/../../.ci/lib.sh"
+
+setup() {
+	# Verify there is a node ready to run containers,
+	# if not, sleep 5s and check again until maximum of 30s
+	cmd="sudo -E oc get nodes | grep \" Ready\""
+	wait_time=60
+	sleep_time=5
+	waitForProcess "$wait_time" "$sleep_time" "$cmd"
+	pod_name="hello-openshift"
+	image="openshift/${pod_name}"
+	sudo -E crictl pull "$image"
+	kata_runtime_bin=$(command -v kata-runtime)
+}
+
+@test "Hello Openshift using Kata Containers" {
+	# The below json file was taken from:
+	# https://github.com/openshift/origin/tree/master/examples/hello-openshift
+	# and modified to be an untrusted workload and then use Kata Containers.
+	sudo -E oc create -f "${BATS_TEST_DIRNAME}/data/hello-pod-kata.json"
+	output_file=$(mktemp)
+	cmd="sudo -E oc describe pod/${pod_name} | grep State | grep Running"
+	# Wait for nginx service to come up
+	waitForProcess "$wait_time" "$sleep_time" "$cmd"
+	container_id=$(sudo -E oc describe pod/${pod_name} | grep "Container ID" | cut -d '/' -f3)
+	# Verify that the running container is a Clear Container
+	sudo -E "$kata_runtime_bin" list | grep "$container_id" | grep "running"
+	# Verify connectivity
+	container_ip=$(sudo -E oc get pod "${pod_name}" -o yaml | grep "podIP" | awk '{print $2}')
+	container_port=$(sudo -E oc get pod "${pod_name}" -o yaml | grep "Port" | awk '{print $3}')
+	curl "${container_ip}:${container_port}" &> "$output_file"
+	grep "Hello OpenShift" "$output_file"
+}
+
+teardown() {
+	sudo -E oc describe pod/${pod_name} | grep State
+	rm "$output_file"
+	sudo -E oc delete pod "$pod_name"
+	# Wait for the pod to be deleted
+	cmd="sudo -E oc get pods | grep found."
+	waitForProcess "$wait_time" "$sleep_time" "$cmd"
+	sudo -E oc get pods
+}

integration/openshift/init.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/openshiftrc"
+
+echo "Start crio service"
+sudo systemctl start crio
+
+echo "Create configuration files"
+openshift start --write-config="$openshift_config_path"
+
+cp "$node_config" "$node_crio_config"
+
+cat << EOF >> "$node_crio_config"
+kubeletArguments:
+  node-labels:
+  - region=infra
+  image-service-endpoint:
+  - "unix:///var/run/crio/crio.sock"
+  container-runtime-endpoint:
+  - "unix:///var/run/crio/crio.sock"
+  container-runtime:
+  - "remote"
+  runtime-request-timeout:
+  - "15m"
+  cgroup-driver:
+  - "cgroupfs"
+EOF
+
+echo "Start Master"
+sudo -E openshift start master --config "$master_config" &> master.log &
+
+echo "Start Node"
+sudo -E openshift start node --config "$node_crio_config" &> node.log &

integration/openshift/openshiftrc

@@ -0,0 +1,11 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# These variables will be used for launching the cluster and
+# for executing the tests.
+openshift_config_path="$(pwd)/openshift.local.config"
+master_config="$openshift_config_path/master/master-config.yaml"
+node_config="$openshift_config_path/node-$(hostname)/node-config.yaml"
+node_crio_config="$openshift_config_path/node-$(hostname)/node-config-crio.yaml"
+export KUBECONFIG="$openshift_config_path/master/admin.kubeconfig"

integration/openshift/run_openshift_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+source /etc/os-release
+openshift_dir=$(dirname $0)
+
+# Currently, Kubernetes tests only work on Ubuntu.
+# We should delete this condition, when it works for other Distros.
+if [ "$ID" != ubuntu  ]; then
+    echo "Skip - Openshift tests on $ID aren't supported yet"
+    exit
+fi
+
+pushd "$openshift_dir"
+./init.sh
+bats hello_world.bats
+./teardown.sh
+popd

integration/openshift/teardown.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/openshiftrc"
+
+echo "Terminate Openshift master and node processes"
+pgrep openshift | xargs sudo kill -9
+
+echo "Stop cri-o service"
+sudo systemctl stop crio