[RELEASE 0.1] Update test infrastructure for the 0.1 serving release (#…

…2208)

* Fix SSH keys workaround for kubetest

Create the ~/.ssh dir if it doesn't exist, don't assume it always exist.

Backported from knative/test-infra#151

* Fix authentication for test clusters

Instead of relying on default options, use basic authentication for test cluster.

Also make acquire_cluster_admin_role() handle auth through certificates, since it's used also on deployment.

Backport of knative/test-infra#115

* Increase E2E tests timeout to 20 minutes

Currently 10 minutes may not be enough, for example as in #1701.

Backport of #1702.

* Drop Dockerfile in test images

* removed dockerfiles
* moved test images to the same location
* adjust repo
* fix command and docs
* update upload-test-images.sh
* preserve import paths
* fix appYaml path
* preserve path in upload-test-images.sh

Backport of #1792.

test/README.md

@@ -98,6 +98,8 @@ go test -v -tags=e2e -count=1 ./test/e2e --resolvabledomain
 
 ### Building the test images
 
+Note: this is only required when you run conformance/e2e tests locally with `go test` commands.
+
 The [`upload-test-images.sh`](./upload-test-images.sh) script can be used to build and push the
 test images used by the conformance and e2e tests. It requires:
 
@@ -109,17 +111,12 @@ test images used by the conformance and e2e tests. It requires:
 To run the script for all end to end test images:
 
 ```bash
-./test/upload-test-images.sh ./test/e2e/test_images
-./test/upload-test-images.sh ./test/conformance/test_images
+./test/upload-test-images.sh
 ```
 
 ### Adding new test images
 
-New test images should be placed in their own subdirectories. Be sure to to include a `Dockerfile`
-for building and running the test image.
-
-The new test images will also need to be uploaded to the e2e tests Docker repo. You will need one
-of the owners found in [`/test/OWNERS`](OWNERS) to do this.
+New test images should be placed in `./test/test_images`.
 
 ## Flags
 

test/e2e-tests.sh

@@ -133,13 +133,20 @@ function dump_stack_info() {
   echo "***************************************"
 }
 
+function publish_test_images() {
+  local image_dirs="$(find ${SERVING_ROOT_DIR}/test/test_images -mindepth 1 -maxdepth 1 -type d)"
+  for image_dir in ${image_dirs}; do
+    ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})"
+  done
+}
+
 function run_e2e_tests() {
   header "Running tests in $1"
   kubectl create namespace $2
   kubectl label namespace $2 istio-injection=enabled --overwrite
   local options=""
   (( EMIT_METRICS )) && options="-emitmetrics"
-  report_go_test -v -tags=e2e -count=1 ./test/$1 -dockerrepo gcr.io/knative-tests/test-images/$3 ${options}
+  report_go_test -v -tags=e2e -count=1 -timeout=20m ./test/$1 ${options}
 
   local result=$?
   [[ ${result} -ne 0 ]] && dump_stack_info
@@ -178,7 +185,7 @@ if (( ! RUN_TESTS )); then
   header "Creating test cluster"
   # Smallest cluster required to run the end-to-end-tests
   CLUSTER_CREATION_ARGS=(
-    --gke-create-args="--enable-autoscaling --min-nodes=1 --max-nodes=${E2E_CLUSTER_NODES} --scopes=cloud-platform"
+    --gke-create-args="--enable-autoscaling --min-nodes=1 --max-nodes=${E2E_CLUSTER_NODES} --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate"
     --gke-shape={\"default\":{\"Nodes\":${E2E_CLUSTER_NODES}\,\"MachineType\":\"${E2E_CLUSTER_MACHINE}\"}}
     --provider=gke
     --deployment=gke
@@ -192,6 +199,7 @@ if (( ! RUN_TESTS )); then
   fi
   # SSH keys are not used, but kubetest checks for their existence.
   # Touch them so if they don't exist, empty files are create to satisfy the check.
+  mkdir -p $HOME/.ssh
   touch $HOME/.ssh/google_compute_engine.pub
   touch $HOME/.ssh/google_compute_engine
   # Clear user and cluster variables, so they'll be set to the test cluster.
@@ -279,6 +287,7 @@ if (( USING_EXISTING_CLUSTER )); then
 fi
 
 create_everything
+publish_test_images
 
 # Handle failures ourselves, so we can dump useful info.
 set +o errexit
@@ -291,9 +300,9 @@ abort_if_failed
 
 # Run the tests
 
-run_e2e_tests conformance pizzaplanet conformance
+run_e2e_tests conformance pizzaplanet
 result=$?
-run_e2e_tests e2e noodleburg e2e
+run_e2e_tests e2e noodleburg
 [[ $? -ne 0 || ${result} -ne 0 ]] && exit 1
 
 # kubetest teardown might fail and thus incorrectly report failure of the

test/e2e/helloworld_shell_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 const (
-	appYaml              = "test_images/helloworld/helloworld.yaml"
+	appYaml              = "../test_images/helloworld/helloworld.yaml"
 	yamlImagePlaceholder = "github.com/knative/serving/test_images/helloworld"
 	ingressTimeout       = 5 * time.Minute
 	servingTimeout       = 2 * time.Minute

test/e2e_flags.go

@@ -45,7 +45,7 @@ func initializeFlags() *EnvironmentFlags {
 	flag.StringVar(&f.Cluster, "cluster", defaultCluster,
 		"Provide the cluster to test against. Defaults to $K8S_CLUSTER_OVERRIDE, then current cluster in kubeconfig if $K8S_CLUSTER_OVERRIDE is unset.")
 
-	defaultRepo := os.Getenv("DOCKER_REPO_OVERRIDE")
+	defaultRepo := path.Join(os.Getenv("DOCKER_REPO_OVERRIDE"), "github.com/knative/serving/test/test_images")
 	flag.StringVar(&f.DockerRepo, "dockerrepo", defaultRepo,
 		"Provide the uri of the docker repo you have uploaded the test image to using `uploadtestimage.sh`. Defaults to $DOCKER_REPO_OVERRIDE")
 

test/library.sh

@@ -113,10 +113,29 @@ function acquire_cluster_admin_role() {
   # might not have the necessary permission.
   local password=$(gcloud --format="value(masterAuth.password)" \
       container clusters describe $2 --zone=$3)
-  kubectl --username=admin --password=$password \
-      create clusterrolebinding cluster-admin-binding \
+  if [[ -n "${password}" ]]; then
+    # Cluster created with basic authentication
+    kubectl config set-credentials cluster-admin \
+        --username=admin --password=${password}
+  else
+    local cert=$(mktemp)
+    local key=$(mktemp)
+    echo "Certificate in ${cert}, key in ${key}"
+    gcloud --format="value(masterAuth.clientCertificate)" \
+      container clusters describe $2 --zone=$3 | base64 -d > ${cert}
+    gcloud --format="value(masterAuth.clientKey)" \
+      container clusters describe $2 --zone=$3 | base64 -d > ${key}
+    kubectl config set-credentials cluster-admin \
+      --client-certificate=${cert} --client-key=${key}
+  fi
+  kubectl config set-context $(kubectl config current-context) \
+      --user=cluster-admin
+  kubectl create clusterrolebinding cluster-admin-binding \
       --clusterrole=cluster-admin \
       --user=$1
+  # Reset back to the default account
+  gcloud container clusters get-credentials \
+      $2 --zone=$3 --project $(gcloud config get-value project)
 }
 
 # Runs a go test and generate a junit summary through bazel.

test/conformance/test_images/README.md → test/test_images/README.md

@@ -1,6 +1,6 @@
 # Test images node
 
-The subdirectories contain the test images used in the conformance tests.
+The subdirectories contain the test images used in the conformance and e2e tests.
 
 For details about building and adding new images, see the [section about test
 images](/test/README.md#test-images).

test/upload-test-images.sh

@@ -16,15 +16,11 @@
 
 set -o errexit
 
-: ${1:?"Pass the directory with the test images as argument"}
 : ${DOCKER_REPO_OVERRIDE:?"You must set 'DOCKER_REPO_OVERRIDE', see DEVELOPMENT.md"}
 
-DOCKER_FILES="$(ls -1 $1/*/Dockerfile)"
-: ${DOCKER_FILES:?"No subdirectories with Dockerfile files found in $1"}
+export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
+IMAGE_DIRS="$(find $(dirname $0)/test_images -mindepth 1 -maxdepth 1 -type d)"
 
-for docker_file in ${DOCKER_FILES}; do
-  image_dir="$(dirname ${docker_file})"
-  versioned_name="${DOCKER_REPO_OVERRIDE}/$(basename ${image_dir})"
-  docker build "${image_dir}" -f "${docker_file}" -t "${versioned_name}"
-  docker push "${versioned_name}"
+for image_dir in ${IMAGE_DIRS}; do
+  ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})"
 done