Add integration test to validate CRD mode

test/integration/suites/k8s-crd-mode/00-setup

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Create a temporary path that will be added to the PATH to avoid picking up
+# binaries from the environment that aren't a version match.
+mkdir -p ./bin
+
+docker build --target example-crd-agent -t example-crd-agent .
+
+KIND_PATH=./bin/kind
+KUBECTL_PATH=./bin/kubectl
+
+# Download kind at the expected version at the given path.
+download-kind "${KIND_PATH}"
+
+# Download kubectl at the expected version.
+download-kubectl "${KUBECTL_PATH}"
+
+# We must supply an absolute path to the configuration directory. Replace the
+# CONFDIR variable in the kind configuration with the conf directory of the 
+# running test.
+sed -i.bak "s#CONFDIR#${PWD}/conf#g" conf/kind-config.yaml
+rm conf/kind-config.yaml.bak
+
+# Start the kind cluster.
+start-kind-cluster "${KIND_PATH}" k8stest ./conf/kind-config.yaml
+
+# Load the given images in the cluster.
+container_images=("spire-server:latest-local" "spire-agent:latest-local" "k8s-workload-registrar:latest-local" "example-crd-agent:latest")
+load-images "${KIND_PATH}" k8stest "${container_images[@]}"
+
+# Set the kubectl context.
+set-kubectl-context "${KUBECTL_PATH}" kind-k8stest

test/integration/suites/k8s-crd-mode/01-apply-config

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+source init-kubectl
+
+wait-for-rollout() {
+    ns=$1
+    obj=$2
+    MAXROLLOUTCHECKS=12
+    ROLLOUTCHECKINTERVAL=15s
+    for ((i=0; i<${MAXROLLOUTCHECKS}; i++)); do
+        log-info "checking rollout status for ${ns} ${obj}..."
+        if ./bin/kubectl "-n${ns}" rollout status "$obj" --timeout="${ROLLOUTCHECKINTERVAL}"; then
+            return
+        fi
+        log-warn "describing ${ns} ${obj}..."
+        ./bin/kubectl "-n${ns}" describe "$obj" || true
+        log-warn "logs for ${ns} ${obj}..."
+        ./bin/kubectl "-n${ns}" logs --all-containers "$obj" || true
+    done
+    fail-now "Failed waiting for ${obj} to roll out."
+}
+
+./bin/kubectl apply -f ./conf/spiffeid.spiffe.io_spiffeids.yaml
+./bin/kubectl create namespace spire
+./bin/kubectl apply -k ./conf/server
+wait-for-rollout spire deployment/spire-server
+
+./bin/kubectl apply -k ./conf/agent
+wait-for-rollout spire daemonset/spire-agent
+
+# Apply this separately after all of the spire infrastructure has been rolled
+# out, otherwise the k8s-workload-registrar might miss its chance to create
+# an entry for it
+./bin/kubectl apply -f ./conf/workload.yaml
+wait-for-rollout spire deployment/example-workload

test/integration/suites/k8s-crd-mode/02-check-for-workload-svid

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+source init-kubectl
+
+MAXFETCHCHECKS=60
+FETCHCHECKINTERVAL=1
+for ((i=1; i<=${MAXFETCHCHECKS}; i++)); do
+    EXAMPLEPOD=$(./bin/kubectl -nspire get pod -l app=example-workload -o jsonpath="{.items[0].metadata.name}")
+    log-info "checking for workload SPIFFE ID ($i of $MAXFETCHCHECKS max)..."
+    if ./bin/kubectl -nspire exec -t "${EXAMPLEPOD}" -- \
+        /opt/spire/bin/spire-agent api fetch -write /tmp \
+            | grep "SPIFFE ID:"; then
+        DONE=1
+
+        data=$(./bin/kubectl -nspire exec -t "${EXAMPLEPOD}" -- \
+           openssl x509 -in /tmp/svid.0.pem -text -noout)
+
+        echo $data | grep -q "URI:spiffe://example.org/workload" || fail-now "unexpected SPIFFE ID: $data"
+        echo $data | grep -q "DNS:dns1, DNS:dns2," || fail-now "unexpected DNS: $data"
+
+        break
+    fi
+    sleep "${FETCHCHECKINTERVAL}" 
+done
+
+if [ "${DONE}" -eq 1 ]; then
+    log-info "SPIFFE ID found."
+else
+    fail-now "timed out waiting for workload to obtain credentials."
+fi

test/integration/suites/k8s-crd-mode/Dockerfile

@@ -0,0 +1,4 @@
+FROM spire-agent:latest-local AS example-crd-agent
+RUN apk add --update openssl && \
+    rm -rf /var/cache/apk/*
+CMD []

test/integration/suites/k8s-crd-mode/README.md

@@ -0,0 +1,11 @@
+# Kubernetes with CRD mode Suite
+
+## Description
+
+This suite sets up a Kubernetes cluster using [Kind](https://kind.sigs.k8s.io) and asserts the following:
+
+* SPIRE server attests SPIRE agents by verifying Kubernetes Projected Service
+  Account Tokens (i.e. `k8s_psat`) via the Token Review API.
+* Workloads are registered via the K8S Workload Registrar (crd mode) and are able to
+  obtain identities with expected DNS and SPIFFE ID without the need for manually maintained registration
+  entries.

test/integration/suites/k8s-crd-mode/conf/admctrl/admission-control.yaml

@@ -0,0 +1,8 @@
+apiVersion: apiserver.k8s.io/v1alpha1
+kind: AdmissionConfiguration
+plugins:
+- name: ValidatingAdmissionWebhook
+  configuration:
+    apiVersion: apiserver.config.k8s.io/v1alpha1
+    kind: WebhookAdmission
+    kubeConfigFile: /etc/kubernetes/pki/admctrl/kubeconfig.yaml

test/integration/suites/k8s-crd-mode/conf/admctrl/kubeconfig.yaml

@@ -0,0 +1,9 @@
+# KubeConfig with client credentials for the API Server to use to call the
+# K8S Workload Registrar service
+apiVersion: v1
+kind: Config
+users:
+- name: k8s-workload-registrar.spire.svc
+  user:
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ1VENDQVYrZ0F3SUJBZ0lJVVNIdmpGQTFxRHd3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF4TVoKU3poVElGZFBVa3RNVDBGRUlGSkZSMGxUVkZKQlVpQkRRVEFnRncweE9UQTFNVE14T1RFME1qTmFHQTg1T1RrNQpNVEl6TVRJek5UazFPVm93S0RFbU1DUUdBMVVFQXhNZFN6aFRJRmRQVWt0TVQwRkVJRkpGUjBsVFZGSkJVaUJEClRFbEZUbFF3V1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVM3SDIrMjJOcEFhTmVRdXQvZEYwdUYKMXk0VDVKTVdBczJOYm9NOXhZdlFKb1FXTVVNNERobWZQT1hVaE5STXdkb1JzTmhSdXZsYkROY2FEU29tNE1DYQpvM1V3Y3pBT0JnTlZIUThCQWY4RUJBTUNBNmd3RXdZRFZSMGxCQXd3Q2dZSUt3WUJCUVVIQXdJd0RBWURWUjBUCkFRSC9CQUl3QURBZEJnTlZIUTRFRmdRVW9EYlBiOUpWNXhqZlZVMnBhSzd2UUNsZ2d3SXdId1lEVlIwakJCZ3cKRm9BVW02eFNULzJCUzRYdmhVcXVzaDJCTEwwdlJNSXdDZ1lJS29aSXpqMEVBd0lEU0FBd1JRSWdHNzRQeWkyZQpONlBEcVRGRnY1UDFjNFhjVVdERzMwdzJIZEU4Wm8rMStVWUNJUURUL2xMa2dUUjUzV01INVRqWkllblhmYzFjCmxkMGlqSmpvRFJIR3lIRjJxdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    client-key-data: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JR0hBZ0VBTUJNR0J5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ1BhSWtTTVowUmduQllWYncKMDIrdlN5UUpDM2RtZ0VDNFBLN2svTnk4Qnh1aFJBTkNBQVM3SDIrMjJOcEFhTmVRdXQvZEYwdUYxeTRUNUpNVwpBczJOYm9NOXhZdlFKb1FXTVVNNERobWZQT1hVaE5STXdkb1JzTmhSdXZsYkROY2FEU29tNE1DYQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tCg==

test/integration/suites/k8s-crd-mode/conf/agent/kustomization.yaml

@@ -0,0 +1,10 @@
+# kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# list of Resource Config to be Applied
+resources:
+  - spire-agent.yaml
+
+# namespace to deploy all Resources to
+namespace: spire

test/integration/suites/k8s-crd-mode/conf/agent/spire-agent.yaml

@@ -0,0 +1,163 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: spire-agent
+  namespace: spire
+
+---
+
+# Required cluster role to allow spire-agent to query k8s API server
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: spire-agent-cluster-role
+rules:
+- apiGroups: [""]
+  resources: ["pods","nodes","nodes/proxy"]
+  verbs: ["get"]
+
+---
+
+# Binds above cluster role to spire-agent service account
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: spire-agent-cluster-role-binding
+subjects:
+- kind: ServiceAccount
+  name: spire-agent
+  namespace: spire
+roleRef:
+  kind: ClusterRole
+  name: spire-agent-cluster-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: spire-agent
+  namespace: spire
+data:
+  agent.conf: |
+    agent {
+      data_dir = "/run/spire"
+      log_level = "DEBUG"
+      server_address = "spire-server"
+      server_port = "8081"
+      socket_path = "/run/spire/sockets/api.sock"
+      trust_bundle_path = "/run/spire/bundle/bundle.crt"
+      trust_domain = "example.org"
+    }
+
+    plugins {
+      NodeAttestor "k8s_psat" {
+        plugin_data {
+          cluster = "example-cluster"
+        }
+      }
+
+      KeyManager "memory" {
+        plugin_data {
+        }
+      }
+
+      WorkloadAttestor "k8s" {
+        plugin_data {
+          # Defaults to the secure kubelet port by default.
+          # Minikube does not have a cert in the cluster CA bundle that
+          # can authenticate the kubelet cert, so skip validation.
+          skip_kubelet_verification = true
+        }
+      }
+    }
+
+    health_checks {
+       listener_enabled = true
+       bind_address = "0.0.0.0"
+       bind_port = "8089"
+       live_path = "/live"
+       ready_path = "/ready"
+    }
+
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: spire-agent
+  namespace: spire
+  labels:
+    app: spire-agent
+spec:
+  selector:
+    matchLabels:
+      app: spire-agent
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      namespace: spire
+      labels:
+        app: spire-agent
+    spec:
+      hostPID: true
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: spire-agent
+      initContainers:
+        - name: init
+          # This is a small image with wait-for-it, choose whatever image
+          # you prefer that waits for a service to be up. This image is built
+          # from https://github.com/lqhl/wait-for-it
+          image: gcr.io/spiffe-io/wait-for-it
+          args: ["-t", "30", "spire-server:8081"]
+      containers:
+        - name: spire-agent
+          image: spire-agent:latest-local
+          imagePullPolicy: Never
+          args: ["-config", "/run/spire/config/agent.conf"]
+          volumeMounts:
+            - name: spire-config
+              mountPath: /run/spire/config
+              readOnly: true
+            - name: spire-agent-socket
+              mountPath: /run/spire/sockets
+              readOnly: false
+            - name: spire-bundle
+              mountPath: /run/spire/bundle
+              readOnly: true
+            - name: spire-token
+              mountPath: /var/run/secrets/tokens
+          livenessProbe:
+            httpGet:
+              path: /live
+              port: 8089
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8089
+            initialDelaySeconds: 10
+            periodSeconds: 10
+      volumes:
+        - name: spire-config
+          configMap:
+            name: spire-agent
+        - name: spire-agent-socket
+          hostPath:
+            path: /run/spire/sockets
+            type: DirectoryOrCreate
+        - name: spire-bundle
+          configMap:
+            name: spire-bundle
+        - name: spire-token
+          projected:
+           sources:
+           - serviceAccountToken:
+              path: spire-agent
+              expirationSeconds: 7200
+              audience: spire-server

test/integration/suites/k8s-crd-mode/conf/kind-config.yaml

@@ -0,0 +1,20 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+kubeadmConfigPatches:
+- |
+  apiVersion: kubeadm.k8s.io/v1beta2
+  kind: ClusterConfiguration
+  metadata:
+    name: config
+  apiServer:
+    extraArgs:
+        "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
+        "service-account-issuer": "api"
+        "service-account-api-audiences": "api,spire-server"
+        "admission-control-config-file": "/etc/kubernetes/pki/admctrl/admission-control.yaml"
+nodes:
+- role: control-plane
+  image: kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
+  extraMounts:
+  - containerPath: /etc/kubernetes/pki/admctrl
+    hostPath: CONFDIR/admctrl

test/integration/suites/k8s-crd-mode/conf/server/kustomization.yaml

@@ -0,0 +1,10 @@
+# kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# list of Resource Config to be Applied
+resources:
+  - spire-server.yaml
+
+# namespace to deploy all Resources to
+namespace: spire