Add conformance testing workflow for GitHub Actions

.github/workflows/conformance.yml

@@ -0,0 +1,111 @@
+name: conformance
+
+on:
+  push:
+     branches:
+      - "*-conformance"
+      - "conformance-*"
+
+  schedule:
+    - cron:  '0 0 * * *'
+
+  workflow_dispatch:
+
+
+env:
+  GO_VERSION: "1.17"
+
+jobs:
+  run-on-kind:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Goenv
+        uses: ./.github/actions/goenv
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Create Kind cluster
+        uses: helm/kind-action@2a525709fd0874b75d7ae842d257981b0e0f557d
+        with:
+          cluster_name: "consul-api-gateway-test"
+          kubectl_version: "v1.21.0"
+
+      - name: Install MetalLB
+        run: |
+          kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
+          kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/metallb.yaml
+          kubectl apply -f ./internal/testing/conformance/metallb-config.yaml
+          kubectl wait --for=condition=Ready --timeout=60s --namespace=metallb-system pods --all
+
+      - name: Build binary
+        env:
+          CGO_ENABLED: "0"
+          GOARCH: "amd64"
+          GOOS: "linux"
+        run: go build -o ./consul-api-gateway
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: "linux/amd64"
+          file: "Dockerfile.local"
+          load: true
+          push: false
+          tags: "consul-api-gateway:test"
+
+      - name: Load Docker image into Kind
+        run: kind load docker-image consul-api-gateway:test --name consul-api-gateway-test
+
+      - name: Install Consul API Gateway CRDs
+        run: kubectl apply --kustomize="./config/crd"
+
+      - name: Clone consul-k8s
+        uses: actions/checkout@v2
+        with:
+          repository: "hashicorp/consul-k8s"
+          path: "./internal/testing/conformance/consul-k8s"
+          fetch-depth: "1"
+
+      - name: Clone gateway-api
+        uses: actions/checkout@v2
+        with:
+          repository: "nathancoleman/gateway-api"
+          ref: "eventually-consistent-conformance"
+          path: "./internal/testing/conformance/gateway-api"
+          fetch-depth: "1"
+
+      - name: Install Consul
+        working-directory: "./internal/testing/conformance"
+        run: |
+          helm install --values ./consul-config.yaml consul ./consul-k8s/charts/consul --create-namespace --namespace=consul
+          kubectl wait --for=condition=Ready --timeout=60s --namespace=consul pods --all
+
+      - name: Patch testing resources
+        working-directory: "./internal/testing/conformance"
+        run: |
+          cp kustomization.yaml proxydefaults.yaml ./gateway-api/conformance/
+          cd ./gateway-api/conformance/
+          kubectl kustomize ./ --output ./base/manifests.yaml
+
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      #   timeout-minutes: 10
+
+      - name: Run tests
+        working-directory: "./internal/testing/conformance/gateway-api/conformance"
+        run: go test -v -timeout 10m ./ --gateway-class consul-api-gateway
+      # - name: Report Status
+      #   if: always()
+      #   uses: ravsamhq/notify-slack-action@v1
+      #   with:
+      #     status: ${{ job.status }}
+      #     notify_when: 'failure'
+      #   env:
+      #     SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

.gitignore

@@ -7,4 +7,8 @@ refresh.yml
 cover.out
 demo-deployment
 bin
-pkg/bin
+pkg/bin
+
+# Repos cloned for conformance testing
+internal/testing/conformance/consul-k8s/
+internal/testing/conformance/gateway-api

Dockerfile.local

@@ -1,5 +1,5 @@
-FROM alpine:3.13
+FROM alpine:latest
 
 COPY ./consul-api-gateway /bin/consul-api-gateway
 ENTRYPOINT ["/bin/consul-api-gateway"]
-CMD ["version"]
+CMD ["version"]

internal/testing/conformance/README.md

@@ -15,57 +15,10 @@ need for these patches.
     each `Deployment`'s template. They all use the same port.
 - The Consul services default to a protocol of `tcp`; however, the testing framework uses `http`. To make this work, we create
     a `ProxyDefaults` resource which sets the protocol to `http` globally.
+- GitHub Actions' default hosted runner is not powerful enough to run all pods specified upstream in kind.
+    To cope with this, we reduce all `Deployments` to 1 replica.
 
 ## Status
 
-The conformance tests cannot currently run in an automated fashion. They are not included in our CI yet.
-
-Due to the controller not currently knowing when Consul/Envoy are ready after syncing in new routes, the route appears
-"ready" to the conformance testing framework before the gateway can actually respond to requests for the route. The
-framework then sends HTTP requests as soon as the route appears ready and the request is rejected with an error like
-the following:
-
-```log
-Get "http://35.229.22.36": dial tcp 35.229.22.36:80: connect: connection refused
-```
-
-This doesn't mean we cannot run the conformance tests, they just have to be run manually, one at a time.  
-To run a particular conformance test, you need to:
-
-1. Create a GKE cluster (or any other standard Kubernetes cluster) and install Consul + Consul API Gateway.
-     The [usage docs](https://www.consul.io/docs/api-gateway/api-gateway-usage#installation) explain how to do this.
-
-2. clone the [kubernetes-sigs/gateway-api](https://github.com/kubernetes-sigs/gateway-api)
-repo and copy our patches into the `conformance` subdirectory:
-
-    ```shell
-    git clone --depth 1 git@github.com:kubernetes-sigs/gateway-api
-    cp kustomization.yaml proxydefaults.yaml gateway-api/conformance/
-    ```
-
-3. make your way into the `conformance` directory, then patch and install the base resources:
-
-    ```shell
-    cd gateway-api/conformance/
-    kubectl kustomize ./ --output ./base/manifests.yaml
-    kubectl apply -f ./base/manifests.yaml --validate=false
-    ```
-
-4. install the test-specific resources (adjust name appropriately):
-
-    ```shell
-    kubectl apply -f tests/httproute-matching.yaml
-    ```
-
-5. modify the last line of `conformance_test.go` that passes the list of tests to include only the test that you want to run:
-
-    ```go
-    cSuite.Run(t, []suite.ConformanceTest{tests.HTTPRouteMatchingAcrossRoutes})
-    ```
-
-6. run the test:
-    ```shell
-    go test ./ --gateway-class consul-api-gateway --cleanup=0
-    ```
-
-7. repeat steps 4-6 for other tests
+The conformance tests are run nightly in GitHub Actions using the workflow [here](/.github/workflows/conformance.yml).
+You may also run the workflow on demand from this repo's Actions tab or by following the branch naming conventions listed in the workflow.

internal/testing/conformance/consul-config.yaml

@@ -0,0 +1,20 @@
+global:
+  name: consul
+  datacenter: dc1
+  tls:
+    enabled: true
+server:
+  replicas: 1
+ui:
+  enabled: true
+  service:
+    type: NodePort
+connectInject:
+  enabled: true
+  default: true
+controller:
+  enabled: true
+apiGateway:
+  enabled: true
+  logLevel: info
+  image: "consul-api-gateway:test"

internal/testing/conformance/kustomization.yaml

@@ -29,3 +29,10 @@ patches:
         value: {'consul.hashicorp.com/connect-inject': 'true', 'consul.hashicorp.com/connect-service-port': '3000'}
     target:
       kind: Deployment
+  # We don't have enough resources in the GitHub-hosted Actions runner to support 2 replicas
+  - patch: |-
+      - op: replace
+        path: "/spec/replicas"
+        value: 1
+    target:
+      kind: Deployment

internal/testing/conformance/metallb-config.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - 172.18.255.200-172.18.255.250