diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..bb9a8d139 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,120 @@ +name: conformance + +on: + pull_request: + types: ["opened", "reopened", "synchronize", "labeled"] + + push: + branches: ["conformance/*"] + + schedule: + - cron: '0 0 * * *' + + workflow_dispatch: + + +env: + GO_VERSION: "1.17" + +jobs: + run-on-kind: + # Run on PR only if there is a `pr/run-conformance` label + if: "github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr/run-conformance')" + runs-on: ubuntu-latest + steps: + # Clone repos side-by-side: + # GITHUB_WORKSPACE/ + # consul-api-gateway/ + # consul-k8s/ + # gateway-api/ + - name: Checkout consul-api-gateway + uses: actions/checkout@v2 + with: + path: "consul-api-gateway" + + - name: Clone consul-k8s + uses: actions/checkout@v2 + with: + repository: "hashicorp/consul-k8s" + path: "consul-k8s" + + - name: Clone gateway-api + uses: actions/checkout@v2 + with: + repository: "nathancoleman/gateway-api" + ref: "eventually-consistent-conformance" + path: "gateway-api" + + - name: Setup Goenv + uses: ./consul-api-gateway/.github/actions/goenv + with: + go-version: ${{ env.GO_VERSION }} + + - name: Create Kind cluster + uses: helm/kind-action@2a525709fd0874b75d7ae842d257981b0e0f557d + with: + cluster_name: "consul-api-gateway-test" + kubectl_version: "v1.21.0" + + - name: Install MetalLB + run: | + kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml + kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/metallb.yaml + kubectl apply -f ./consul-api-gateway/internal/testing/conformance/metallb-config.yaml + kubectl wait --for=condition=Ready --timeout=60s --namespace=metallb-system pods --all + + - name: Build binary + env: + CGO_ENABLED: "0" + GOARCH: "amd64" + GOOS: "linux" + working-directory: "consul-api-gateway" + run: go build -o ./consul-api-gateway + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + # docker build -f Dockerfile.local -t consul-api-gateway:test --platform linux/amd64 . + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: "./consul-api-gateway" + platforms: "linux/amd64" + file: "./consul-api-gateway/Dockerfile.local" + load: true + push: false + tags: "consul-api-gateway:test" + + - name: Load Docker image into Kind + run: kind load docker-image consul-api-gateway:test --name consul-api-gateway-test + + - name: Install Consul API Gateway CRDs + working-directory: "consul-api-gateway" + run: kubectl apply --kustomize="./config/crd" + + - name: Install Consul + working-directory: "consul-api-gateway/internal/testing/conformance" + run: | + helm install --values ./consul-config.yaml consul $GITHUB_WORKSPACE/consul-k8s/charts/consul --create-namespace --namespace=consul + kubectl wait --for=condition=Ready --timeout=60s --namespace=consul pods --all + + - name: Patch testing resources + working-directory: "consul-api-gateway/internal/testing/conformance" + run: | + cp kustomization.yaml proxydefaults.yaml $GITHUB_WORKSPACE/gateway-api/conformance/ + cd $GITHUB_WORKSPACE/gateway-api/conformance/ + kubectl kustomize ./ --output ./base/manifests.yaml + + - name: Run tests + working-directory: "gateway-api/conformance" + run: go test -v -timeout 10m ./ --gateway-class consul-api-gateway + + # TODO Waiting for webhook approval + # - name: Report Status + # if: always() + # uses: ravsamhq/notify-slack-action@v1 + # with: + # status: ${{ job.status }} + # notify_when: 'failure' + # env: + # SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} diff --git a/.gitignore b/.gitignore index 52ca2e3fa..28a2643f3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ refresh.yml cover.out demo-deployment bin -pkg/bin \ No newline at end of file +pkg/bin diff --git a/Dockerfile.local b/Dockerfile.local index aa2e72789..65aa25daf 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,5 +1,5 @@ -FROM alpine:3.13 +FROM alpine:latest COPY ./consul-api-gateway /bin/consul-api-gateway ENTRYPOINT ["/bin/consul-api-gateway"] -CMD ["version"] \ No newline at end of file +CMD ["version"] diff --git a/internal/testing/conformance/README.md b/internal/testing/conformance/README.md index 685025dfe..8ab8abca6 100644 --- a/internal/testing/conformance/README.md +++ b/internal/testing/conformance/README.md @@ -8,64 +8,19 @@ The framework defines its own set of Kubernetes resources using kustomization ya however, we currently have to make a few patches in order for things to work with Consul. Our goal long-term is to remove the need for these patches. -- Consul isn't, by default, aware of each of the created services. To make this work, we patch the `connect-inject` annotation onto - each `Deployment`'s template. - The `Deployments` defined upstream do not specify a `containerPort` in the `Pod` template. Consul relies on this `containerPort` when a `connect-service-port` annotation is not present. To make this work, we patch the `connect-service-port` annotation onto each `Deployment`'s template. They all use the same port. - The Consul services default to a protocol of `tcp`; however, the testing framework uses `http`. To make this work, we create a `ProxyDefaults` resource which sets the protocol to `http` globally. +- GitHub Actions' default hosted runner is not powerful enough to run all pods specified upstream in kind. + To cope with this, we reduce all `Deployments` to 1 replica. ## Status -The conformance tests cannot currently run in an automated fashion. They are not included in our CI yet. +The conformance tests are run nightly in GitHub Actions using the workflow [here](/.github/workflows/conformance.yml). +You may also run the workflow on demand from this repo's Actions tab, by following the `conformance/*` branch naming convention, or by adding the `pr/run-conformance` label to your pull request. -Due to the controller not currently knowing when Consul/Envoy are ready after syncing in new routes, the route appears -"ready" to the conformance testing framework before the gateway can actually respond to requests for the route. The -framework then sends HTTP requests as soon as the route appears ready and the request is rejected with an error like -the following: +## Tips -```log -Get "http://35.229.22.36": dial tcp 35.229.22.36:80: connect: connection refused -``` - -This doesn't mean we cannot run the conformance tests, they just have to be run manually, one at a time. -To run a particular conformance test, you need to: - -1. Create a GKE cluster (or any other standard Kubernetes cluster) and install Consul + Consul API Gateway. - The [usage docs](https://www.consul.io/docs/api-gateway/api-gateway-usage#installation) explain how to do this. - -2. clone the [kubernetes-sigs/gateway-api](https://github.com/kubernetes-sigs/gateway-api) -repo and copy our patches into the `conformance` subdirectory: - - ```shell - git clone --depth 1 git@github.com:kubernetes-sigs/gateway-api - cp kustomization.yaml proxydefaults.yaml gateway-api/conformance/ - ``` - -3. make your way into the `conformance` directory, then patch and install the base resources: - - ```shell - cd gateway-api/conformance/ - kubectl kustomize ./ --output ./base/manifests.yaml - kubectl apply -f ./base/manifests.yaml --validate=false - ``` - -4. install the test-specific resources (adjust name appropriately): - - ```shell - kubectl apply -f tests/httproute-matching.yaml - ``` - -5. modify the last line of `conformance_test.go` that passes the list of tests to include only the test that you want to run: - - ```go - cSuite.Run(t, []suite.ConformanceTest{tests.HTTPRouteMatchingAcrossRoutes}) - ``` - -6. run the test: - ```shell - go test ./ --gateway-class consul-api-gateway --cleanup=0 - ``` - -7. repeat steps 4-6 for other tests +- If you need shell access to debug any steps in the workflow, [mxschmitt/action-tmate](https://github.com/mxschmitt/action-tmate) can be added before/after any step. diff --git a/internal/testing/conformance/consul-config.yaml b/internal/testing/conformance/consul-config.yaml new file mode 100644 index 000000000..7a7903547 --- /dev/null +++ b/internal/testing/conformance/consul-config.yaml @@ -0,0 +1,14 @@ +global: + tls: + enabled: true +server: + replicas: 1 +connectInject: + enabled: true + default: true +controller: + enabled: true +apiGateway: + enabled: true + logLevel: info + image: "consul-api-gateway:test" diff --git a/internal/testing/conformance/kustomization.yaml b/internal/testing/conformance/kustomization.yaml index 17f29f6c1..c1cc5403f 100644 --- a/internal/testing/conformance/kustomization.yaml +++ b/internal/testing/conformance/kustomization.yaml @@ -19,13 +19,20 @@ patches: value: "consul-api-gateway" target: kind: Gateway - # Add connect-inject annotations to each Deployment. This is required due to + # Add connect-inject annotation to each Deployment. This is required due to # containerPort not being defined on Deployments upstream. Though containerPort # is optional, Consul relies on it as a default value in the absence of a # connect-service-port annotation. - patch: |- - op: add path: "/spec/template/metadata/annotations" - value: {'consul.hashicorp.com/connect-inject': 'true', 'consul.hashicorp.com/connect-service-port': '3000'} + value: {'consul.hashicorp.com/connect-service-port': '3000'} + target: + kind: Deployment + # We don't have enough resources in the GitHub-hosted Actions runner to support 2 replicas + - patch: |- + - op: replace + path: "/spec/replicas" + value: 1 target: kind: Deployment diff --git a/internal/testing/conformance/metallb-config.yaml b/internal/testing/conformance/metallb-config.yaml new file mode 100644 index 000000000..3ec87b63c --- /dev/null +++ b/internal/testing/conformance/metallb-config.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - 172.18.255.200-172.18.255.250