Run acceptance tests against kind (hashicorp#712)

* Add a new flag to the framework, -use-kind. The only thing it affects currently are the mesh gateway tests: it uses NodePort service instead of the LoadBalancer service to expose the mesh gateways. This is because load balancer services are not available on kind. For the NodePort services, we don't need to do anything to allow cross-cluster communication because kind uses the docker bridge network, and so the nodes are reachable by default.

* Change the main pipeline to use kind instead of gke, and move gke tests to the nightly pipeline.

* Use circleci's parallelism feature to run tests in parallel. Currently, tests are split by package, which is not the best efficient way to parallelize them, but we can address that at a later time. Currently, using 6 parallel nodes reduces the test runtime to about 30min.

.circleci/config.yml

@@ -92,6 +92,84 @@ jobs:
           path: /tmp/test-results
 
   acceptance:
+    environment:
+      - TEST_RESULTS: /tmp/test-results
+    machine:
+      image: ubuntu-2004:202010-01
+    parallelism: 6
+    steps:
+      - checkout
+      - run:
+          name: Install gotestsum, kind, kubectl, and helm
+          command: |
+            go get gotest.tools/gotestsum
+
+            curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64
+            chmod +x ./kind
+            sudo mv ./kind /usr/local/bin/kind
+
+            curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
+            chmod +x ./kubectl
+            sudo mv ./kubectl /usr/local/bin/kubectl
+
+            curl https://baltocdn.com/helm/signing.asc | sudo apt-key add -
+            sudo apt-get install apt-transport-https --yes
+            echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
+            sudo apt-get update
+            sudo apt-get install helm
+      - run:
+          name: Create kind clusters
+          command: |
+            kind create cluster --name dc1 --image kindest/node:v1.18.4
+            kind create cluster --name dc2 --image kindest/node:v1.18.4
+      - restore_cache:
+          keys:
+            - consul-helm-modcache-v2-{{ checksum "test/acceptance/go.mod" }}
+      - run:
+          name: go mod download
+          working_directory: test/acceptance
+          command: go mod download
+      - save_cache:
+          key: consul-helm-modcache-v2-{{ checksum "test/acceptance/go.mod" }}
+          paths:
+            - ~/.go_workspace/pkg/mod
+      - run: mkdir -p $TEST_RESULTS
+      - run:
+          name: Run acceptance tests
+          working_directory: test/acceptance/tests
+          no_output_timeout: 1h
+          command: |
+            # We have to run the tests for each package separately so that we can
+            # exit early if any test fails (-failfast only works within a single
+            # package).
+            exit_code=0
+            pkgs=$(go list ./... | circleci tests split)
+            echo "Running $pkgs"
+            for pkg in $pkgs
+            do
+              if ! gotestsum --no-summary=all --jsonfile=jsonfile-${pkg////-} -- $pkg -p 1 -timeout 30m -failfast \
+                    -use-kind \
+                    -enable-multi-cluster \
+                    -enable-enterprise \
+                    -kubecontext="kind-dc1" \
+                    -secondary-kubecontext="kind-dc2" \
+                    -debug-directory="$TEST_RESULTS/debug" \
+                    -consul-k8s-image=hashicorpdev/consul-k8s:latest
+              then
+                echo "Tests in ${pkg} failed, aborting early"
+                exit_code=1
+                break
+              fi
+            done
+            gotestsum --raw-command --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- cat jsonfile*
+            exit $exit_code
+
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+
+  acceptance-gke-1-16:
     environment:
       - TEST_RESULTS: /tmp/test-results
     docker:
@@ -441,8 +519,9 @@ workflows:
                 - master
     jobs:
       - acceptance-openshift
-      - acceptance-aks-1-19
+      - acceptance-gke-1-16
       - acceptance-eks-1-17
+      - acceptance-aks-1-19
   update-helm-charts-index:
     jobs:
       - update-helm-charts-index:

test/acceptance/framework/config/config.go

@@ -35,6 +35,8 @@ type TestConfig struct {
 	NoCleanupOnFailure bool
 	DebugDirectory     string
 
+	UseKind bool
+
 	helmChartPath string
 }
 

test/acceptance/framework/flags/flags.go

@@ -31,6 +31,8 @@ type TestFlags struct {
 
 	flagDebugDirectory string
 
+	flagUseKind bool
+
 	once sync.Once
 }
 
@@ -77,6 +79,9 @@ func (t *TestFlags) init() {
 
 	flag.StringVar(&t.flagDebugDirectory, "debug-directory", "", "The directory where to write debug information about failed test runs, "+
 		"such as logs and pod definitions. If not provided, a temporary directory will be created by the tests.")
+
+	flag.BoolVar(&t.flagUseKind, "use-kind", false,
+		"If true, the tests will assume they are running against a local kind cluster(s).")
 }
 
 func (t *TestFlags) Validate() error {
@@ -119,5 +124,6 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig {
 
 		NoCleanupOnFailure: t.flagNoCleanupOnFailure,
 		DebugDirectory:     tempDir,
+		UseKind:            t.flagUseKind,
 	}
 }

test/acceptance/tests/mesh-gateway/mesh_gateway_test.go

@@ -41,6 +41,11 @@ func TestMeshGatewayDefault(t *testing.T) {
 		"meshGateway.replicas": "1",
 	}
 
+	if cfg.UseKind {
+		primaryHelmValues["meshGateway.service.type"] = "NodePort"
+		primaryHelmValues["meshGateway.service.nodePort"] = "30000"
+	}
+
 	releaseName := helpers.RandomName()
 
 	// Install the primary consul cluster in the default kubernetes context
@@ -86,6 +91,11 @@ func TestMeshGatewayDefault(t *testing.T) {
 		"meshGateway.replicas": "1",
 	}
 
+	if cfg.UseKind {
+		secondaryHelmValues["meshGateway.service.type"] = "NodePort"
+		secondaryHelmValues["meshGateway.service.nodePort"] = "30000"
+	}
+
 	// Install the secondary consul cluster in the secondary kubernetes context
 	secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName)
 	secondaryConsulCluster.Create(t)
@@ -150,6 +160,11 @@ func TestMeshGatewaySecure(t *testing.T) {
 				"meshGateway.replicas": "1",
 			}
 
+			if cfg.UseKind {
+				primaryHelmValues["meshGateway.service.type"] = "NodePort"
+				primaryHelmValues["meshGateway.service.nodePort"] = "30000"
+			}
+
 			releaseName := helpers.RandomName()
 
 			// Install the primary consul cluster in the default kubernetes context
@@ -200,6 +215,11 @@ func TestMeshGatewaySecure(t *testing.T) {
 				"meshGateway.replicas": "1",
 			}
 
+			if cfg.UseKind {
+				secondaryHelmValues["meshGateway.service.type"] = "NodePort"
+				secondaryHelmValues["meshGateway.service.nodePort"] = "30000"
+			}
+
 			// Install the secondary consul cluster in the secondary kubernetes context
 			secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName)
 			secondaryConsulCluster.Create(t)