Add support to create authmethod that can create global tokens for

secondary datacenters

.circleci/config.yml

@@ -9,8 +9,8 @@ executors:
       - image: docker.mirror.hashicorp.services/cimg/go:1.17.5
     environment:
       TEST_RESULTS: /tmp/test-results # path to where test results are saved
-      CONSUL_VERSION: 1.11.2 # Consul's OSS version to use in tests
-      CONSUL_ENT_VERSION: 1.11.2+ent # Consul's enterprise version to use in tests
+      CONSUL_VERSION: 1.11.4 # Consul's OSS version to use in tests
+      CONSUL_ENT_VERSION: 1.11.4+ent # Consul's enterprise version to use in tests
 
 control-plane-path : &control-plane-path control-plane
 cli-path : &cli-path cli
@@ -71,7 +71,7 @@ commands:
       consul-k8s-image:
         type: string
         #default: "docker.mirror.hashicorp.services/hashicorpdev/consul-k8s-control-plane:latest"
-        default: "kyleschochenmaier/consul-k8s-acls"
+        default: "thisisnotashwin/consul-k8s@sha256:4712cc990e1a3dfa7b3906ebd3bf2e8cae346f928403265d29775a055d862dae"
       go-path:
         type: string
         default: "/home/circleci/.go_workspace"

acceptance/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/hashicorp/consul-k8s/control-plane v0.0.0-20211207212234-aea9efea5638
 	github.com/hashicorp/consul/api v1.12.0
 	github.com/hashicorp/consul/sdk v0.9.0
+	github.com/hashicorp/go-uuid v1.0.2
 	github.com/hashicorp/vault/api v1.2.0
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -49,7 +50,6 @@ require (
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.1 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/go-version v1.2.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect

acceptance/tests/mesh-gateway/mesh_gateway_test.go

@@ -33,8 +33,9 @@ func TestMeshGatewayDefault(t *testing.T) {
 		"global.federation.enabled":                "true",
 		"global.federation.createFederationSecret": "true",
 
-		"connectInject.enabled": "true",
-		"controller.enabled":    "true",
+		"connectInject.enabled":  "true",
+		"connectInject.replicas": "1",
+		"controller.enabled":     "true",
 
 		"meshGateway.enabled":  "true",
 		"meshGateway.replicas": "1",
@@ -79,7 +80,9 @@ func TestMeshGatewayDefault(t *testing.T) {
 		"server.extraVolumes[0].items[0].key":  "serverConfigJSON",
 		"server.extraVolumes[0].items[0].path": "config.json",
 
-		"connectInject.enabled": "true",
+		"connectInject.enabled":  "true",
+		"connectInject.replicas": "1",
+		"controller.enabled":     "true",
 
 		"meshGateway.enabled":  "true",
 		"meshGateway.replicas": "1",
@@ -112,9 +115,9 @@ func TestMeshGatewayDefault(t *testing.T) {
 	// gateways.
 	logger.Log(t, "creating proxy-defaults config")
 	kustomizeDir := "../fixtures/bases/mesh-gateway"
-	k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir)
+	k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), kustomizeDir)
 	helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() {
-		k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir)
+		k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), kustomizeDir)
 	})
 
 	// Check that we can connect services over the mesh gateways
@@ -164,8 +167,9 @@ func TestMeshGatewaySecure(t *testing.T) {
 				"global.federation.enabled":                "true",
 				"global.federation.createFederationSecret": "true",
 
-				"connectInject.enabled": "true",
-				"controller.enabled":    "true",
+				"connectInject.enabled":  "true",
+				"connectInject.replicas": "1",
+				"controller.enabled":     "true",
 
 				"meshGateway.enabled":  "true",
 				"meshGateway.replicas": "1",
@@ -191,6 +195,19 @@ func TestMeshGatewaySecure(t *testing.T) {
 			_, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(context.Background(), federationSecret, metav1.CreateOptions{})
 			require.NoError(t, err)
 
+			var k8sAuthMethodHost string
+			// When running on kind, the kube API address in kubeconfig will have a localhost address
+			// which will not work from inside the container. That's why we need to use the endpoints address instead
+			// which will point the node IP.
+			if cfg.UseKind {
+				// The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service.
+				kubernetesEndpoint, err := secondaryContext.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{})
+				require.NoError(t, err)
+				k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port)
+			} else {
+				k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryContext.KubectlOptions(t))
+			}
+
 			// Create secondary cluster
 			secondaryHelmValues := map[string]string{
 				"global.datacenter": "dc2",
@@ -207,15 +224,19 @@ func TestMeshGatewaySecure(t *testing.T) {
 				"global.acls.replicationToken.secretName": federationSecretName,
 				"global.acls.replicationToken.secretKey":  "replicationToken",
 
-				"global.federation.enabled": "true",
+				"global.federation.enabled":           "true",
+				"global.federation.k8sAuthMethodHost": k8sAuthMethodHost,
+				"global.federation.primaryDatacenter": "dc1",
 
 				"server.extraVolumes[0].type":          "secret",
 				"server.extraVolumes[0].name":          federationSecretName,
 				"server.extraVolumes[0].load":          "true",
 				"server.extraVolumes[0].items[0].key":  "serverConfigJSON",
 				"server.extraVolumes[0].items[0].path": "config.json",
 
-				"connectInject.enabled": "true",
+				"connectInject.enabled":  "true",
+				"connectInject.replicas": "1",
+				"controller.enabled":     "true",
 
 				"meshGateway.enabled":  "true",
 				"meshGateway.replicas": "1",
@@ -248,9 +269,9 @@ func TestMeshGatewaySecure(t *testing.T) {
 			// gateways.
 			logger.Log(t, "creating proxy-defaults config")
 			kustomizeDir := "../fixtures/bases/mesh-gateway"
-			k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir)
+			k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), kustomizeDir)
 			helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() {
-				k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir)
+				k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), kustomizeDir)
 			})
 
 			// Check that we can connect services over the mesh gateways

charts/consul/templates/controller-deployment.yaml

@@ -87,7 +87,12 @@ spec:
           - |
             consul-k8s-control-plane acl-init \
               -component-name=controller \
+              {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
+              -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \
+              -primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \
+              {{- else }}
               -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \
+              {{- end }}
               {{- if .Values.global.adminPartitions.enabled }}
               -partition={{ .Values.global.adminPartitions.name }} \
               {{- end }}
@@ -139,10 +144,7 @@ spec:
                 - "/bin/sh"
                 - "-ec"
                 - |
-                  consul-k8s-control-plane consul-logout \
-                  {{- if .Values.global.adminPartitions.enabled }}
-                  -partition={{ .Values.global.adminPartitions.name }} \
-                  {{- end }}
+                  consul-k8s-control-plane consul-logout
         {{- end }}
         env:
         {{- if .Values.global.acls.manageSystemACLs }}

charts/consul/templates/server-acl-init-job.yaml

@@ -174,10 +174,14 @@ spec:
                 {{- if (or (and (ne (.Values.connectInject.enabled | toString) "-") .Values.connectInject.enabled) (and (eq (.Values.connectInject.enabled | toString) "-") .Values.global.enabled)) }}
                 -create-inject-token=true \
                 {{- if and .Values.externalServers.enabled .Values.externalServers.k8sAuthMethodHost }}
-                -inject-auth-method-host={{ .Values.externalServers.k8sAuthMethodHost }} \
+                -auth-method-host={{ .Values.externalServers.k8sAuthMethodHost }} \
                 {{- end }}
                 {{- end }}
 
+                {{- if .Values.global.federation.k8sAuthMethodHost }}
+                -auth-method-host={{ .Values.global.federation.k8sAuthMethodHost }} \
+                {{- end }}
+
                 {{- if .Values.meshGateway.enabled }}
                 -create-mesh-gateway-token=true \
                 {{- end }}

charts/consul/test/unit/controller-deployment.bats

@@ -57,20 +57,6 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
-@test "controller/Deployment: consul-logout preStop hook has partition when partitions are enabled" {
-  cd `chart_dir`
-  local actual=$(helm template \
-      -s templates/controller-deployment.yaml \
-      --set 'controller.enabled=true' \
-      --set 'global.acls.manageSystemACLs=true' \
-      --set 'global.enableConsulNamespaces=true' \
-      --set 'global.adminPartitions.enabled=true' \
-      --set 'global.adminPartitions.name=default' \
-      . | tee /dev/stderr |
-      yq '[.spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]] | any(contains("-partition=default"))' | tee /dev/stderr)
-  [ "${actual}" = "true" ]
-}
-
 @test "controller/Deployment: CONSUL_HTTP_TOKEN_FILE is not set when acls are disabled" {
   cd `chart_dir`
   local actual=$(helm template \
@@ -243,6 +229,36 @@ load _helpers
   [ "${actual}" = "get-auto-encrypt-client-ca" ]
 }
 
+@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command when federation enabled in non-primary partition" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/controller-deployment.yaml \
+      --set 'controller.enabled=true' \
+      --set 'global.datacenter=dc2' \
+      --set 'global.federation.enabled=true' \
+      --set 'global.federation.primaryDatacenter=dc1' \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.tls.enabled=true' \
+      --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'global.acls.manageSystemACLs=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.initContainers[] | select(.name == "controller-acl-init")' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $object |
+      yq -r '.command | any(contains("-acl-auth-method=RELEASE-NAME-consul-k8s-component-auth-method-dc2"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $object |
+      yq -r '.command | any(contains("-primary-datacenter=dc1"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+}
+
 #--------------------------------------------------------------------
 # global.tls.enabled
 

charts/consul/test/unit/server-acl-init-job.bats

@@ -1613,7 +1613,7 @@ load _helpers
       --set 'global.acls.manageSystemACLs=true' \
       --set 'connectInject.enabled=true' \
       . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[0].command | any(contains("-inject-auth-method-host"))' | tee /dev/stderr)
+      yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host"))' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
 
@@ -1625,11 +1625,11 @@ load _helpers
       --set 'externalServers.k8sAuthMethodHost=foo.com' \
       --set 'connectInject.enabled=true' \
       . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[0].command | any(contains("-inject-auth-method-host"))' | tee /dev/stderr)
+      yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host"))' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
 
-@test "serverACLInit/Job: can provide custom auth method host" {
+@test "serverACLInit/Job: can provide custom auth method host for external servers" {
   cd `chart_dir`
   local actual=$(helm template \
       -s templates/server-acl-init-job.yaml  \
@@ -1640,7 +1640,24 @@ load _helpers
       --set 'externalServers.hosts[0]=foo.com' \
       --set 'externalServers.k8sAuthMethodHost=foo.com' \
       . | tee /dev/stderr|
-      yq '.spec.template.spec.containers[0].command | any(contains("-inject-auth-method-host=foo.com"))' | tee /dev/stderr)
+      yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host=foo.com"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "serverACLInit/Job: can provide custom auth method host for federation" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/server-acl-init-job.yaml  \
+      --set 'global.acls.manageSystemACLs=true' \
+      --set 'global.tls.enabled=true' \
+      --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'connectInject.enabled=true' \
+      --set 'global.federation.enabled=true' \
+      --set 'global.federation.primaryDatacenter=dc1' \
+      --set 'global.federation.k8sAuthMethodHost=foo.com' \
+      --set 'meshGateway.enabled=true' \
+      . | tee /dev/stderr|
+      yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host=foo.com"))' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 }
 

charts/consul/values.yaml

@@ -85,7 +85,7 @@ global:
   # image: "hashicorp/consul-enterprise:1.10.0-ent"
   # ```
   # @default: hashicorp/consul:<latest version>
-  image: "hashicorp/consul:1.11.3"
+  image: "hashicorp/consul:1.11.4"
 
   # Array of objects containing image pull secret names that will be applied to each service account.
   # This can be used to reference image pull secrets if using a custom consul or consul-k8s-control-plane Docker image.
@@ -431,13 +431,29 @@ global:
     createFederationSecret: false
 
     # The name of the primary datacenter.
-    primaryDatacenter: ""
+    # @type: string
+    primaryDatacenter: null
 
     # A list of addresses of the primary mesh gateways in the form <ip>:<port>.
-    # (e.g. ["1.1.1.1:443", "2.3.4.5:443"] 
+    # (e.g. ["1.1.1.1:443", "2.3.4.5:443"]
     # @type: array<string>
     primaryGateways: []
 
+    # If you are setting `global.federation.enabled` to true and are in a secondary datacenter,
+    # set `k8sAuthMethodHost` to the address of the Kubernetes API server of the secondary datacenter.
+    # This address must be reachable from the Consul servers in the primary datacenter.
+    # Please see the Kubernetes Auth Method documentation (https://consul.io/docs/acl/auth-methods/kubernetes).
+    #
+    # You could retrieve this value from your `kubeconfig` by running:
+    #
+    # ```shell-session
+    # $ kubectl config view \
+    #   -o jsonpath="{.clusters[?(@.name=='<your cluster name>')].cluster.server}"
+    # ```
+    #
+    # @type: string
+    k8sAuthMethodHost: null
+
   # Configures metrics for Consul service mesh
   metrics:
     # Configures the Helm chart’s components

control-plane/subcommand/acl-init/command.go

@@ -36,11 +36,12 @@ type Command struct {
 	k8s   *flags.K8SFlags
 	http  *flags.HTTPFlags
 
-	flagSecretName    string
-	flagInitType      string
-	flagNamespace     string
-	flagACLDir        string
-	flagTokenSinkFile string
+	flagSecretName        string
+	flagInitType          string
+	flagNamespace         string
+	flagPrimaryDatacenter string
+	flagACLDir            string
+	flagTokenSinkFile     string
 
 	flagACLAuthMethod string // Auth Method to use for ACLs.
 	flagLogLevel      string
@@ -73,6 +74,7 @@ func (c *Command) init() {
 
 	// Flags related to using consul login to fetch the ACL token.
 	c.flags.StringVar(&c.flagNamespace, "k8s-namespace", "", "Name of Kubernetes namespace where the token Kubernetes secret is stored.")
+	c.flags.StringVar(&c.flagPrimaryDatacenter, "primary-datacenter", "", "Name of the primary datacenter when federation is enabled and the command is run in a secondary partition.")
 	c.flags.StringVar(&c.flagACLAuthMethod, "acl-auth-method", "", "Name of the auth method to login with.")
 	c.flags.StringVar(&c.flagComponentName, "component-name", "",
 		"Name of the component to pass to ACL Login as metadata.")
@@ -153,7 +155,7 @@ func (c *Command) Run(args []string) int {
 		meta := map[string]string{
 			"component": c.flagComponentName,
 		}
-		err := common.ConsulLogin(c.consulClient, cfg, c.logger, c.bearerTokenFile, c.flagACLAuthMethod, c.flagTokenSinkFile, "", "", meta)
+		err := common.ConsulLogin(c.consulClient, cfg, c.flagACLAuthMethod, c.flagPrimaryDatacenter, "", c.bearerTokenFile, "", c.flagTokenSinkFile, meta, c.logger)
 		if err != nil {
 			c.logger.Error("Consul login failed", "error", err)
 			return 1

control-plane/subcommand/common/common.go

@@ -88,7 +88,7 @@ func ValidateUnprivilegedPort(flagName, flagValue string) error {
 
 // ConsulLogin issues an ACL().Login to Consul and writes out the token to tokenSinkFile.
 // The logic of this is taken from the `consul login` command.
-func ConsulLogin(client *api.Client, cfg *api.Config, log hclog.Logger, bearerTokenFile, authMethodName, tokenSinkFile, namespace string, serviceAccountName string, meta map[string]string) error {
+func ConsulLogin(client *api.Client, cfg *api.Config, authMethodName, datacenter, namespace, bearerTokenFile, serviceAccountName, tokenSinkFile string, meta map[string]string, log hclog.Logger) error {
 	// Read the bearerTokenFile.
 	data, err := ioutil.ReadFile(bearerTokenFile)
 	if err != nil {
@@ -105,7 +105,10 @@ func ConsulLogin(client *api.Client, cfg *api.Config, log hclog.Logger, bearerTo
 			BearerToken: bearerToken,
 			Meta:        meta,
 		}
-		tok, _, err := client.ACL().Login(req, &api.WriteOptions{Namespace: namespace})
+		// The datacenter flag will either have the value of the primary datacenter or "". In case of the latter,
+		// the token will be created in the datacenter of the installation. In case a global token is required,
+		// the token will be created in the primary datacenter.
+		tok, _, err := client.ACL().Login(req, &api.WriteOptions{Namespace: namespace, Datacenter: datacenter})
 		if err != nil {
 			log.Error("unable to login", "error", err)
 			return fmt.Errorf("error logging in: %s", err)