From 7cb4c88f6daeb7af3b20a83011485095e4e79bd1 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Thu, 3 Mar 2022 16:05:38 -0500 Subject: [PATCH 1/2] Add support to create authmethod that can create global tokens for secondary datacenters --- .circleci/config.yml | 4 +- acceptance/go.mod | 2 +- .../tests/mesh-gateway/mesh_gateway_test.go | 39 +- .../templates/controller-clusterrole.yaml | 9 - .../templates/controller-deployment.yaml | 10 +- .../consul/templates/server-acl-init-job.yaml | 6 +- .../test/unit/controller-clusterrole.bats | 14 - .../test/unit/controller-deployment.bats | 43 +- .../consul/test/unit/server-acl-init-job.bats | 25 +- charts/consul/values.yaml | 24 +- control-plane/helper/test/test_util.go | 12 +- control-plane/subcommand/acl-init/command.go | 14 +- control-plane/subcommand/common/common.go | 7 +- .../subcommand/common/common_test.go | 32 +- .../subcommand/connect-init/command.go | 2 +- .../subcommand/server-acl-init/command.go | 120 ++- .../server-acl-init/command_ent_test.go | 282 +++++- .../server-acl-init/command_test.go | 860 ++++++++++++------ .../server-acl-init/connect_inject.go | 5 +- .../server-acl-init/create_or_update.go | 113 +-- 20 files changed, 1114 insertions(+), 509 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5018818bc2..25395bffc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,8 @@ executors: - image: docker.mirror.hashicorp.services/cimg/go:1.17.5 environment: TEST_RESULTS: /tmp/test-results # path to where test results are saved - CONSUL_VERSION: 1.11.2 # Consul's OSS version to use in tests - CONSUL_ENT_VERSION: 1.11.2+ent # Consul's enterprise version to use in tests + CONSUL_VERSION: 1.11.4 # Consul's OSS version to use in tests + CONSUL_ENT_VERSION: 1.11.4+ent # Consul's enterprise version to use in tests control-plane-path: &control-plane-path control-plane cli-path: &cli-path cli diff --git a/acceptance/go.mod b/acceptance/go.mod index 3c31530573..82affda09e 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -7,6 +7,7 @@ require ( github.com/hashicorp/consul-k8s/control-plane v0.0.0-20211207212234-aea9efea5638 github.com/hashicorp/consul/api v1.12.0 github.com/hashicorp/consul/sdk v0.9.0 + github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/vault/api v1.2.0 github.com/stretchr/testify v1.7.0 gopkg.in/yaml.v2 v2.4.0 @@ -49,7 +50,6 @@ require ( github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/go-version v1.2.0 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/acceptance/tests/mesh-gateway/mesh_gateway_test.go b/acceptance/tests/mesh-gateway/mesh_gateway_test.go index 230f5b01f4..77139aa706 100644 --- a/acceptance/tests/mesh-gateway/mesh_gateway_test.go +++ b/acceptance/tests/mesh-gateway/mesh_gateway_test.go @@ -33,8 +33,9 @@ func TestMeshGatewayDefault(t *testing.T) { "global.federation.enabled": "true", "global.federation.createFederationSecret": "true", - "connectInject.enabled": "true", - "controller.enabled": "true", + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", "meshGateway.enabled": "true", "meshGateway.replicas": "1", @@ -79,7 +80,9 @@ func TestMeshGatewayDefault(t *testing.T) { "server.extraVolumes[0].items[0].key": "serverConfigJSON", "server.extraVolumes[0].items[0].path": "config.json", - "connectInject.enabled": "true", + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", "meshGateway.enabled": "true", "meshGateway.replicas": "1", @@ -164,8 +167,9 @@ func TestMeshGatewaySecure(t *testing.T) { "global.federation.enabled": "true", "global.federation.createFederationSecret": "true", - "connectInject.enabled": "true", - "controller.enabled": "true", + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", "meshGateway.enabled": "true", "meshGateway.replicas": "1", @@ -191,6 +195,19 @@ func TestMeshGatewaySecure(t *testing.T) { _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(context.Background(), federationSecret, metav1.CreateOptions{}) require.NoError(t, err) + var k8sAuthMethodHost string + // When running on kind, the kube API address in kubeconfig will have a localhost address + // which will not work from inside the container. That's why we need to use the endpoints address instead + // which will point the node IP. + if cfg.UseKind { + // The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service. + kubernetesEndpoint, err := secondaryContext.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{}) + require.NoError(t, err) + k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port) + } else { + k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryContext.KubectlOptions(t)) + } + // Create secondary cluster secondaryHelmValues := map[string]string{ "global.datacenter": "dc2", @@ -207,7 +224,9 @@ func TestMeshGatewaySecure(t *testing.T) { "global.acls.replicationToken.secretName": federationSecretName, "global.acls.replicationToken.secretKey": "replicationToken", - "global.federation.enabled": "true", + "global.federation.enabled": "true", + "global.federation.k8sAuthMethodHost": k8sAuthMethodHost, + "global.federation.primaryDatacenter": "dc1", "server.extraVolumes[0].type": "secret", "server.extraVolumes[0].name": federationSecretName, @@ -215,7 +234,9 @@ func TestMeshGatewaySecure(t *testing.T) { "server.extraVolumes[0].items[0].key": "serverConfigJSON", "server.extraVolumes[0].items[0].path": "config.json", - "connectInject.enabled": "true", + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", "meshGateway.enabled": "true", "meshGateway.replicas": "1", @@ -248,9 +269,9 @@ func TestMeshGatewaySecure(t *testing.T) { // gateways. logger.Log(t, "creating proxy-defaults config") kustomizeDir := "../fixtures/bases/mesh-gateway" - k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir) + k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), kustomizeDir) helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { - k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir) + k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), kustomizeDir) }) // Check that we can connect services over the mesh gateways diff --git a/charts/consul/templates/controller-clusterrole.yaml b/charts/consul/templates/controller-clusterrole.yaml index 45fa8d8458..e2522a2eae 100644 --- a/charts/consul/templates/controller-clusterrole.yaml +++ b/charts/consul/templates/controller-clusterrole.yaml @@ -57,15 +57,6 @@ rules: - get - list - update -{{- if .Values.global.acls.manageSystemACLs }} -- apiGroups: [""] - resources: - - secrets - resourceNames: - - {{ template "consul.fullname" . }}-controller-acl-token - verbs: - - get -{{- end }} {{- if .Values.global.enablePodSecurityPolicies }} - apiGroups: ["policy"] resources: ["podsecuritypolicies"] diff --git a/charts/consul/templates/controller-deployment.yaml b/charts/consul/templates/controller-deployment.yaml index 52234f5a68..5c6f65529b 100644 --- a/charts/consul/templates/controller-deployment.yaml +++ b/charts/consul/templates/controller-deployment.yaml @@ -87,7 +87,12 @@ spec: - | consul-k8s-control-plane acl-init \ -component-name=controller \ + {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }} + -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \ + -primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \ + {{- else }} -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \ + {{- end }} {{- if .Values.global.adminPartitions.enabled }} -partition={{ .Values.global.adminPartitions.name }} \ {{- end }} @@ -139,10 +144,7 @@ spec: - "/bin/sh" - "-ec" - | - consul-k8s-control-plane consul-logout \ - {{- if .Values.global.adminPartitions.enabled }} - -partition={{ .Values.global.adminPartitions.name }} \ - {{- end }} + consul-k8s-control-plane consul-logout {{- end }} env: {{- if .Values.global.acls.manageSystemACLs }} diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml index 050c255930..a6c26a0656 100644 --- a/charts/consul/templates/server-acl-init-job.yaml +++ b/charts/consul/templates/server-acl-init-job.yaml @@ -185,10 +185,14 @@ spec: {{- if (or (and (ne (.Values.connectInject.enabled | toString) "-") .Values.connectInject.enabled) (and (eq (.Values.connectInject.enabled | toString) "-") .Values.global.enabled)) }} -create-inject-token=true \ {{- if and .Values.externalServers.enabled .Values.externalServers.k8sAuthMethodHost }} - -inject-auth-method-host={{ .Values.externalServers.k8sAuthMethodHost }} \ + -auth-method-host={{ .Values.externalServers.k8sAuthMethodHost }} \ {{- end }} {{- end }} + {{- if .Values.global.federation.k8sAuthMethodHost }} + -auth-method-host={{ .Values.global.federation.k8sAuthMethodHost }} \ + {{- end }} + {{- if .Values.meshGateway.enabled }} -create-mesh-gateway-token=true \ {{- end }} diff --git a/charts/consul/test/unit/controller-clusterrole.bats b/charts/consul/test/unit/controller-clusterrole.bats index ed64f9050a..dc0b560e1d 100644 --- a/charts/consul/test/unit/controller-clusterrole.bats +++ b/charts/consul/test/unit/controller-clusterrole.bats @@ -43,17 +43,3 @@ load _helpers yq '.rules | map(select(.resources[0] == "podsecuritypolicies")) | length' | tee /dev/stderr) [ "${actual}" = "1" ] } - -#-------------------------------------------------------------------- -# global.acls.manageSystemACLs - -@test "controller/ClusterRole: allows secret access with global.acls.manageSystemACLs=true" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/controller-clusterrole.yaml \ - --set 'controller.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - . | tee /dev/stderr | - yq -r '.rules | map(select(.resourceNames[0] == "RELEASE-NAME-consul-controller-acl-token")) | length' | tee /dev/stderr) - [ "${actual}" = "1" ] -} diff --git a/charts/consul/test/unit/controller-deployment.bats b/charts/consul/test/unit/controller-deployment.bats index 5161ebeb6d..c9401c4d19 100644 --- a/charts/consul/test/unit/controller-deployment.bats +++ b/charts/consul/test/unit/controller-deployment.bats @@ -57,20 +57,6 @@ load _helpers [ "${actual}" = "true" ] } -@test "controller/Deployment: consul-logout preStop hook has partition when partitions are enabled" { - cd `chart_dir` - local actual=$(helm template \ - -s templates/controller-deployment.yaml \ - --set 'controller.enabled=true' \ - --set 'global.acls.manageSystemACLs=true' \ - --set 'global.enableConsulNamespaces=true' \ - --set 'global.adminPartitions.enabled=true' \ - --set 'global.adminPartitions.name=default' \ - . | tee /dev/stderr | - yq '[.spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]] | any(contains("-partition=default"))' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - @test "controller/Deployment: CONSUL_HTTP_TOKEN_FILE is not set when acls are disabled" { cd `chart_dir` local actual=$(helm template \ @@ -243,6 +229,35 @@ load _helpers [ "${actual}" = "get-auto-encrypt-client-ca" ] } +@test "controller/Deployment: init container is created when global.acls.manageSystemACLs=true and has correct command when federation enabled in non-primary datacenter" { + cd `chart_dir` + local object=$(helm template \ + -s templates/controller-deployment.yaml \ + --set 'controller.enabled=true' \ + --set 'global.datacenter=dc2' \ + --set 'global.federation.enabled=true' \ + --set 'global.federation.primaryDatacenter=dc1' \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.acls.manageSystemACLs=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.initContainers[] | select(.name == "controller-acl-init")' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.command | any(contains("consul-k8s-control-plane acl-init"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq -r '.command | any(contains("-acl-auth-method=RELEASE-NAME-consul-k8s-component-auth-method-dc2"))' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo $object | + yq -r '.command | any(contains("-primary-datacenter=dc1"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # global.tls.enabled diff --git a/charts/consul/test/unit/server-acl-init-job.bats b/charts/consul/test/unit/server-acl-init-job.bats index 0ceeaf38bd..73dd3f6432 100644 --- a/charts/consul/test/unit/server-acl-init-job.bats +++ b/charts/consul/test/unit/server-acl-init-job.bats @@ -1712,7 +1712,7 @@ load _helpers --set 'global.acls.manageSystemACLs=true' \ --set 'connectInject.enabled=true' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | any(contains("-inject-auth-method-host"))' | tee /dev/stderr) + yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host"))' | tee /dev/stderr) [ "${actual}" = "false" ] } @@ -1724,11 +1724,11 @@ load _helpers --set 'externalServers.k8sAuthMethodHost=foo.com' \ --set 'connectInject.enabled=true' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | any(contains("-inject-auth-method-host"))' | tee /dev/stderr) + yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host"))' | tee /dev/stderr) [ "${actual}" = "false" ] } -@test "serverACLInit/Job: can provide custom auth method host" { +@test "serverACLInit/Job: can provide custom auth method host for external servers" { cd `chart_dir` local actual=$(helm template \ -s templates/server-acl-init-job.yaml \ @@ -1739,7 +1739,24 @@ load _helpers --set 'externalServers.hosts[0]=foo.com' \ --set 'externalServers.k8sAuthMethodHost=foo.com' \ . | tee /dev/stderr| - yq '.spec.template.spec.containers[0].command | any(contains("-inject-auth-method-host=foo.com"))' | tee /dev/stderr) + yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host=foo.com"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "serverACLInit/Job: can provide custom auth method host for federation" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/server-acl-init-job.yaml \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.federation.enabled=true' \ + --set 'global.federation.primaryDatacenter=dc1' \ + --set 'global.federation.k8sAuthMethodHost=foo.com' \ + --set 'meshGateway.enabled=true' \ + . | tee /dev/stderr| + yq '.spec.template.spec.containers[0].command | any(contains("-auth-method-host=foo.com"))' | tee /dev/stderr) [ "${actual}" = "true" ] } diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 4d019b36be..15f38105b5 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -85,7 +85,7 @@ global: # image: "hashicorp/consul-enterprise:1.10.0-ent" # ``` # @default: hashicorp/consul: - image: "hashicorp/consul:1.11.3" + image: "hashicorp/consul:1.11.4" # Array of objects containing image pull secret names that will be applied to each service account. # This can be used to reference image pull secrets if using a custom consul or consul-k8s-control-plane Docker image. @@ -431,13 +431,31 @@ global: createFederationSecret: false # The name of the primary datacenter. - primaryDatacenter: "" + # @type: string + primaryDatacenter: null # A list of addresses of the primary mesh gateways in the form `:`. - # (e.g. ["1.1.1.1:443", "2.3.4.5:443"] + # (e.g. ["1.1.1.1:443", "2.3.4.5:443"] # @type: array primaryGateways: [] + # If you are setting `global.federation.enabled` to true and are in a secondary datacenter, + # set `k8sAuthMethodHost` to the address of the Kubernetes API server of the secondary datacenter. + # This address must be reachable from the Consul servers in the primary datacenter. + # This authmethod will be used to provision ACL tokens for Consul components and is different + # from the one used by the Consul Service Mesh. + # Please see the Kubernetes Auth Method documentation (https://consul.io/docs/acl/auth-methods/kubernetes). + # + # You could retrieve this value from your `kubeconfig` by running: + # + # ```shell-session + # $ kubectl config view \ + # -o jsonpath="{.clusters[?(@.name=='')].cluster.server}" + # ``` + # + # @type: string + k8sAuthMethodHost: null + # Configures metrics for Consul service mesh metrics: # Configures the Helm chart’s components diff --git a/control-plane/helper/test/test_util.go b/control-plane/helper/test/test_util.go index c804efdf02..142aa535c5 100644 --- a/control-plane/helper/test/test_util.go +++ b/control-plane/helper/test/test_util.go @@ -69,11 +69,11 @@ func SetupK8sComponentAuthMethod(t *testing.T, consulClient *api.Client, service k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") if r != nil && r.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" && r.Method == "POST" { - w.Write([]byte(tokenReviewsResponse(serviceAccountName, k8sComponentNS))) + w.Write([]byte(TokenReviewsResponse(serviceAccountName, k8sComponentNS))) } if r != nil && r.URL.Path == fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", k8sComponentNS, serviceAccountName) && r.Method == "GET" { - w.Write([]byte(serviceAccountGetResponse(serviceAccountName, k8sComponentNS))) + w.Write([]byte(ServiceAccountGetResponse(serviceAccountName, k8sComponentNS))) } })) t.Cleanup(k8sMockServer.Close) @@ -149,11 +149,11 @@ func SetupK8sAuthMethodWithNamespaces(t *testing.T, consulClient *api.Client, se k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "application/json") if r != nil && r.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" && r.Method == "POST" { - w.Write([]byte(tokenReviewsResponse(serviceName, k8sServiceNS))) + w.Write([]byte(TokenReviewsResponse(serviceName, k8sServiceNS))) } if r != nil && r.URL.Path == fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", k8sServiceNS, serviceName) && r.Method == "GET" { - w.Write([]byte(serviceAccountGetResponse(serviceName, k8sServiceNS))) + w.Write([]byte(ServiceAccountGetResponse(serviceName, k8sServiceNS))) } })) t.Cleanup(k8sMockServer.Close) @@ -196,7 +196,7 @@ func SetupK8sAuthMethodWithNamespaces(t *testing.T, consulClient *api.Client, se require.NoError(t, err) } -func tokenReviewsResponse(name, ns string) string { +func TokenReviewsResponse(name, ns string) string { return fmt.Sprintf(`{ "kind": "TokenReview", "apiVersion": "authentication.k8s.io/v1", @@ -221,7 +221,7 @@ func tokenReviewsResponse(name, ns string) string { }`, ns, name, ns) } -func serviceAccountGetResponse(name, ns string) string { +func ServiceAccountGetResponse(name, ns string) string { return fmt.Sprintf(`{ "kind": "ServiceAccount", "apiVersion": "v1", diff --git a/control-plane/subcommand/acl-init/command.go b/control-plane/subcommand/acl-init/command.go index b4e940db08..9f19708778 100644 --- a/control-plane/subcommand/acl-init/command.go +++ b/control-plane/subcommand/acl-init/command.go @@ -36,11 +36,12 @@ type Command struct { k8s *flags.K8SFlags http *flags.HTTPFlags - flagSecretName string - flagInitType string - flagNamespace string - flagACLDir string - flagTokenSinkFile string + flagSecretName string + flagInitType string + flagNamespace string + flagPrimaryDatacenter string + flagACLDir string + flagTokenSinkFile string flagACLAuthMethod string // Auth Method to use for ACLs. flagLogLevel string @@ -73,6 +74,7 @@ func (c *Command) init() { // Flags related to using consul login to fetch the ACL token. c.flags.StringVar(&c.flagNamespace, "k8s-namespace", "", "Name of Kubernetes namespace where the token Kubernetes secret is stored.") + c.flags.StringVar(&c.flagPrimaryDatacenter, "primary-datacenter", "", "Name of the primary datacenter when federation is enabled and the command is run in a secondary datacenter.") c.flags.StringVar(&c.flagACLAuthMethod, "acl-auth-method", "", "Name of the auth method to login with.") c.flags.StringVar(&c.flagComponentName, "component-name", "", "Name of the component to pass to ACL Login as metadata.") @@ -153,7 +155,7 @@ func (c *Command) Run(args []string) int { meta := map[string]string{ "component": c.flagComponentName, } - err := common.ConsulLogin(c.consulClient, cfg, c.logger, c.bearerTokenFile, c.flagACLAuthMethod, c.flagTokenSinkFile, "", "", meta) + err := common.ConsulLogin(c.consulClient, cfg, c.flagACLAuthMethod, c.flagPrimaryDatacenter, "", c.bearerTokenFile, "", c.flagTokenSinkFile, meta, c.logger) if err != nil { c.logger.Error("Consul login failed", "error", err) return 1 diff --git a/control-plane/subcommand/common/common.go b/control-plane/subcommand/common/common.go index b7929f2dae..dabbe9f4d9 100644 --- a/control-plane/subcommand/common/common.go +++ b/control-plane/subcommand/common/common.go @@ -88,7 +88,7 @@ func ValidateUnprivilegedPort(flagName, flagValue string) error { // ConsulLogin issues an ACL().Login to Consul and writes out the token to tokenSinkFile. // The logic of this is taken from the `consul login` command. -func ConsulLogin(client *api.Client, cfg *api.Config, log hclog.Logger, bearerTokenFile, authMethodName, tokenSinkFile, namespace string, serviceAccountName string, meta map[string]string) error { +func ConsulLogin(client *api.Client, cfg *api.Config, authMethodName, datacenter, namespace, bearerTokenFile, serviceAccountName, tokenSinkFile string, meta map[string]string, log hclog.Logger) error { // Read the bearerTokenFile. data, err := ioutil.ReadFile(bearerTokenFile) if err != nil { @@ -105,7 +105,10 @@ func ConsulLogin(client *api.Client, cfg *api.Config, log hclog.Logger, bearerTo BearerToken: bearerToken, Meta: meta, } - tok, _, err := client.ACL().Login(req, &api.WriteOptions{Namespace: namespace}) + // The datacenter flag will either have the value of the primary datacenter or "". In case of the latter, + // the token will be created in the datacenter of the installation. In case a global token is required, + // the token will be created in the primary datacenter. + tok, _, err := client.ACL().Login(req, &api.WriteOptions{Namespace: namespace, Datacenter: datacenter}) if err != nil { log.Error("unable to login", "error", err) return fmt.Errorf("error logging in: %s", err) diff --git a/control-plane/subcommand/common/common_test.go b/control-plane/subcommand/common/common_test.go index f4bcfd172b..12cfa092b5 100644 --- a/control-plane/subcommand/common/common_test.go +++ b/control-plane/subcommand/common/common_test.go @@ -64,7 +64,7 @@ func TestConsulLogin(t *testing.T) { log, err := Logger("INFO", false) require.NoError(err) client, cfg := startMockServer(t, &counter) - err = ConsulLogin(client, cfg, log, bearerTokenFile, testAuthMethod, tokenFile, "", "", testPodMeta) + err = ConsulLogin(client, cfg, testAuthMethod, "dc1", "", bearerTokenFile, "", tokenFile, testPodMeta, log) require.NoError(err) require.Equal(counter, 1) // Validate that the token file was written to disk. @@ -78,15 +78,7 @@ func TestConsulLogin_EmptyBearerTokenFile(t *testing.T) { require := require.New(t) bearerTokenFile := WriteTempFile(t, "") - err := ConsulLogin( - nil, nil, nil, - bearerTokenFile, - testAuthMethod, - "", - "", - "", - testPodMeta, - ) + err := ConsulLogin(nil, nil, testAuthMethod, "", "", bearerTokenFile, "", "", testPodMeta, hclog.NewNullLogger()) require.EqualError(err, fmt.Sprintf("no bearer token found in %s", bearerTokenFile)) } @@ -94,15 +86,7 @@ func TestConsulLogin_BearerTokenFileDoesNotExist(t *testing.T) { t.Parallel() require := require.New(t) randFileName := fmt.Sprintf("/foo/%d/%d", rand.Int(), rand.Int()) - err := ConsulLogin( - nil, nil, nil, - randFileName, - testAuthMethod, - "", - "", - "", - testPodMeta, - ) + err := ConsulLogin(nil, nil, testAuthMethod, "", "", randFileName, "", "", testPodMeta, hclog.NewNullLogger()) require.Error(err) require.Contains(err.Error(), "unable to read bearerTokenFile") } @@ -117,15 +101,7 @@ func TestConsulLogin_TokenFileUnwritable(t *testing.T) { log, err := Logger("INFO", false) require.NoError(err) randFileName := fmt.Sprintf("/foo/%d/%d", rand.Int(), rand.Int()) - err = ConsulLogin( - client, cfg, log, - bearerTokenFile, - testAuthMethod, - randFileName, - "", - "", - testPodMeta, - ) + err = ConsulLogin(client, cfg, testAuthMethod, "", "", bearerTokenFile, "", randFileName, testPodMeta, log) require.Error(err) require.Contains(err.Error(), "error writing token to file sink") } diff --git a/control-plane/subcommand/connect-init/command.go b/control-plane/subcommand/connect-init/command.go index 883cdc0658..9ae479eea2 100644 --- a/control-plane/subcommand/connect-init/command.go +++ b/control-plane/subcommand/connect-init/command.go @@ -123,7 +123,7 @@ func (c *Command) Run(args []string) int { if c.flagACLAuthMethod != "" { // loginMeta is the default metadata that we pass to the consul login API. loginMeta := map[string]string{"pod": fmt.Sprintf("%s/%s", c.flagPodNamespace, c.flagPodName)} - err = common.ConsulLogin(consulClient, cfg, c.logger, c.flagBearerTokenFile, c.flagACLAuthMethod, c.flagACLTokenSink, c.flagAuthMethodNamespace, c.flagServiceAccountName, loginMeta) + err = common.ConsulLogin(consulClient, cfg, c.flagACLAuthMethod, "", c.flagAuthMethodNamespace, c.flagBearerTokenFile, c.flagServiceAccountName, c.flagACLTokenSink, loginMeta, c.logger) if err != nil { c.logger.Error("unable to complete login", "error", err) return 1 diff --git a/control-plane/subcommand/server-acl-init/command.go b/control-plane/subcommand/server-acl-init/command.go index 17e19f5120..6c5e0f3e4b 100644 --- a/control-plane/subcommand/server-acl-init/command.go +++ b/control-plane/subcommand/server-acl-init/command.go @@ -45,9 +45,9 @@ type Command struct { flagCreateSyncToken bool flagSyncConsulNodeName string - flagCreateInjectToken bool - flagInjectAuthMethodHost string - flagBindingRuleSelector string + flagCreateInjectToken bool + flagAuthMethodHost string + flagBindingRuleSelector string flagCreateControllerPoliciesAndBindings bool @@ -145,7 +145,7 @@ func (c *Command) init() { "Toggle for creating a connect inject auth method. Deprecated: use -create-inject-token instead.") c.flags.BoolVar(&c.flagCreateInjectToken, "create-inject-token", false, "Toggle for creating a connect inject auth method and an ACL token.") - c.flags.StringVar(&c.flagInjectAuthMethodHost, "inject-auth-method-host", "", + c.flags.StringVar(&c.flagAuthMethodHost, "auth-method-host", "", "Kubernetes Host config parameter for the auth method."+ "If not provided, the default cluster Kubernetes service will be used.") c.flags.StringVar(&c.flagBindingRuleSelector, "acl-binding-rule-selector", "", @@ -378,11 +378,11 @@ func (c *Command) Run(args []string) int { return 1 } c.log.Info("Current datacenter", "datacenter", consulDC, "primaryDC", primaryDC) - isPrimary := consulDC == primaryDC + primary := consulDC == primaryDC - if c.flagEnablePartitions && c.flagPartitionName == consulDefaultPartition && isPrimary { + if c.flagEnablePartitions && c.flagPartitionName == consulDefaultPartition && primary { // Partition token is local because only the Primary datacenter can have Admin Partitions. - err := c.createLocalACL("partitions", partitionRules, consulDC, isPrimary, consulClient) + err := c.createLocalACL("partitions", partitionRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -440,13 +440,22 @@ func (c *Command) Run(args []string) int { // Create the component auth method, this is the auth method that Consul components will use // to issue an `ACL().Login()` against at startup, for local tokens. - componentAuthMethodName := c.withPrefix("k8s-component-auth-method") - err = c.configureComponentAuthMethod(consulClient, componentAuthMethodName) + localComponentAuthMethodName := c.withPrefix("k8s-component-auth-method") + err = c.configureLocalComponentAuthMethod(consulClient, localComponentAuthMethodName) if err != nil { c.log.Error(err.Error()) return 1 } + globalComponentAuthMethodName := fmt.Sprintf("%s-%s", localComponentAuthMethodName, consulDC) + if !primary && c.flagAuthMethodHost != "" { + err = c.configureGlobalComponentAuthMethod(consulClient, globalComponentAuthMethodName, primaryDC) + if err != nil { + c.log.Error(err.Error()) + return 1 + } + } + if c.flagCreateClientToken { agentRules, err := c.agentRules() if err != nil { @@ -454,14 +463,14 @@ func (c *Command) Run(args []string) int { return 1 } - err = c.createLocalACL("client", agentRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("client", agentRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 } } - if c.createAnonymousPolicy(isPrimary) { + if c.createAnonymousPolicy(primary) { // When the default partition is in a VM, the anonymous policy does not allow cross-partition // DNS lookups. The anonymous policy in the default partition needs to be updated in order to // support this use-case. Creating a separate anonymous token client that updates the anonymous @@ -493,9 +502,9 @@ func (c *Command) Run(args []string) int { // If namespaces are enabled, the policy and token needs to be global // to be allowed to create namespaces. if c.flagEnableNamespaces { - err = c.createGlobalACL("catalog-sync", syncRules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("catalog-sync", syncRules, consulDC, primary, consulClient) } else { - err = c.createLocalACL("catalog-sync", syncRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("catalog-sync", syncRules, consulDC, primary, consulClient) } if err != nil { c.log.Error(err.Error()) @@ -521,9 +530,9 @@ func (c *Command) Run(args []string) int { // If namespaces are enabled, the policy and token need to be global // to be allowed to create namespaces. if c.flagEnableNamespaces { - err = c.createGlobalACL("connect-inject", injectRules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("connect-inject", injectRules, consulDC, primary, consulClient) } else { - err = c.createLocalACL("connect-inject", injectRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("connect-inject", injectRules, consulDC, primary, consulClient) } if err != nil { @@ -535,9 +544,9 @@ func (c *Command) Run(args []string) int { if c.flagCreateEntLicenseToken { var err error if c.flagEnablePartitions { - err = c.createLocalACL("enterprise-license", entPartitionLicenseRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("enterprise-license", entPartitionLicenseRules, consulDC, primary, consulClient) } else { - err = c.createLocalACL("enterprise-license", entLicenseRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("enterprise-license", entLicenseRules, consulDC, primary, consulClient) } if err != nil { c.log.Error(err.Error()) @@ -546,7 +555,7 @@ func (c *Command) Run(args []string) int { } if c.flagCreateSnapshotAgentToken { - err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, isPrimary, consulClient) + err := c.createLocalACL("client-snapshot-agent", snapshotAgentRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -559,7 +568,7 @@ func (c *Command) Run(args []string) int { c.log.Error("Error templating api gateway rules", "err", err) return 1 } - err = c.createLocalACL("api-gateway-controller", apigwRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL("api-gateway-controller", apigwRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -575,7 +584,7 @@ func (c *Command) Run(args []string) int { // Mesh gateways require a global policy/token because they must // discover services in other datacenters. - err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL("mesh-gateway", meshGatewayRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -630,7 +639,7 @@ func (c *Command) Run(args []string) int { // the words "ingress-gateway". We need to create unique names for tokens // across all gateway types and so must suffix with `-ingress-gateway`. tokenName := fmt.Sprintf("%s-ingress-gateway", name) - err = c.createLocalACL(tokenName, ingressGatewayRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL(tokenName, ingressGatewayRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -686,7 +695,7 @@ func (c *Command) Run(args []string) int { // the words "ingress-gateway". We need to create unique names for tokens // across all gateway types and so must suffix with `-terminating-gateway`. tokenName := fmt.Sprintf("%s-terminating-gateway", name) - err = c.createLocalACL(tokenName, terminatingGatewayRules, consulDC, isPrimary, consulClient) + err = c.createLocalACL(tokenName, terminatingGatewayRules, consulDC, primary, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -703,9 +712,9 @@ func (c *Command) Run(args []string) int { // Policy must be global because it replicates from the primary DC // and so the primary DC needs to be able to accept the token. if aclReplicationToken != "" { - err = c.createGlobalACLWithSecretID(common.ACLReplicationTokenName, rules, consulDC, isPrimary, consulClient, aclReplicationToken) + err = c.createGlobalACLWithSecretID(common.ACLReplicationTokenName, rules, consulDC, primary, consulClient, aclReplicationToken) } else { - err = c.createGlobalACL(common.ACLReplicationTokenName, rules, consulDC, isPrimary, consulClient) + err = c.createGlobalACL(common.ACLReplicationTokenName, rules, consulDC, primary, consulClient) } if err != nil { c.log.Error(err.Error()) @@ -726,7 +735,11 @@ func (c *Command) Run(args []string) int { // Controller token must be global because config entry writes all // go to the primary datacenter. This means secondary datacenters need // a token that is known by the primary datacenters. - err = c.createACLPolicyRoleAndBindingRule("controller", rules, consulDC, isPrimary, componentAuthMethodName, serviceAccountName, consulClient) + authMethodName := localComponentAuthMethodName + if !primary { + authMethodName = globalComponentAuthMethodName + } + err = c.createACLPolicyRoleAndBindingRule("controller", rules, consulDC, primaryDC, globalToken, primary, authMethodName, serviceAccountName, consulClient) if err != nil { c.log.Error(err.Error()) return 1 @@ -736,6 +749,43 @@ func (c *Command) Run(args []string) int { return 0 } +// configureGlobalComponentAuthMethod sets up an AuthMethod in the primary datacenter, +// that the Consul components will use to issue global ACL tokens with. +func (c *Command) configureGlobalComponentAuthMethod(consulClient *api.Client, authMethodName, primaryDC string) error { + // Create the auth method template. This requires calls to the kubernetes environment. + authMethod, err := c.createAuthMethodTmpl(authMethodName, false) + if err != nil { + return err + } + authMethod.TokenLocality = "global" + writeOptions := &api.WriteOptions{Datacenter: primaryDC} + return c.createAuthMethod(consulClient, &authMethod, writeOptions) +} + +// configureLocalComponentAuthMethod sets up an AuthMethod in the same datacenter, +// that the Consul components will use to issue local ACL tokens with. +func (c *Command) configureLocalComponentAuthMethod(consulClient *api.Client, authMethodName string) error { + // Create the auth method template. This requires calls to the kubernetes environment. + authMethod, err := c.createAuthMethodTmpl(authMethodName, false) + if err != nil { + return err + } + return c.createAuthMethod(consulClient, &authMethod, &api.WriteOptions{}) +} + +// createAuthMethod creates the desired Authmethod. +func (c *Command) createAuthMethod(consulClient *api.Client, authMethod *api.ACLAuthMethod, writeOptions *api.WriteOptions) error { + return c.untilSucceeds(fmt.Sprintf("creating auth method %s", authMethod.Name), + func() error { + var err error + // `AuthMethodCreate` will also be able to update an existing + // AuthMethod based on the name provided. This means that any + // configuration changes will correctly update the AuthMethod. + _, _, err = consulClient.ACL().AuthMethodCreate(authMethod, writeOptions) + return err + }) +} + // getBootstrapToken returns the existing bootstrap token if there is one by // reading the Kubernetes Secret with name secretName. // If there is no bootstrap token yet, then it returns an empty string (not an error). @@ -897,27 +947,9 @@ func (c *Command) validateFlags() error { return nil } -// configureComponentAuthMethod sets up an AuthMethod that the Consul components will use to issue ACL logins with. -func (c *Command) configureComponentAuthMethod(consulClient *api.Client, authMethodName string) error { - // Create the auth method template. This requires calls to the kubernetes environment. - authMethodTmpl, err := c.createAuthMethodTmpl(authMethodName, false) - if err != nil { - return err - } - err = c.untilSucceeds(fmt.Sprintf("creating auth method %s", authMethodTmpl.Name), - func() error { - var err error - // `AuthMethodCreate` will also be able to update an existing - // AuthMethod based on the name provided. This means that any - // configuration changes will correctly update the AuthMethod. - _, _, err = consulClient.ACL().AuthMethodCreate(&authMethodTmpl, &api.WriteOptions{}) - return err - }) - return err -} - const consulDefaultNamespace = "default" const consulDefaultPartition = "default" +const globalToken = true const synopsis = "Initialize ACLs on Consul servers and other components." const help = ` Usage: consul-k8s-control-plane server-acl-init [options] diff --git a/control-plane/subcommand/server-acl-init/command_ent_test.go b/control-plane/subcommand/server-acl-init/command_ent_test.go index a539e01e56..cfbfeffa8c 100644 --- a/control-plane/subcommand/server-acl-init/command_ent_test.go +++ b/control-plane/subcommand/server-acl-init/command_ent_test.go @@ -4,10 +4,14 @@ package serveraclinit import ( "context" + "fmt" + "net/http" + "net/http/httptest" "strings" "testing" "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil" @@ -24,7 +28,7 @@ func TestRun_ConnectInject_SingleDestinationNamespace(t *testing.T) { consulDestNamespaces := []string{"default", "destination"} for _, consulDestNamespace := range consulDestNamespaces { t.Run(consulDestNamespace, func(tt *testing.T) { - k8s, testAgent := completeEnterpriseSetup(tt) + k8s, testAgent := completeSetup(tt) defer testAgent.Stop() setUpK8sServiceAccount(tt, k8s, ns) require := require.New(tt) @@ -152,7 +156,7 @@ func TestRun_ConnectInject_NamespaceMirroring(t *testing.T) { for name, c := range cases { t.Run(name, func(tt *testing.T) { - k8s, testAgent := completeEnterpriseSetup(tt) + k8s, testAgent := completeSetup(tt) defer testAgent.Stop() setUpK8sServiceAccount(tt, k8s, ns) require := require.New(tt) @@ -269,7 +273,7 @@ func TestRun_ACLPolicyUpdates(t *testing.T) { k8sNamespaceFlags := []string{"default", "other"} for _, k8sNamespaceFlag := range k8sNamespaceFlags { t.Run(k8sNamespaceFlag, func(t *testing.T) { - k8s, testAgent := completeEnterpriseSetup(t) + k8s, testAgent := completeSetup(t) setUpK8sServiceAccount(t, k8s, k8sNamespaceFlag) defer testAgent.Stop() require := require.New(t) @@ -583,7 +587,7 @@ func TestRun_ConnectInject_Updates(t *testing.T) { for name, c := range cases { t.Run(name, func(tt *testing.T) { require := require.New(tt) - k8s, testAgent := completeEnterpriseSetup(tt) + k8s, testAgent := completeSetup(tt) defer testAgent.Stop() setUpK8sServiceAccount(tt, k8s, ns) @@ -763,7 +767,7 @@ func TestRun_TokensWithNamespacesEnabled(t *testing.T) { } for testName, c := range cases { t.Run(testName, func(t *testing.T) { - k8s, testSvr := completeEnterpriseSetup(t) + k8s, testSvr := completeSetup(t) setUpK8sServiceAccount(t, k8s, ns) defer testSvr.Stop() require := require.New(t) @@ -1019,7 +1023,7 @@ partition "default" { } for _, c := range cases { t.Run(c.TestName, func(t *testing.T) { - k8s, testSvr := completeEnterpriseSetup(t) + k8s, testSvr := completeSetup(t) defer testSvr.Stop() setUpK8sServiceAccount(t, k8s, ns) require := require.New(t) @@ -1075,12 +1079,12 @@ partition "default" { } } -// Test creating the correct ACL policies and Binding Rules for components that use the auth method. +// Test creating the correct ACL policies and Binding Rules for components in the primary datacenter. // The test works by running the command and then ensuring that: // * An ACLBindingRule exists which references the ACLRole. // * An ACLRole exists and has the correct PolicyName in it's ACLPolicyLinkRule list. // * The ACLPolicy exists. -func TestRun_PoliciesAndBindingRulesForACLLoginNamespacesEnabled(t *testing.T) { +func TestRun_PrimaryDatacenter_PoliciesAndBindingRulesForACLLogin_NamespacesEnabled(t *testing.T) { t.Parallel() cases := []struct { @@ -1113,7 +1117,9 @@ func TestRun_PoliciesAndBindingRulesForACLLoginNamespacesEnabled(t *testing.T) { cmdArgs := append([]string{ "-timeout=500ms", "-resource-prefix=" + resourcePrefix, - "-k8s-namespace=" + ns, + "-k8s-namespace=" + c.Namespace, + "-enable-namespaces", + "-consul-inject-destination-namespace", c.Namespace, "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], }, c.TokenFlags...) @@ -1152,7 +1158,7 @@ func TestRun_PoliciesAndBindingRulesForACLLoginNamespacesEnabled(t *testing.T) { require.True(t, found) // Check that there exists a BindingRule that references this Role. - rb, _, err := consul.ACL().BindingRuleList("release-name-"+componentAuthMethod, &api.QueryOptions{}) + rb, _, err := consul.ACL().BindingRuleList(fmt.Sprintf("%s-%s", resourcePrefix, componentAuthMethod), &api.QueryOptions{}) require.NoError(t, err) require.NotNil(t, rb) found = false @@ -1168,16 +1174,256 @@ func TestRun_PoliciesAndBindingRulesForACLLoginNamespacesEnabled(t *testing.T) { } } -// Set up test consul agent and kubernetes cluster. -func completeEnterpriseSetup(t *testing.T) (*fake.Clientset, *testutil.TestServer) { - k8s := fake.NewSimpleClientset() +// Test creating the correct ACL policies and Binding Rules for components running in the secondary datacenter. +// The test works by running the command and then ensuring that: +// * An ACLBindingRule exists which references the ACLRole. +// * An ACLRole exists and has the correct PolicyName in it's ACLPolicyLinkRule list. +// * The ACLPolicy exists. +func TestRun_SecondaryDatacenter_PoliciesAndBindingRulesForACLLogin_NamespacesEnabled(t *testing.T) { + t.Parallel() - svr, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { - c.ACL.Enabled = true - }) - require.NoError(t, err) + const ( + secondaryDatacenter = "dc2" + primaryDatacenter = "dc1" + ) + cases := []struct { + TestName string + TokenFlags []string + PolicyNames []string + Roles []string + Namespace string + }{ + { + TestName: "Controller", + TokenFlags: []string{"-create-controller-token"}, + PolicyNames: []string{"controller-policy-" + secondaryDatacenter}, + Roles: []string{resourcePrefix + "-controller-acl-role-" + secondaryDatacenter}, + Namespace: ns, + }, + } + for _, c := range cases { + t.Run(c.TestName, func(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile := common.WriteTempFile(t, bootToken) + k8s, consul, consulHTTPAddr, cleanup := mockReplicatedSetup(t, bootToken) + setUpK8sServiceAccount(t, k8s, ns) + defer cleanup() + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-federation", + "-timeout=1m", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + c.Namespace, + "-enable-namespaces", + "-consul-inject-destination-namespace", c.Namespace, + "-auth-method-host=" + "https://my-kube.com", + "-acl-replication-token-file", tokenFile, + "-server-address", strings.Split(consulHTTPAddr, ":")[0], + "-server-port", strings.Split(consulHTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // Check that the Role exists + has correct Policy and is associated with a BindingRule. + for i := range c.Roles { + // Check that the Policy exists. + + policy, _, err := consul.ACL().PolicyReadByName(c.PolicyNames[i], &api.QueryOptions{Datacenter: primaryDatacenter}) + require.NoError(t, err) + require.NotNil(t, policy) + + // Check that the Role exists. + role, _, err := consul.ACL().RoleReadByName(c.Roles[i], &api.QueryOptions{Datacenter: primaryDatacenter}) + require.NoError(t, err) + require.NotNil(t, role) + + // Check that the Role references the Policy. + found := false + for x := range role.Policies { + if role.Policies[x].Name == policy.Name { + found = true + break + } + } + require.True(t, found) + + // Check that there exists a BindingRule that references this Role. + rb, _, err := consul.ACL().BindingRuleList(fmt.Sprintf("%s-%s-%s", resourcePrefix, componentAuthMethod, secondaryDatacenter), &api.QueryOptions{Datacenter: primaryDatacenter}) + require.NoError(t, err) + require.NotNil(t, rb) + found = false + for x := range rb { + if rb[x].BindName == c.Roles[i] { + found = true + break + } + } + require.True(t, found) + } + }) + } +} + +// Test that server-acl-init used the local auth method to create the desired token in the primary datacenter. +// The test works by running the login command and then ensuring that the token +// returned has the correct role for the component. +func TestRun_NamespaceEnabled_ValidateLoginToken_PrimaryDatacenter(t *testing.T) { + t.Parallel() + + cases := []struct { + ComponentName string + TokenFlags []string + Roles []string + Namespace string + GlobalToken bool + }{} + for _, c := range cases { + t.Run(c.ComponentName, func(t *testing.T) { + authMethodName := fmt.Sprintf("%s-%s", resourcePrefix, componentAuthMethod) + serviceAccountName := fmt.Sprintf("%s-%s", resourcePrefix, c.ComponentName) + + k8s, testSvr := completeSetup(t) + defer testSvr.Stop() + _, jwtToken := setUpK8sServiceAccount(t, k8s, c.Namespace) + + k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if r != nil && r.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" && r.Method == "POST" { + w.Write([]byte(test.TokenReviewsResponse(serviceAccountName, c.Namespace))) + } + if r != nil && r.URL.Path == fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", c.Namespace, serviceAccountName) && + r.Method == "GET" { + w.Write([]byte(test.ServiceAccountGetResponse(serviceAccountName, c.Namespace))) + } + })) + t.Cleanup(k8sMockServer.Close) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-timeout=500ms", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + c.Namespace, + "-enable-namespaces", + "-consul-inject-destination-namespace", c.Namespace, + "-auth-method-host=" + k8sMockServer.URL, + "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], + "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + client, err := api.NewClient(&api.Config{ + Address: testSvr.HTTPAddr, + }) + require.NoError(t, err) + + tok, _, err := client.ACL().Login(&api.ACLLoginParams{ + AuthMethod: authMethodName, + BearerToken: jwtToken, + Meta: map[string]string{}, + }, &api.WriteOptions{}) + require.NoError(t, err) + + require.Equal(t, len(tok.Roles), len(c.Roles)) + for _, role := range tok.Roles { + require.Contains(t, c.Roles, role.Name) + } + require.Equal(t, !c.GlobalToken, tok.Local) + }) + } +} + +// Test that server-acl-init used the global auth method to create the desired token in the secondary datacenter. +// The test works by running the login command and then ensuring that the token +// returned has the correct role for the component. +func TestRun_NamespaceEnabled_ValidateLoginToken_SecondaryDatacenter(t *testing.T) { + t.Parallel() - return k8s, svr + cases := []struct { + ComponentName string + TokenFlags []string + Roles []string + Namespace string + GlobalToken bool + }{} + for _, c := range cases { + t.Run(c.ComponentName, func(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile := common.WriteTempFile(t, bootToken) + authMethodName := fmt.Sprintf("%s-%s-%s", resourcePrefix, componentAuthMethod, "dc2") + serviceAccountName := fmt.Sprintf("%s-%s", resourcePrefix, c.ComponentName) + + k8s, _, consulHTTPAddr, cleanup := mockReplicatedSetup(t, bootToken) + defer cleanup() + _, jwtToken := setUpK8sServiceAccount(t, k8s, c.Namespace) + + k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if r != nil && r.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" && r.Method == "POST" { + w.Write([]byte(test.TokenReviewsResponse(serviceAccountName, c.Namespace))) + } + if r != nil && r.URL.Path == fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", c.Namespace, serviceAccountName) && + r.Method == "GET" { + w.Write([]byte(test.ServiceAccountGetResponse(serviceAccountName, c.Namespace))) + } + })) + t.Cleanup(k8sMockServer.Close) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-federation", + "-timeout=1m", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + c.Namespace, + "-enable-namespaces", + "-consul-inject-destination-namespace", c.Namespace, + "-acl-replication-token-file", tokenFile, + "-auth-method-host=" + k8sMockServer.URL, + "-server-address", strings.Split(consulHTTPAddr, ":")[0], + "-server-port", strings.Split(consulHTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + client, err := api.NewClient(&api.Config{ + Address: consulHTTPAddr, + Datacenter: "dc1", + }) + require.NoError(t, err) + + tok, _, err := client.ACL().Login(&api.ACLLoginParams{ + AuthMethod: authMethodName, + BearerToken: jwtToken, + Meta: map[string]string{}, + }, &api.WriteOptions{}) + require.NoError(t, err) + + require.Equal(t, len(tok.Roles), len(c.Roles)) + for _, role := range tok.Roles { + require.Contains(t, c.Roles, role.Name) + } + require.Equal(t, !c.GlobalToken, tok.Local) + }) + } } // partitionedSetup is a helper function which creates a server and a consul agent that runs as diff --git a/control-plane/subcommand/server-acl-init/command_test.go b/control-plane/subcommand/server-acl-init/command_test.go index ac7785a75b..9091e12d1e 100644 --- a/control-plane/subcommand/server-acl-init/command_test.go +++ b/control-plane/subcommand/server-acl-init/command_test.go @@ -41,7 +41,7 @@ var ns = "default" var resourcePrefix = "release-name-consul" const ( - componentAuthMethod = "consul-k8s-component-auth-method" + componentAuthMethod = "k8s-component-auth-method" ) func TestRun_FlagValidation(t *testing.T) { @@ -109,7 +109,6 @@ func TestRun_Defaults(t *testing.T) { k8s, testSvr := completeSetup(t) defer testSvr.Stop() setUpK8sServiceAccount(t, k8s, ns) - require := require.New(t) // Run the command. ui := cli.NewMockUi() @@ -125,7 +124,7 @@ func TestRun_Defaults(t *testing.T) { "-resource-prefix=" + resourcePrefix, } responseCode := cmd.Run(args) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Test that the bootstrap kube secret is created. bootToken := getBootToken(t, k8s, resourcePrefix, ns) @@ -135,15 +134,15 @@ func TestRun_Defaults(t *testing.T) { Address: testSvr.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) tokenData, _, err := consul.ACL().TokenReadSelf(nil) - require.NoError(err) - require.Equal("global-management", tokenData.Policies[0].Name) + require.NoError(t, err) + require.Equal(t, "global-management", tokenData.Policies[0].Name) // Check that the agent policy was created. agentPolicy := policyExists(t, "agent-token", consul) // Should be a global policy. - require.Len(agentPolicy.Datacenters, 0) + require.Len(t, agentPolicy.Datacenters, 0) // We should also test that the server's token was updated, however I // couldn't find a way to test that with the test agent. Instead we test @@ -263,7 +262,6 @@ func TestRun_TokensPrimaryDC(t *testing.T) { k8s, testSvr := completeSetup(t) setUpK8sServiceAccount(t, k8s, ns) defer testSvr.Stop() - require := require.New(t) // Run the command. ui := cli.NewMockUi() @@ -281,7 +279,7 @@ func TestRun_TokensPrimaryDC(t *testing.T) { }, c.TokenFlags...) responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check that the expected policy was created. bootToken := getBootToken(t, k8s, resourcePrefix, ns) @@ -289,24 +287,24 @@ func TestRun_TokensPrimaryDC(t *testing.T) { Address: testSvr.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) for i := range c.PolicyNames { policy := policyExists(t, c.PolicyNames[i], consul) - require.Equal(c.PolicyDCs, policy.Datacenters) + require.Equal(t, c.PolicyDCs, policy.Datacenters) // Test that the token was created as a Kubernetes Secret. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), c.SecretNames[i], metav1.GetOptions{}) - require.NoError(err) - require.NotNil(tokenSecret) + require.NoError(t, err) + require.NotNil(t, tokenSecret) token, ok := tokenSecret.Data["token"] - require.True(ok) + require.True(t, ok) // Test that the token has the expected policies in Consul. tokenData, _, err := consul.ACL().TokenReadSelf(&api.QueryOptions{Token: string(token)}) - require.NoError(err) - require.Equal(c.PolicyNames[i], tokenData.Policies[0].Name) - require.Equal(c.LocalToken, tokenData.Local) + require.NoError(t, err) + require.Equal(t, c.PolicyNames[i], tokenData.Policies[0].Name) + require.Equal(t, c.LocalToken, tokenData.Local) } // Test that if the same command is run again, it doesn't error. @@ -318,7 +316,7 @@ func TestRun_TokensPrimaryDC(t *testing.T) { } cmd.init() responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) }) }) } @@ -329,12 +327,11 @@ func TestRun_ReplicationTokenPrimaryDC_WithProvidedSecretID(t *testing.T) { k8s, testSvr := completeSetup(t) defer testSvr.Stop() - require := require.New(t) setUpK8sServiceAccount(t, k8s, ns) replicationToken := "123e4567-e89b-12d3-a456-426614174000" replicationTokenFile, err := ioutil.TempFile("", "replicationtoken") - require.NoError(err) + require.NoError(t, err) defer os.Remove(replicationTokenFile.Name()) replicationTokenFile.WriteString(replicationToken) @@ -356,24 +353,24 @@ func TestRun_ReplicationTokenPrimaryDC_WithProvidedSecretID(t *testing.T) { } responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check that this token is created. consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, Token: replicationToken, }) - require.NoError(err) + require.NoError(t, err) token, _, err := consul.ACL().TokenReadSelf(nil) - require.NoError(err) + require.NoError(t, err) for _, policyLink := range token.Policies { policy := policyExists(t, policyLink.Name, consul) - require.Nil(policy.Datacenters) + require.Nil(t, policy.Datacenters) // Test that the token was not created as a Kubernetes Secret. _, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), resourcePrefix+"-acl-replication-acl-token", metav1.GetOptions{}) - require.True(k8serrors.IsNotFound(err)) + require.True(t, k8serrors.IsNotFound(err)) } // Test that if the same command is run again, it doesn't error. @@ -385,7 +382,7 @@ func TestRun_ReplicationTokenPrimaryDC_WithProvidedSecretID(t *testing.T) { } cmd.init() responseCode = cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) }) } @@ -832,17 +829,17 @@ func TestRun_ConnectInjectAuthMethod(t *testing.T) { flags: []string{"-create-inject-auth-method"}, expectedHost: "https://kubernetes.default.svc", }, - "-inject-auth-method-host flag (deprecated)": { + "-auth-method-host flag (deprecated)": { flags: []string{ "-create-inject-auth-method", - "-inject-auth-method-host=https://my-kube.com", + "-auth-method-host=https://my-kube.com", }, expectedHost: "https://my-kube.com", }, - "-inject-auth-method-host flag": { + "-auth-method-host flag": { flags: []string{ "-create-inject-token", - "-inject-auth-method-host=https://my-kube.com", + "-auth-method-host=https://my-kube.com", }, expectedHost: "https://my-kube.com", }, @@ -853,7 +850,6 @@ func TestRun_ConnectInjectAuthMethod(t *testing.T) { k8s, testSvr := completeSetup(t) defer testSvr.Stop() caCert, jwtToken := setUpK8sServiceAccount(t, k8s, ns) - require := require.New(t) // Run the command. ui := cli.NewMockUi() @@ -873,32 +869,32 @@ func TestRun_ConnectInjectAuthMethod(t *testing.T) { } cmdArgs = append(cmdArgs, c.flags...) responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check that the auth method was created. bootToken := getBootToken(t, k8s, resourcePrefix, ns) consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, }) - require.NoError(err) + require.NoError(t, err) authMethodName := resourcePrefix + "-k8s-auth-method" authMethod, _, err := consul.ACL().AuthMethodRead(authMethodName, &api.QueryOptions{Token: bootToken}) - require.NoError(err) - require.Contains(authMethod.Config, "Host") - require.Equal(authMethod.Config["Host"], c.expectedHost) - require.Contains(authMethod.Config, "CACert") - require.Equal(authMethod.Config["CACert"], caCert) - require.Contains(authMethod.Config, "ServiceAccountJWT") - require.Equal(authMethod.Config["ServiceAccountJWT"], jwtToken) + require.NoError(t, err) + require.Contains(t, authMethod.Config, "Host") + require.Equal(t, authMethod.Config["Host"], c.expectedHost) + require.Contains(t, authMethod.Config, "CACert") + require.Equal(t, authMethod.Config["CACert"], caCert) + require.Contains(t, authMethod.Config, "ServiceAccountJWT") + require.Equal(t, authMethod.Config["ServiceAccountJWT"], jwtToken) // Check that the binding rule was created. rules, _, err := consul.ACL().BindingRuleList(authMethodName, &api.QueryOptions{Token: bootToken}) - require.NoError(err) - require.Len(rules, 1) - require.Equal("service", string(rules[0].BindType)) - require.Equal("${serviceaccount.name}", rules[0].BindName) - require.Equal(bindingRuleSelector, rules[0].Selector) + require.NoError(t, err) + require.Len(t, rules, 1) + require.Equal(t, "service", string(rules[0].BindType)) + require.Equal(t, "${serviceaccount.name}", rules[0].BindName) + require.Equal(t, bindingRuleSelector, rules[0].Selector) // Test that if the same command is re-run it doesn't error. t.Run("retried", func(t *testing.T) { @@ -909,7 +905,7 @@ func TestRun_ConnectInjectAuthMethod(t *testing.T) { } cmd.init() responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) }) }) } @@ -928,7 +924,6 @@ func TestRun_ConnectInjectAuthMethodUpdates(t *testing.T) { k8s, testSvr := completeSetup(t) defer testSvr.Stop() caCert, jwtToken := setUpK8sServiceAccount(t, k8s, ns) - require := require.New(t) ui := cli.NewMockUi() cmd := Command{ @@ -948,29 +943,29 @@ func TestRun_ConnectInjectAuthMethodUpdates(t *testing.T) { flag, "-acl-binding-rule-selector=" + bindingRuleSelector, }) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check that the auth method was created. bootToken := getBootToken(t, k8s, resourcePrefix, ns) consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, }) - require.NoError(err) + require.NoError(t, err) authMethodName := resourcePrefix + "-k8s-auth-method" authMethod, _, err := consul.ACL().AuthMethodRead(authMethodName, &api.QueryOptions{Token: bootToken}) - require.NoError(err) - require.NotNil(authMethod) - require.Contains(authMethod.Config, "Host") - require.Equal(authMethod.Config["Host"], defaultKubernetesHost) - require.Contains(authMethod.Config, "CACert") - require.Equal(authMethod.Config["CACert"], caCert) - require.Contains(authMethod.Config, "ServiceAccountJWT") - require.Equal(authMethod.Config["ServiceAccountJWT"], jwtToken) + require.NoError(t, err) + require.NotNil(t, authMethod) + require.Contains(t, authMethod.Config, "Host") + require.Equal(t, authMethod.Config["Host"], defaultKubernetesHost) + require.Contains(t, authMethod.Config, "CACert") + require.Equal(t, authMethod.Config["CACert"], caCert) + require.Contains(t, authMethod.Config, "ServiceAccountJWT") + require.Equal(t, authMethod.Config["ServiceAccountJWT"], jwtToken) // Generate a new CA certificate _, _, caCertPem, _, err := cert.GenerateCA("kubernetes") - require.NoError(err) + require.NoError(t, err) // Overwrite the default kubernetes api, service account token and CA cert kubernetesHost := "https://kubernetes.example.com" @@ -990,21 +985,21 @@ func TestRun_ConnectInjectAuthMethodUpdates(t *testing.T) { "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], "-acl-binding-rule-selector=" + bindingRuleSelector, flag, - "-inject-auth-method-host=" + kubernetesHost, + "-auth-method-host=" + kubernetesHost, }) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check that the auth method has been updated authMethod, _, err = consul.ACL().AuthMethodRead(authMethodName, &api.QueryOptions{Token: bootToken}) - require.NoError(err) - require.NotNil(authMethod) - require.Contains(authMethod.Config, "Host") - require.Equal(authMethod.Config["Host"], kubernetesHost) - require.Contains(authMethod.Config, "CACert") - require.Equal(authMethod.Config["CACert"], updatedCACert) - require.Contains(authMethod.Config, "ServiceAccountJWT") - require.Equal(authMethod.Config["ServiceAccountJWT"], updatedJWTToken) + require.NoError(t, err) + require.NotNil(t, authMethod) + require.Contains(t, authMethod.Config, "Host") + require.Equal(t, authMethod.Config["Host"], kubernetesHost) + require.Contains(t, authMethod.Config, "CACert") + require.Equal(t, authMethod.Config["CACert"], updatedCACert) + require.Contains(t, authMethod.Config, "ServiceAccountJWT") + require.Equal(t, authMethod.Config["ServiceAccountJWT"], updatedJWTToken) }) } } @@ -1020,12 +1015,11 @@ func TestRun_BindingRuleUpdates(tt *testing.T) { k8s, testSvr := completeSetup(t) setUpK8sServiceAccount(t, k8s, ns) defer testSvr.Stop() - require := require.New(t) consul, err := api.NewClient(&api.Config{ Address: testSvr.HTTPAddr, }) - require.NoError(err) + require.NoError(t, err) ui := cli.NewMockUi() commonArgs := []string{ @@ -1049,22 +1043,22 @@ func TestRun_BindingRuleUpdates(tt *testing.T) { clientset: k8s, } responseCode := cmd.Run(firstRunArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Validate the binding rule. { queryOpts := &api.QueryOptions{Token: getBootToken(t, k8s, resourcePrefix, ns)} authMethodName := resourcePrefix + "-k8s-auth-method" rules, _, err := consul.ACL().BindingRuleList(authMethodName, queryOpts) - require.NoError(err) - require.Len(rules, 1) + require.NoError(t, err) + require.Len(t, rules, 1) actRule, _, err := consul.ACL().BindingRuleRead(rules[0].ID, queryOpts) - require.NoError(err) - require.NotNil(actRule) - require.Equal("Kubernetes binding rule", actRule.Description) - require.Equal(api.BindingRuleBindTypeService, actRule.BindType) - require.Equal("${serviceaccount.name}", actRule.BindName) - require.Equal("serviceaccount.name!=default", actRule.Selector) + require.NoError(t, err) + require.NotNil(t, actRule) + require.Equal(t, "Kubernetes binding rule", actRule.Description) + require.Equal(t, api.BindingRuleBindTypeService, actRule.BindType) + require.Equal(t, "${serviceaccount.name}", actRule.BindName) + require.Equal(t, "serviceaccount.name!=default", actRule.Selector) } // Re-run the command with namespace flags. The policies should be updated. @@ -1075,22 +1069,22 @@ func TestRun_BindingRuleUpdates(tt *testing.T) { clientset: k8s, } responseCode = cmd.Run(secondRunArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check the binding rule is changed expected. { queryOpts := &api.QueryOptions{Token: getBootToken(t, k8s, resourcePrefix, ns)} authMethodName := resourcePrefix + "-k8s-auth-method" rules, _, err := consul.ACL().BindingRuleList(authMethodName, queryOpts) - require.NoError(err) - require.Len(rules, 1) + require.NoError(t, err) + require.Len(t, rules, 1) actRule, _, err := consul.ACL().BindingRuleRead(rules[0].ID, queryOpts) - require.NoError(err) - require.NotNil(actRule) - require.Equal("Kubernetes binding rule", actRule.Description) - require.Equal(api.BindingRuleBindTypeService, actRule.BindType) - require.Equal("${serviceaccount.name}", actRule.BindName) - require.Equal("serviceaccount.name!=changed", actRule.Selector) + require.NoError(t, err) + require.NotNil(t, actRule) + require.Equal(t, "Kubernetes binding rule", actRule.Description) + require.Equal(t, api.BindingRuleBindTypeService, actRule.BindType) + require.Equal(t, "${serviceaccount.name}", actRule.BindName) + require.Equal(t, "serviceaccount.name!=changed", actRule.Selector) } }) } @@ -1102,7 +1096,6 @@ func TestRun_SyncPolicyUpdates(t *testing.T) { k8s, testSvr := completeSetup(t) defer testSvr.Stop() setUpK8sServiceAccount(t, k8s, ns) - require := require.New(t) ui := cli.NewMockUi() commonArgs := []string{ @@ -1126,7 +1119,7 @@ func TestRun_SyncPolicyUpdates(t *testing.T) { clientset: k8s, } responseCode := cmd.Run(firstRunArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Create consul client bootToken := getBootToken(t, k8s, resourcePrefix, ns) @@ -1134,19 +1127,19 @@ func TestRun_SyncPolicyUpdates(t *testing.T) { Address: testSvr.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) // Get and check the sync policy details firstPolicies, _, err := consul.ACL().PolicyList(nil) - require.NoError(err) + require.NoError(t, err) for _, p := range firstPolicies { if p.Name == "catalog-sync-token" { policy, _, err := consul.ACL().PolicyRead(p.ID, nil) - require.NoError(err) + require.NoError(t, err) // Check the node name in the policy - require.Contains(policy.Rules, "k8s-sync") + require.Contains(t, policy.Rules, "k8s-sync") } } @@ -1157,19 +1150,19 @@ func TestRun_SyncPolicyUpdates(t *testing.T) { clientset: k8s, } responseCode = cmd.Run(secondRunArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Get and check the sync policy details secondPolicies, _, err := consul.ACL().PolicyList(nil) - require.NoError(err) + require.NoError(t, err) for _, p := range secondPolicies { if p.Name == "catalog-sync-token" { policy, _, err := consul.ACL().PolicyRead(p.ID, nil) - require.NoError(err) + require.NoError(t, err) // Check the node name in the policy - require.Contains(policy.Rules, "new-node-name") + require.Contains(t, policy.Rules, "new-node-name") } } } @@ -1182,7 +1175,6 @@ func TestRun_SyncPolicyUpdates(t *testing.T) { // that we try to update will work for testing. func TestRun_ErrorsOnDuplicateACLPolicy(t *testing.T) { t.Parallel() - require := require.New(t) // Create Consul with ACLs already bootstrapped so that we can // then seed it with our manually created policy. @@ -1196,7 +1188,7 @@ func TestRun_ErrorsOnDuplicateACLPolicy(t *testing.T) { Address: testAgent.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) // Create the policy manually. description := "not the expected description" @@ -1204,7 +1196,7 @@ func TestRun_ErrorsOnDuplicateACLPolicy(t *testing.T) { Name: "catalog-sync-token", Description: description, }, nil) - require.NoError(err) + require.NoError(t, err) // Run the command. ui := cli.NewMockUi() @@ -1225,22 +1217,21 @@ func TestRun_ErrorsOnDuplicateACLPolicy(t *testing.T) { responseCode := cmd.Run(cmdArgs) // We expect the command to time out. - require.Equal(1, responseCode) + require.Equal(t, 1, responseCode) // NOTE: Since the error is logged through the logger instead of the UI // there's no good way to test that we logged the expected error however // we also test this directly in create_or_update_test.go. // Check that the policy wasn't modified. rereadPolicy, _, err := consul.ACL().PolicyRead(policy.ID, nil) - require.NoError(err) - require.Equal(description, rereadPolicy.Description) + require.NoError(t, err) + require.Equal(t, description, rereadPolicy.Description) } // Test that if the servers aren't available at first that bootstrap // still succeeds. func TestRun_DelayedServers(t *testing.T) { t.Parallel() - require := require.New(t) k8s := fake.NewSimpleClientset() setUpK8sServiceAccount(t, k8s, ns) randomPorts := freeport.GetN(t, 6) @@ -1288,7 +1279,7 @@ func TestRun_DelayedServers(t *testing.T) { Server: randomPorts[5], } }) - require.NoError(err) + require.NoError(t, err) close(testServerReady) }() @@ -1297,15 +1288,15 @@ func TestRun_DelayedServers(t *testing.T) { case <-testServerReady: defer srv.Stop() case <-time.After(5 * time.Second): - require.FailNow("test server took longer than 5s to come up") + require.FailNow(t, "test server took longer than 5s to come up") } // Wait for the command to exit. select { case <-done: - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) case <-time.After(5 * time.Second): - require.FailNow("command did not exit after 5s") + require.FailNow(t, "command did not exit after 5s") } // Test that the bootstrap kube secret is created. @@ -1316,10 +1307,10 @@ func TestRun_DelayedServers(t *testing.T) { Address: srv.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) tokenData, _, err := consul.ACL().TokenReadSelf(nil) - require.NoError(err) - require.Equal("global-management", tokenData.Policies[0].Name) + require.NoError(t, err) + require.Equal(t, "global-management", tokenData.Policies[0].Name) // Check that the agent policy was created. policyExists(t, "agent-token", consul) @@ -1328,7 +1319,6 @@ func TestRun_DelayedServers(t *testing.T) { // Test that if there's no leader, we retry until one is elected. func TestRun_NoLeader(t *testing.T) { t.Parallel() - require := require.New(t) k8s := fake.NewSimpleClientset() setUpK8sServiceAccount(t, k8s, ns) @@ -1369,7 +1359,7 @@ func TestRun_NoLeader(t *testing.T) { defer consulServer.Close() serverURL, err := url.Parse(consulServer.URL) - require.NoError(err) + require.NoError(t, err) // Run the command. ui := cli.NewMockUi() @@ -1393,16 +1383,16 @@ func TestRun_NoLeader(t *testing.T) { select { case <-done: - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) case <-time.After(5 * time.Second): - require.FailNow("command did not complete within 5s") + require.FailNow(t, "command did not complete within 5s") } // Test that the bootstrap kube secret is created. getBootToken(t, k8s, resourcePrefix, ns) // Test that the expected API calls were made. - require.Equal([]APICall{ + require.Equal(t, []APICall{ // Bootstrap will have been called 3 times. { "PUT", @@ -1551,7 +1541,6 @@ func TestConsulDatacenterList(t *testing.T) { // Test that if creating client tokens fails at first, we retry. func TestRun_ClientTokensRetry(t *testing.T) { t.Parallel() - require := require.New(t) k8s := fake.NewSimpleClientset() setUpK8sServiceAccount(t, k8s, ns) @@ -1592,7 +1581,7 @@ func TestRun_ClientTokensRetry(t *testing.T) { defer consulServer.Close() serverURL, err := url.Parse(consulServer.URL) - require.NoError(err) + require.NoError(t, err) // Run the command. ui := cli.NewMockUi() @@ -1607,10 +1596,10 @@ func TestRun_ClientTokensRetry(t *testing.T) { "-server-address=" + serverURL.Hostname(), "-server-port=" + serverURL.Port(), }) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Test that the expected API calls were made. - require.Equal([]APICall{ + require.Equal(t, []APICall{ { "PUT", "/v1/acl/bootstrap", @@ -1666,7 +1655,6 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { for name, tokenFromK8sSecret := range cases { t.Run(name, func(t *testing.T) { - require := require.New(t) k8s := fake.NewSimpleClientset() type APICall struct { @@ -1695,7 +1683,7 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { defer consulServer.Close() serverURL, err := url.Parse(consulServer.URL) - require.NoError(err) + require.NoError(t, err) setUpK8sServiceAccount(t, k8s, ns) cmdArgs := []string{ @@ -1720,17 +1708,17 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { }, }, metav1.CreateOptions{}) - require.NoError(err) + require.NoError(t, err) } else { // Write token to a file. bootTokenFile, err := ioutil.TempFile("", "") - require.NoError(err) + require.NoError(t, err) defer os.Remove(bootTokenFile.Name()) _, err = bootTokenFile.WriteString("old-token") - require.NoError(err) + require.NoError(t, err) - require.NoError(err) + require.NoError(t, err) cmdArgs = append(cmdArgs, "-bootstrap-token-file", bootTokenFile.Name()) } @@ -1742,18 +1730,18 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { } responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Test that the Secret is the same. if tokenFromK8sSecret { secret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), resourcePrefix+"-bootstrap-acl-token", metav1.GetOptions{}) - require.NoError(err) - require.Contains(secret.Data, "token") - require.Equal("old-token", string(secret.Data["token"])) + require.NoError(t, err) + require.Contains(t, secret.Data, "token") + require.Equal(t, "old-token", string(secret.Data["token"])) } // Test that the expected API calls were made. - require.Equal([]APICall{ + require.Equal(t, []APICall{ // We expect calls for updating the server policy, setting server tokens, // and updating client policy. { @@ -1804,7 +1792,6 @@ func TestRun_AlreadyBootstrapped_ServerTokenExists(t *testing.T) { for name, tokenInK8sSecret := range cases { t.Run(name, func(t *testing.T) { - require := require.New(t) // First set everything up with ACLs bootstrapped. bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" @@ -1829,17 +1816,17 @@ func TestRun_AlreadyBootstrapped_ServerTokenExists(t *testing.T) { "token": []byte(bootToken), }, }, metav1.CreateOptions{}) - require.NoError(err) + require.NoError(t, err) } else { // Write token to a file. bootTokenFile, err := ioutil.TempFile("", "") - require.NoError(err) + require.NoError(t, err) defer os.Remove(bootTokenFile.Name()) _, err = bootTokenFile.WriteString(bootToken) - require.NoError(err) + require.NoError(t, err) - require.NoError(err) + require.NoError(t, err) cmdArgs = append(cmdArgs, "-bootstrap-token-file", bootTokenFile.Name()) } @@ -1847,7 +1834,7 @@ func TestRun_AlreadyBootstrapped_ServerTokenExists(t *testing.T) { Address: testAgent.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) ui := cli.NewMockUi() cmd := Command{ UI: ui, @@ -1856,13 +1843,13 @@ func TestRun_AlreadyBootstrapped_ServerTokenExists(t *testing.T) { // Create the server policy and token _before_ we run the command. agentPolicyRules, err := cmd.agentRules() - require.NoError(err) + require.NoError(t, err) policy, _, err := consulClient.ACL().PolicyCreate(&api.ACLPolicy{ Name: "agent-token", Description: "Agent Token Policy", Rules: agentPolicyRules, }, nil) - require.NoError(err) + require.NoError(t, err) _, _, err = consulClient.ACL().TokenCreate(&api.ACLToken{ Description: fmt.Sprintf("Server Token for %s", strings.Split(testAgent.HTTPAddr, ":")[0]), Policies: []*api.ACLTokenPolicyLink{ @@ -1871,23 +1858,23 @@ func TestRun_AlreadyBootstrapped_ServerTokenExists(t *testing.T) { }, }, }, nil) - require.NoError(err) + require.NoError(t, err) // Run the command. responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Check that only one server token exists, i.e. it didn't create an // extra token. tokens, _, err := consulClient.ACL().TokenList(nil) - require.NoError(err) + require.NoError(t, err) count := 0 for _, token := range tokens { if len(token.Policies) == 1 && token.Policies[0].Name == policy.Name { count++ } } - require.Equal(1, count) + require.Equal(t, 1, count) }) } } @@ -1896,7 +1883,6 @@ func TestRun_AlreadyBootstrapped_ServerTokenExists(t *testing.T) { // and continue on to the next step. func TestRun_SkipBootstrapping_WhenServersAreDisabled(t *testing.T) { t.Parallel() - require := require.New(t) k8s := fake.NewSimpleClientset() bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" @@ -1928,7 +1914,7 @@ func TestRun_SkipBootstrapping_WhenServersAreDisabled(t *testing.T) { defer consulServer.Close() serverURL, err := url.Parse(consulServer.URL) - require.NoError(err) + require.NoError(t, err) // Run the command. ui := cli.NewMockUi() @@ -1947,11 +1933,11 @@ func TestRun_SkipBootstrapping_WhenServersAreDisabled(t *testing.T) { "-set-server-tokens=false", "-create-client-token=false", // disable client token, so there are fewer calls }) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Test that the expected API calls were made. // We expect not to see the call to /v1/acl/bootstrap. - require.Equal([]APICall{ + require.Equal(t, []APICall{ // We only expect the calls to get the datacenter { "GET", @@ -1967,7 +1953,6 @@ func TestRun_SkipBootstrapping_WhenServersAreDisabled(t *testing.T) { // Test that we exit after timeout. func TestRun_Timeout(t *testing.T) { t.Parallel() - require := require.New(t) k8s := fake.NewSimpleClientset() ui := cli.NewMockUi() cmd := Command{ @@ -1981,14 +1966,13 @@ func TestRun_Timeout(t *testing.T) { "-k8s-namespace=" + ns, "-server-address=foo", }) - require.Equal(1, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 1, responseCode, ui.ErrorWriter.String()) } // Test that the bootstrapping process can make calls to Consul API over HTTPS // when the consul agent is configured with HTTPS. func TestRun_HTTPS(t *testing.T) { t.Parallel() - require := require.New(t) k8s := fake.NewSimpleClientset() setUpK8sServiceAccount(t, k8s, ns) @@ -2001,7 +1985,7 @@ func TestRun_HTTPS(t *testing.T) { c.CertFile = certFile c.KeyFile = keyFile }) - require.NoError(err) + require.NoError(t, err) defer srv.Stop() // Run the command. @@ -2021,15 +2005,15 @@ func TestRun_HTTPS(t *testing.T) { "-server-address=" + strings.Split(srv.HTTPSAddr, ":")[0], "-server-port=" + strings.Split(srv.HTTPSAddr, ":")[1], }) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // Test that the bootstrap token is created to make sure the bootstrapping succeeded. // The presence of the bootstrap token tells us that the API calls to Consul have been successful. tokenSecret, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), resourcePrefix+"-bootstrap-acl-token", metav1.GetOptions{}) - require.NoError(err) - require.NotNil(tokenSecret) + require.NoError(t, err) + require.NotNil(t, tokenSecret) _, ok := tokenSecret.Data["token"] - require.True(ok) + require.True(t, ok) } // Test that the ACL replication token created from the primary DC can be used @@ -2136,7 +2120,6 @@ func TestRun_CloudAutoJoin(t *testing.T) { k8s, testSvr := completeSetup(t) defer testSvr.Stop() setUpK8sServiceAccount(t, k8s, ns) - require := require.New(t) // create a mock provider // that always returns the server address @@ -2161,7 +2144,7 @@ func TestRun_CloudAutoJoin(t *testing.T) { "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], } responseCode := cmd.Run(args) - require.Equal(0, responseCode, ui.ErrorWriter.String()) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) // check that the provider has been called provider.AssertNumberOfCalls(t, "Addrs", 1) @@ -2174,15 +2157,15 @@ func TestRun_CloudAutoJoin(t *testing.T) { Address: testSvr.HTTPAddr, Token: bootToken, }) - require.NoError(err) + require.NoError(t, err) tokenData, _, err := consul.ACL().TokenReadSelf(nil) - require.NoError(err) - require.Equal("global-management", tokenData.Policies[0].Name) + require.NoError(t, err) + require.Equal(t, "global-management", tokenData.Policies[0].Name) // Check that the agent policy was created. agentPolicy := policyExists(t, "agent-token", consul) // Should be a global policy. - require.Len(agentPolicy.Datacenters, 0) + require.Len(t, agentPolicy.Datacenters, 0) } func TestRun_GatewayErrors(t *testing.T) { @@ -2238,6 +2221,431 @@ func TestRun_GatewayErrors(t *testing.T) { } } +// Test creating the correct ACL policies and Binding Rules for components in the primary datacenter. +// The test works by running the command and then ensuring that: +// * An ACLBindingRule exists which references the ACLRole. +// * An ACLRole exists and has the correct PolicyName in it's ACLPolicyLinkRule list. +// * The ACLPolicy exists. +func TestRun_PoliciesAndBindingRulesForACLLogin_PrimaryDatacenter(t *testing.T) { + t.Parallel() + + cases := []struct { + TestName string + TokenFlags []string + PolicyNames []string + Roles []string + }{ + { + TestName: "Controller", + TokenFlags: []string{"-create-controller-token"}, + PolicyNames: []string{"controller-policy"}, + Roles: []string{resourcePrefix + "-controller-acl-role"}, + }, + } + for _, c := range cases { + t.Run(c.TestName, func(t *testing.T) { + k8s, testSvr := completeSetup(t) + defer testSvr.Stop() + setUpK8sServiceAccount(t, k8s, ns) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-timeout=500ms", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + ns, + "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], + "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + bootToken := getBootToken(t, k8s, resourcePrefix, ns) + consul, err := api.NewClient(&api.Config{ + Address: testSvr.HTTPAddr, + Token: bootToken, + }) + require.NoError(t, err) + + // Check that the Role exists + has correct Policy and is associated with a BindingRule. + for i := range c.Roles { + // Check that the Policy exists. + policy, _, err := consul.ACL().PolicyReadByName(c.PolicyNames[i], &api.QueryOptions{}) + require.NoError(t, err) + require.NotNil(t, policy) + + // Check that the Role exists. + role, _, err := consul.ACL().RoleReadByName(c.Roles[i], &api.QueryOptions{}) + require.NoError(t, err) + require.NotNil(t, role) + + // Check that the Role references the Policy. + found := false + for x := range role.Policies { + if role.Policies[x].Name == policy.Name { + found = true + break + } + } + require.True(t, found) + + // Check that there exists a BindingRule that references this Role. + rb, _, err := consul.ACL().BindingRuleList(fmt.Sprintf("%s-%s", resourcePrefix, componentAuthMethod), &api.QueryOptions{}) + require.NoError(t, err) + require.NotNil(t, rb) + found = false + for x := range rb { + if rb[x].BindName == c.Roles[i] { + found = true + break + } + } + require.True(t, found) + } + }) + } +} + +// Test creating the correct ACL policies and Binding Rules for components in the secondary datacenter. +// This tests specifically tests that policies and roles for global tokens are correctly created. +// The test works by running the command and then ensuring that: +// * An ACLBindingRule exists which references the ACLRole. +// * An ACLRole exists and has the correct PolicyName in it's ACLPolicyLinkRule list. +// * The ACLPolicy exists. +func TestRun_PoliciesAndBindingRulesACLLogin_SecondaryDatacenter(t *testing.T) { + t.Parallel() + + const ( + secondaryDatacenter = "dc2" + primaryDatacenter = "dc1" + ) + cases := []struct { + TestName string + TokenFlags []string + PolicyNames []string + Roles []string + }{ + { + TestName: "Controller", + TokenFlags: []string{"-create-controller-token"}, + PolicyNames: []string{"controller-policy-" + secondaryDatacenter}, + Roles: []string{resourcePrefix + "-controller-acl-role-" + secondaryDatacenter}, + }, + } + for _, c := range cases { + t.Run(c.TestName, func(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile := common.WriteTempFile(t, bootToken) + k8s, consul, consulHTTPAddr, cleanup := mockReplicatedSetup(t, bootToken) + setUpK8sServiceAccount(t, k8s, ns) + defer cleanup() + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-federation", + "-timeout=1m", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + ns, + "-auth-method-host=" + "https://my-kube.com", + "-acl-replication-token-file", tokenFile, + "-server-address", strings.Split(consulHTTPAddr, ":")[0], + "-server-port", strings.Split(consulHTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // Check that the Role exists + has correct Policy and is associated with a BindingRule. + for i := range c.Roles { + // Check that the Policy exists. + + policy, _, err := consul.ACL().PolicyReadByName(c.PolicyNames[i], &api.QueryOptions{Datacenter: primaryDatacenter}) + require.NoError(t, err) + require.NotNil(t, policy) + + // Check that the Role exists. + role, _, err := consul.ACL().RoleReadByName(c.Roles[i], &api.QueryOptions{Datacenter: primaryDatacenter}) + require.NoError(t, err) + require.NotNil(t, role) + + // Check that the Role references the Policy. + found := false + for x := range role.Policies { + if role.Policies[x].Name == policy.Name { + found = true + break + } + } + require.True(t, found) + + // Check that there exists a BindingRule that references this Role. + rb, _, err := consul.ACL().BindingRuleList(fmt.Sprintf("%s-%s-%s", resourcePrefix, componentAuthMethod, secondaryDatacenter), &api.QueryOptions{Datacenter: primaryDatacenter}) + require.NoError(t, err) + require.NotNil(t, rb) + found = false + for x := range rb { + if rb[x].BindName == c.Roles[i] { + found = true + break + } + } + require.True(t, found) + } + }) + } +} + +// Test that server-acl-init in the primary datacenter creates the desired token. +// The test works by running the login command and then ensuring that the token +// returned has the correct role for the component. +func TestRun_ValidateLoginToken_PrimaryDatacenter(t *testing.T) { + t.Parallel() + + cases := []struct { + ComponentName string + TokenFlags []string + Roles []string + GlobalToken bool + }{ + { + ComponentName: "controller", + TokenFlags: []string{"-create-controller-token"}, + Roles: []string{resourcePrefix + "-controller-acl-role"}, + GlobalToken: false, + }, + } + for _, c := range cases { + t.Run(c.ComponentName, func(t *testing.T) { + authMethodName := fmt.Sprintf("%s-%s", resourcePrefix, componentAuthMethod) + serviceAccountName := fmt.Sprintf("%s-%s", resourcePrefix, c.ComponentName) + + k8s, testSvr := completeSetup(t) + defer testSvr.Stop() + _, jwtToken := setUpK8sServiceAccount(t, k8s, ns) + + k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if r != nil && r.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" && r.Method == "POST" { + w.Write([]byte(test.TokenReviewsResponse(serviceAccountName, ns))) + } + if r != nil && r.URL.Path == fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", ns, serviceAccountName) && + r.Method == "GET" { + w.Write([]byte(test.ServiceAccountGetResponse(serviceAccountName, ns))) + } + })) + t.Cleanup(k8sMockServer.Close) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-timeout=500ms", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + ns, + "-auth-method-host=" + k8sMockServer.URL, + "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], + "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + client, err := api.NewClient(&api.Config{ + Address: testSvr.HTTPAddr, + }) + require.NoError(t, err) + + tok, _, err := client.ACL().Login(&api.ACLLoginParams{ + AuthMethod: authMethodName, + BearerToken: jwtToken, + Meta: map[string]string{}, + }, &api.WriteOptions{}) + require.NoError(t, err) + + require.Equal(t, len(tok.Roles), len(c.Roles)) + for _, role := range tok.Roles { + require.Contains(t, c.Roles, role.Name) + } + require.Equal(t, !c.GlobalToken, tok.Local) + }) + } +} + +// Test that server-acl-init in the secondary datacenter creates the desired token. +// The test works by running the login command and then ensuring that the token +// returned has the correct role for the component. +func TestRun_ValidateLoginToken_SecondaryDatacenter(t *testing.T) { + t.Parallel() + + cases := []struct { + ComponentName string + TokenFlags []string + Roles []string + GlobalToken bool + }{ + { + ComponentName: "controller", + TokenFlags: []string{"-create-controller-token"}, + Roles: []string{resourcePrefix + "-controller-acl-role-dc2"}, + GlobalToken: true, + }, + } + for _, c := range cases { + t.Run(c.ComponentName, func(t *testing.T) { + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile := common.WriteTempFile(t, bootToken) + authMethodName := fmt.Sprintf("%s-%s-%s", resourcePrefix, componentAuthMethod, "dc2") + serviceAccountName := fmt.Sprintf("%s-%s", resourcePrefix, c.ComponentName) + + k8s, _, consulHTTPAddr, cleanup := mockReplicatedSetup(t, bootToken) + defer cleanup() + _, jwtToken := setUpK8sServiceAccount(t, k8s, ns) + + k8sMockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if r != nil && r.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" && r.Method == "POST" { + w.Write([]byte(test.TokenReviewsResponse(serviceAccountName, ns))) + } + if r != nil && r.URL.Path == fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s", ns, serviceAccountName) && + r.Method == "GET" { + w.Write([]byte(test.ServiceAccountGetResponse(serviceAccountName, ns))) + } + })) + t.Cleanup(k8sMockServer.Close) + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmdArgs := append([]string{ + "-federation", + "-timeout=1m", + "-resource-prefix=" + resourcePrefix, + "-k8s-namespace=" + ns, + "-acl-replication-token-file", tokenFile, + "-auth-method-host=" + k8sMockServer.URL, + "-server-address", strings.Split(consulHTTPAddr, ":")[0], + "-server-port", strings.Split(consulHTTPAddr, ":")[1], + }, c.TokenFlags...) + cmd.init() + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + client, err := api.NewClient(&api.Config{ + Address: consulHTTPAddr, + Datacenter: "dc1", + }) + require.NoError(t, err) + + tok, _, err := client.ACL().Login(&api.ACLLoginParams{ + AuthMethod: authMethodName, + BearerToken: jwtToken, + Meta: map[string]string{}, + }, &api.WriteOptions{}) + require.NoError(t, err) + + require.Equal(t, len(tok.Roles), len(c.Roles)) + for _, role := range tok.Roles { + require.Contains(t, c.Roles, role.Name) + } + require.Equal(t, !c.GlobalToken, tok.Local) + }) + } +} + +// Test that the component auth method gets created. +func TestRun_PrimaryDatacenter_ComponentAuthMethod(t *testing.T) { + t.Parallel() + + k8s, testSvr := completeSetup(t) + setUpK8sServiceAccount(t, k8s, ns) + defer testSvr.Stop() + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + cmdArgs := []string{ + "-timeout=1m", + "-k8s-namespace=" + ns, + "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], + "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], + "-resource-prefix=" + resourcePrefix} + + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // Check that the expected policy was created. + bootToken := getBootToken(t, k8s, resourcePrefix, ns) + consulClient, err := api.NewClient(&api.Config{ + Address: testSvr.HTTPAddr, + Token: bootToken, + }) + require.NoError(t, err) + authMethod, _, err := consulClient.ACL().AuthMethodRead(resourcePrefix+"-k8s-component-auth-method", &api.QueryOptions{}) + require.NoError(t, err) + require.NotNil(t, authMethod) +} + +// Test that the local and global component auth methods gets created when run in the +// secondary datacenter. +func TestRun_SecondaryDatacenter_ComponentAuthMethod(t *testing.T) { + t.Parallel() + + bootToken := "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + tokenFile := common.WriteTempFile(t, bootToken) + k8s, consul, consulHTTPAddr, cleanup := mockReplicatedSetup(t, bootToken) + setUpK8sServiceAccount(t, k8s, ns) + defer cleanup() + + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + cmdArgs := []string{ + "-federation", + "-timeout=1m", + "-k8s-namespace=" + ns, + "-auth-method-host=" + "https://my-kube.com", + "-acl-replication-token-file", tokenFile, + "-server-address", strings.Split(consulHTTPAddr, ":")[0], + "-server-port", strings.Split(consulHTTPAddr, ":")[1], + "-resource-prefix=" + resourcePrefix} + + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // Check that the expected local auth-method was created. + localAuthMethod, _, err := consul.ACL().AuthMethodRead(resourcePrefix+"-k8s-component-auth-method", &api.QueryOptions{}) + require.NoError(t, err) + require.NotNil(t, localAuthMethod) + // Check that the expected global auth-method was created. + globalAuthMethod, _, err := consul.ACL().AuthMethodRead(resourcePrefix+"-k8s-component-auth-method-dc2", &api.QueryOptions{Datacenter: "dc1"}) + require.NoError(t, err) + require.NotNil(t, globalAuthMethod) +} + // Set up test consul agent and kubernetes cluster. func completeSetup(t *testing.T) (*fake.Clientset, *testutil.TestServer) { k8s := fake.NewSimpleClientset() @@ -2398,134 +2806,6 @@ func replicatedSetup(t *testing.T, bootToken string) (*fake.Clientset, *api.Clie } } -// Test creating the correct ACL policies and Binding Rules for components that use the auth method. -// The test works by running the command and then ensuring that: -// * An ACLBindingRule exists which references the ACLRole. -// * An ACLRole exists and has the correct PolicyName in it's ACLPolicyLinkRule list. -// * The ACLPolicy exists. -func TestRun_PoliciesAndBindingRulesForACLLogin(t *testing.T) { - t.Parallel() - - cases := []struct { - TestName string - TokenFlags []string - PolicyNames []string - Roles []string - }{ - { - TestName: "Controller", - TokenFlags: []string{"-create-controller-token"}, - PolicyNames: []string{"controller-policy"}, - Roles: []string{resourcePrefix + "-controller-acl-role"}, - }, - } - for _, c := range cases { - t.Run(c.TestName, func(t *testing.T) { - k8s, testSvr := completeSetup(t) - defer testSvr.Stop() - setUpK8sServiceAccount(t, k8s, ns) - - // Run the command. - ui := cli.NewMockUi() - cmd := Command{ - UI: ui, - clientset: k8s, - } - cmdArgs := append([]string{ - "-timeout=500ms", - "-resource-prefix=" + resourcePrefix, - "-k8s-namespace=" + ns, - "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], - "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], - }, c.TokenFlags...) - cmd.init() - responseCode := cmd.Run(cmdArgs) - require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) - - bootToken := getBootToken(t, k8s, resourcePrefix, ns) - consul, err := api.NewClient(&api.Config{ - Address: testSvr.HTTPAddr, - Token: bootToken, - }) - require.NoError(t, err) - - // Check that the Role exists + has correct Policy and is associated with a BindingRule. - for i := range c.Roles { - // Check that the Policy exists. - policy, _, err := consul.ACL().PolicyReadByName(c.PolicyNames[i], &api.QueryOptions{}) - require.NoError(t, err) - require.NotNil(t, policy) - - // Check that the Role exists. - role, _, err := consul.ACL().RoleReadByName(c.Roles[i], &api.QueryOptions{}) - require.NoError(t, err) - require.NotNil(t, role) - - // Check that the Role references the Policy. - found := false - for x := range role.Policies { - if role.Policies[x].Name == policy.Name { - found = true - break - } - } - require.True(t, found) - - // Check that there exists a BindingRule that references this Role. - rb, _, err := consul.ACL().BindingRuleList("release-name-"+componentAuthMethod, &api.QueryOptions{}) - require.NoError(t, err) - require.NotNil(t, rb) - found = false - for x := range rb { - if rb[x].BindName == c.Roles[i] { - found = true - break - } - } - require.True(t, found) - } - }) - } -} - -// Test that the component auth method gets created. -func TestRun_ComponentAuthMethod(t *testing.T) { - t.Parallel() - - k8s, testSvr := completeSetup(t) - setUpK8sServiceAccount(t, k8s, ns) - defer testSvr.Stop() - require := require.New(t) - - // Run the command. - ui := cli.NewMockUi() - cmd := Command{ - UI: ui, - clientset: k8s, - } - cmd.init() - cmdArgs := []string{ - "-timeout=1m", - "-k8s-namespace=" + ns, - "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], - "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], - "-resource-prefix=" + resourcePrefix} - - responseCode := cmd.Run(cmdArgs) - require.Equal(0, responseCode, ui.ErrorWriter.String()) - - // Check that the expected policy was created. - bootToken := getBootToken(t, k8s, resourcePrefix, ns) - consulClient, err := api.NewClient(&api.Config{ - Address: testSvr.HTTPAddr, - Token: bootToken, - }) - require.NoError(err) - authMethod, _, err := consulClient.ACL().AuthMethodRead(resourcePrefix+"-k8s-component-auth-method", &api.QueryOptions{}) - require.NoError(err) - require.NotNil(authMethod) -} - // getBootToken gets the bootstrap token from the Kubernetes secret. It will // cause a test failure if the Secret doesn't exist or is malformed. func getBootToken(t *testing.T, k8s *fake.Clientset, prefix string, k8sNamespace string) string { diff --git a/control-plane/subcommand/server-acl-init/connect_inject.go b/control-plane/subcommand/server-acl-init/connect_inject.go index f8fc11b4fc..8869fcaef3 100644 --- a/control-plane/subcommand/server-acl-init/connect_inject.go +++ b/control-plane/subcommand/server-acl-init/connect_inject.go @@ -2,6 +2,7 @@ package serveraclinit import ( "fmt" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" "github.com/hashicorp/consul/api" apiv1 "k8s.io/api/core/v1" @@ -124,8 +125,8 @@ func (c *Command) createAuthMethodTmpl(authMethodName string, useNS bool) (api.A kubernetesHost := defaultKubernetesHost // Check if custom auth method Host and CACert are provided - if c.flagInjectAuthMethodHost != "" { - kubernetesHost = c.flagInjectAuthMethodHost + if c.flagAuthMethodHost != "" { + kubernetesHost = c.flagAuthMethodHost } // Now we're ready to set up Consul's auth method. diff --git a/control-plane/subcommand/server-acl-init/create_or_update.go b/control-plane/subcommand/server-acl-init/create_or_update.go index d7d0cb991a..954b1e83c6 100644 --- a/control-plane/subcommand/server-acl-init/create_or_update.go +++ b/control-plane/subcommand/server-acl-init/create_or_update.go @@ -11,12 +11,63 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// addRoleAndBindingRule adds an ACLRole and ACLBindingRule which reference the authMethod. -func (c *Command) addRoleAndBindingRule(client *api.Client, serviceAccountName string, authMethodName string, policies []*api.ACLRolePolicyLink) error { +// createACLPolicyRoleAndBindingRule will create the ACL Policy for the component +// then create a set of ACLRole and ACLBindingRule which tie the component's serviceaccount +// to the authMethod, allowing the serviceaccount to later be allowed to issue a Consul Login. +func (c *Command) createACLPolicyRoleAndBindingRule(componentName, rules, dc, primaryDC string, global, primary bool, authMethodName, serviceAccountName string, client *api.Client) error { + // Create policy with the given rules. + policyName := fmt.Sprintf("%s-policy", componentName) + if c.flagFederation && !primary { + // If performing ACL replication, we must ensure policy names are + // globally unique so we append the datacenter name but only in secondary datacenters.. + policyName += fmt.Sprintf("-%s", dc) + } + var datacenters []string + if global { + datacenters = append(datacenters, dc) + if !primary { + datacenters = append(datacenters, primaryDC) + } + } + policyTmpl := api.ACLPolicy{ + Name: policyName, + Description: fmt.Sprintf("%s Token Policy", policyName), + Rules: rules, + Datacenters: datacenters, + } + err := c.untilSucceeds(fmt.Sprintf("creating %s policy", policyTmpl.Name), + func() error { + return c.createOrUpdateACLPolicy(policyTmpl, client) + }) + if err != nil { + return err + } + + // Create an ACLRolePolicyLink list to attach to the ACLRole. + ap := &api.ACLRolePolicyLink{ + Name: policyName, + } + apl := []*api.ACLRolePolicyLink{} + apl = append(apl, ap) + // Add the ACLRole and ACLBindingRule. + err = c.addRoleAndBindingRule(client, serviceAccountName, authMethodName, apl, global, primary, primaryDC, dc) + if err != nil { + return err + } + return err +} + +// addRoleAndBindingRule adds an ACLRole and ACLBindingRule which reference the authMethod. +func (c *Command) addRoleAndBindingRule(client *api.Client, serviceAccountName string, authMethodName string, policies []*api.ACLRolePolicyLink, global, primary bool, primaryDC, dc string) error { // This is the ACLRole which will allow the component which uses the serviceaccount // to be able to do a consul login. aclRoleName := fmt.Sprintf("%s-acl-role", serviceAccountName) + if c.flagFederation && !primary { + // If performing ACL replication, we must ensure policy names are + // globally unique so we append the datacenter name but only in secondary datacenters. + aclRoleName += fmt.Sprintf("-%s", dc) + } role := &api.ACLRole{ Name: aclRoleName, Description: fmt.Sprintf("ACL Role for %s", serviceAccountName), @@ -36,7 +87,15 @@ func (c *Command) addRoleAndBindingRule(client *api.Client, serviceAccountName s BindType: api.BindingRuleBindTypeRole, BindName: aclRoleName, } - return c.updateOrCreateBindingRule(client, authMethodName, &abr, true) + writeOptions := &api.WriteOptions{} + if global && dc != primaryDC { + writeOptions.Datacenter = primaryDC + } + return c.untilSucceeds(fmt.Sprintf("creating acl binding rule for %s", authMethodName), + func() error { + _, _, err := client.ACL().BindingRuleCreate(&abr, writeOptions) + return err + }) } // updateOrCreateACLRole will query to see if existing role is in place and update them @@ -125,54 +184,6 @@ func (c *Command) updateOrCreateBindingRule(client *api.Client, authMethodName s _, _, err := client.ACL().BindingRuleCreate(abr, nil) return err }) - - } - return err -} - -// createACLPolicyRoleAndBindingRule will create the ACL Policy for the component -// then create a set of ACLRole and ACLBindingRule which tie the component's serviceaccount -// to the authMethod, allowing the serviceaccount to later be allowed to issue a Consul Login. -func (c *Command) createACLPolicyRoleAndBindingRule(componentName string, rules string, dc string, isPrimary bool, - authMethodName string, serviceAccountName string, client *api.Client) error { - // Create policy with the given rules. - policyName := fmt.Sprintf("%s-policy", componentName) - if c.flagFederation && !isPrimary { - // If performing ACL replication, we must ensure policy names are - // globally unique so we append the datacenter name but only in secondary datacenters.. - policyName += fmt.Sprintf("-%s", dc) - } - var datacenters []string - // TODO: when we support global auth method logins we will need to append the list of dcs. - //if !globalToken && dc != "" { - if dc != "" { - datacenters = append(datacenters, dc) - } - policyTmpl := api.ACLPolicy{ - Name: policyName, - Description: fmt.Sprintf("%s Token Policy", policyName), - Rules: rules, - Datacenters: datacenters, - } - err := c.untilSucceeds(fmt.Sprintf("creating %s policy", policyTmpl.Name), - func() error { - return c.createOrUpdateACLPolicy(policyTmpl, client) - }) - if err != nil { - return err - } - - // Create an ACLRolePolicyLink list to attach to the ACLRole. - ap := &api.ACLRolePolicyLink{ - Name: policyName, - } - apl := []*api.ACLRolePolicyLink{} - apl = append(apl, ap) - - // Add the ACLRole and ACLBindingRule. - err = c.addRoleAndBindingRule(client, serviceAccountName, authMethodName, apl) - if err != nil { - return err } return err } From b84816a98be0032f3723150a5d1fecf40be2e2d6 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Mon, 7 Mar 2022 18:38:08 -0500 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Kyle Schochenmaier --- control-plane/subcommand/server-acl-init/command_ent_test.go | 1 - control-plane/subcommand/server-acl-init/command_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/control-plane/subcommand/server-acl-init/command_ent_test.go b/control-plane/subcommand/server-acl-init/command_ent_test.go index cfbfeffa8c..3dd8c675b2 100644 --- a/control-plane/subcommand/server-acl-init/command_ent_test.go +++ b/control-plane/subcommand/server-acl-init/command_ent_test.go @@ -1234,7 +1234,6 @@ func TestRun_SecondaryDatacenter_PoliciesAndBindingRulesForACLLogin_NamespacesEn // Check that the Role exists + has correct Policy and is associated with a BindingRule. for i := range c.Roles { // Check that the Policy exists. - policy, _, err := consul.ACL().PolicyReadByName(c.PolicyNames[i], &api.QueryOptions{Datacenter: primaryDatacenter}) require.NoError(t, err) require.NotNil(t, policy) diff --git a/control-plane/subcommand/server-acl-init/command_test.go b/control-plane/subcommand/server-acl-init/command_test.go index 9091e12d1e..76671b49fa 100644 --- a/control-plane/subcommand/server-acl-init/command_test.go +++ b/control-plane/subcommand/server-acl-init/command_test.go @@ -2368,7 +2368,6 @@ func TestRun_PoliciesAndBindingRulesACLLogin_SecondaryDatacenter(t *testing.T) { // Check that the Role exists + has correct Policy and is associated with a BindingRule. for i := range c.Roles { // Check that the Policy exists. - policy, _, err := consul.ACL().PolicyReadByName(c.PolicyNames[i], &api.QueryOptions{Datacenter: primaryDatacenter}) require.NoError(t, err) require.NotNil(t, policy)