diff --git a/acceptance/tests/vault/helpers.go b/acceptance/tests/vault/helpers.go index fe27dc6aae..fc7f25ad21 100644 --- a/acceptance/tests/vault/helpers.go +++ b/acceptance/tests/vault/helpers.go @@ -19,18 +19,13 @@ path "consul/data/secret/gossip" { capabilities = ["read"] }` - bootstrapTokenPolicy = ` -path "consul/data/secret/bootstrap" { - capabilities = ["read"] -}` - - replicationTokenPolicy = ` -path "consul/data/secret/replication" { + tokenPolicyTemplate = ` +path "consul/data/secret/%s" { capabilities = ["read"] }` enterpriseLicensePolicy = ` -path "consul/data/secret/enterpriselicense" { +path "consul/data/secret/license" { capabilities = ["read"] }` @@ -85,7 +80,7 @@ func generateGossipSecret() (string, error) { func configureGossipVaultSecret(t *testing.T, vaultClient *vapi.Client) string { // Create the Vault Policy for the gossip key. logger.Log(t, "Creating gossip policy") - err := vaultClient.Sys().PutPolicy("consul-gossip", gossipPolicy) + err := vaultClient.Sys().PutPolicy("gossip", gossipPolicy) require.NoError(t, err) // Generate the gossip secret. @@ -111,88 +106,49 @@ func configureEnterpriseLicenseVaultSecret(t *testing.T, vaultClient *vapi.Clien logger.Log(t, "Creating the Enterprise License secret") params := map[string]interface{}{ "data": map[string]interface{}{ - "enterpriselicense": cfg.EnterpriseLicense, + "license": cfg.EnterpriseLicense, }, } - _, err := vaultClient.Logical().Write("consul/data/secret/enterpriselicense", params) + _, err := vaultClient.Logical().Write("consul/data/secret/license", params) require.NoError(t, err) - // Create the Vault Policy for the consul-enterpriselicense. - err = vaultClient.Sys().PutPolicy("consul-enterpriselicense", enterpriseLicensePolicy) + err = vaultClient.Sys().PutPolicy("license", enterpriseLicensePolicy) require.NoError(t, err) } -// configureKubernetesAuthRoles configures roles for the Kubernetes auth method +// configureKubernetesAuthRole configures a role for the component for the Kubernetes auth method // that will be used by the test Helm chart installation. -func configureKubernetesAuthRoles( - t *testing.T, - vaultClient *vapi.Client, - consulReleaseName, ns, authPath, datacenter string, - cfg *config.TestConfig, - isPrimary bool, -) { - consulClientServiceAccountName := fmt.Sprintf("%s-consul-client", consulReleaseName) - consulServerServiceAccountName := fmt.Sprintf("%s-consul-server", consulReleaseName) - sharedPolicies := "consul-gossip" - if cfg.EnableEnterprise { - sharedPolicies += ",consul-enterpriselicense" - } +func configureKubernetesAuthRole(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns, authPath, component, policies string) { + componentServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, component) - // Create the Auth Roles for consul-server and consul-client. + // Create the Auth Roles for the component. // Auth roles bind policies to Kubernetes service accounts, which // then enables the Vault agent init container to call 'vault login' // with the Kubernetes auth method to obtain a Vault token. // Please see https://www.vaultproject.io/docs/auth/kubernetes#configuration // for more details. - logger.Log(t, "Creating the consul-server and consul-client roles") + logger.Logf(t, "Creating the %q", componentServiceAccountName) params := map[string]interface{}{ - "bound_service_account_names": consulClientServiceAccountName, - "bound_service_account_namespaces": ns, - "policies": sharedPolicies, - "ttl": "24h", - } - _, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-client", authPath), params) - require.NoError(t, err) - - // Both primary and secondary datacenters need access to the replication token, but - // only the primary needs to be able to read the bootstrap token. - policies := fmt.Sprintf(sharedPolicies+",connect-ca-%s,consul-server-%s,consul-replication-token", datacenter, datacenter) - if isPrimary { - policies += ",consul-bootstrap-token" - } - params = map[string]interface{}{ - "bound_service_account_names": consulServerServiceAccountName, + "bound_service_account_names": componentServiceAccountName, "bound_service_account_namespaces": ns, "policies": policies, "ttl": "24h", } - _, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params) + _, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/%s", authPath, component), params) require.NoError(t, err) +} +// configureKubernetesAuthRole configures a role that allows all service accounts within the installation +// namespace access to the Consul server CA. +func configureConsulCAKubernetesAuthRole(t *testing.T, vaultClient *vapi.Client, ns, authPath string) { // Create the CA role that all components will use to fetch the Server CA certs. - params = map[string]interface{}{ + params := map[string]interface{}{ "bound_service_account_names": "*", "bound_service_account_namespaces": ns, "policies": "consul-ca", "ttl": "24h", } - _, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-ca", authPath), params) - require.NoError(t, err) - - logger.Log(t, "Creating kubernetes auth role for the server-acl-init job") - policies = "consul-replication-token" - if isPrimary { - policies += ",consul-bootstrap-token" - } - serverACLInitSAName := fmt.Sprintf("%s-consul-server-acl-init", consulReleaseName) - params = map[string]interface{}{ - "bound_service_account_names": serverACLInitSAName, - "bound_service_account_namespaces": ns, - "policies": policies, - "ttl": "24h", - } - - _, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/server-acl-init", authPath), params) + _, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-ca", authPath), params) require.NoError(t, err) } @@ -227,7 +183,7 @@ func configurePKICertificates(t *testing.T, vaultClient *vapi.Client, consulRele "max_ttl": "1h", } - pkiRoleName := fmt.Sprintf("consul-server-%s", datacenter) + pkiRoleName := fmt.Sprintf("server-cert-%s", datacenter) _, err := vaultClient.Logical().Write(fmt.Sprintf("pki/roles/%s", pkiRoleName), params) require.NoError(t, err) @@ -245,37 +201,14 @@ path %q { return certificateIssuePath } -// configureReplicationTokenVaultSecret generates a replication token secret ID, -// stores it in vault as a secret and configures a policy to access it. -func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client) string { - // Create the Vault Policy for the replication token. - logger.Log(t, "Creating replication token policy") - err := vaultClient.Sys().PutPolicy("consul-replication-token", replicationTokenPolicy) - require.NoError(t, err) - - // Generate the token secret. - token, err := uuid.GenerateUUID() - require.NoError(t, err) - - // Create the replication token secret. - logger.Log(t, "Creating the replication token secret") - params := map[string]interface{}{ - "data": map[string]interface{}{ - "token": token, - }, - } - _, err = vaultClient.Logical().Write("consul/data/secret/replication", params) - require.NoError(t, err) - - return token -} - -// configureBootstrapTokenVaultSecret generates the bootstrap token secret ID, +// configureACLTokenVaultSecret generates a token secret ID for a given name, // stores it in vault as a secret and configures a policy to access it. -func configureBootstrapTokenVaultSecret(t *testing.T, vaultClient *vapi.Client) string { - // Create the Vault Policy for the bootstrap token. - logger.Log(t, "Creating bootstrap token policy") - err := vaultClient.Sys().PutPolicy("consul-bootstrap-token", bootstrapTokenPolicy) +func configureACLTokenVaultSecret(t *testing.T, vaultClient *vapi.Client, tokenName string) string { + // Create the Vault Policy for the token. + logger.Logf(t, "Creating %s token policy", tokenName) + policyName := fmt.Sprintf("%s-token", tokenName) + tokenPolicy := fmt.Sprintf(tokenPolicyTemplate, tokenName) + err := vaultClient.Sys().PutPolicy(policyName, tokenPolicy) require.NoError(t, err) // Generate the token secret. @@ -283,13 +216,13 @@ func configureBootstrapTokenVaultSecret(t *testing.T, vaultClient *vapi.Client) require.NoError(t, err) // Create the replication token secret. - logger.Log(t, "Creating the bootstrap token secret") + logger.Logf(t, "Creating the %s token secret", tokenName) params := map[string]interface{}{ "data": map[string]interface{}{ "token": token, }, } - _, err = vaultClient.Logical().Write("consul/data/secret/bootstrap", params) + _, err = vaultClient.Logical().Write(fmt.Sprintf("consul/data/secret/%s", tokenName), params) require.NoError(t, err) return token diff --git a/acceptance/tests/vault/vault_partitions_test.go b/acceptance/tests/vault/vault_partitions_test.go new file mode 100644 index 0000000000..4db9641482 --- /dev/null +++ b/acceptance/tests/vault/vault_partitions_test.go @@ -0,0 +1,250 @@ +package vault + +import ( + "context" + "fmt" + "testing" + + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul-k8s/acceptance/framework/vault" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestVault_Partitions(t *testing.T) { + env := suite.Environment() + cfg := suite.Config() + serverClusterCtx := env.DefaultContext(t) + clientClusterCtx := env.Context(t, environment.SecondaryContextName) + ns := serverClusterCtx.KubectlOptions(t).Namespace + + const secondaryPartition = "secondary" + + if !cfg.EnableEnterprise { + t.Skipf("skipping this test because -enable-enterprise is not set") + } + if !cfg.EnableMultiCluster { + t.Skipf("skipping this test because -enable-multi-cluster is not set") + } + vaultReleaseName := helpers.RandomName() + consulReleaseName := helpers.RandomName() + + // In the primary cluster, we will expose Vault server as a Load balancer + // or a NodePort service so that the secondary can connect to it. + serverClusterVaultHelmValues := map[string]string{ + "server.service.type": "LoadBalancer", + } + if cfg.UseKind { + serverClusterVaultHelmValues["server.service.type"] = "NodePort" + serverClusterVaultHelmValues["server.service.nodePort"] = "31000" + } + serverClusterVault := vault.NewVaultCluster(t, serverClusterCtx, cfg, vaultReleaseName, serverClusterVaultHelmValues) + serverClusterVault.Create(t, serverClusterCtx) + + externalVaultAddress := vaultAddress(t, cfg, serverClusterCtx, vaultReleaseName) + + // In the secondary cluster, we will only deploy the agent injector and provide + // it with the primary's Vault address. We also want to configure the injector with + // a different k8s auth method path since the secondary cluster will need its own auth method. + clientClusterVaultHelmValues := map[string]string{ + "server.enabled": "false", + "injector.externalVaultAddr": externalVaultAddress, + "injector.authPath": "auth/kubernetes-" + secondaryPartition, + } + + secondaryVaultCluster := vault.NewVaultCluster(t, clientClusterCtx, cfg, vaultReleaseName, clientClusterVaultHelmValues) + secondaryVaultCluster.Create(t, clientClusterCtx) + + vaultClient := serverClusterVault.VaultClient(t) + + // Configure Vault Kubernetes auth method for the secondary cluster. + { + // Create auth method service account and ClusterRoleBinding. The Vault server + // in the primary cluster will use this service account token to talk to the secondary + // Kubernetes cluster. + // This ClusterRoleBinding is adapted from the Vault server's role: + // https://github.com/hashicorp/vault-helm/blob/b0528fce49c529f2c37953ea3a14f30ed651e0d6/templates/server-clusterrolebinding.yaml + + // Use a single name for all RBAC objects. + authMethodRBACName := fmt.Sprintf("%s-vault-auth-method", vaultReleaseName) + _, err := clientClusterCtx.KubernetesClient(t).RbacV1().ClusterRoleBindings().Create(context.Background(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: authMethodRBACName, + }, + Subjects: []rbacv1.Subject{{Kind: rbacv1.ServiceAccountKind, Name: authMethodRBACName, Namespace: ns}}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Name: "system:auth-delegator", Kind: "ClusterRole"}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create service account for the auth method in the secondary cluster. + _, err = clientClusterCtx.KubernetesClient(t).CoreV1().ServiceAccounts(ns).Create(context.Background(), &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: authMethodRBACName, + }, + }, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + clientClusterCtx.KubernetesClient(t).RbacV1().ClusterRoleBindings().Delete(context.Background(), authMethodRBACName, metav1.DeleteOptions{}) + clientClusterCtx.KubernetesClient(t).CoreV1().ServiceAccounts(ns).Delete(context.Background(), authMethodRBACName, metav1.DeleteOptions{}) + }) + + // Figure out the host for the Kubernetes API. This needs to be reachable from the Vault server + // in the primary cluster. + k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, clientClusterCtx) + + // Now, configure the auth method in Vault. + secondaryVaultCluster.ConfigureAuthMethod(t, vaultClient, "kubernetes-"+secondaryPartition, k8sAuthMethodHost, authMethodRBACName, ns) + } + + configureGossipVaultSecret(t, vaultClient) + createConnectCAPolicy(t, vaultClient, "dc1") + configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg) + configureACLTokenVaultSecret(t, vaultClient, "bootstrap") + configureACLTokenVaultSecret(t, vaultClient, "partition") + + serverPolicies := "gossip,license,connect-ca-dc1,server-cert-dc1,bootstrap-token" + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "server", serverPolicies) + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "client", "gossip") + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "server-acl-init", "bootstrap-token,partition-token") + configureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes") + + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-"+secondaryPartition, "client", "gossip") + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-"+secondaryPartition, "server-acl-init", "partition-token") + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-"+secondaryPartition, "partition-init", "partition-token") + configureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes-"+secondaryPartition) + configurePKICA(t, vaultClient) + certPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1") + + vaultCASecretName := vault.CASecretName(vaultReleaseName) + + commonHelmValues := map[string]string{ + "global.adminPartitions.enabled": "true", + + "global.enableConsulNamespaces": "true", + + "connectInject.enabled": "true", + "connectInject.replicas": "1", + "controller.enabled": "true", + + "global.secretsBackend.vault.enabled": "true", + "global.secretsBackend.vault.consulClientRole": "client", + "global.secretsBackend.vault.consulCARole": "consul-ca", + "global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init", + + "global.secretsBackend.vault.ca.secretName": vaultCASecretName, + "global.secretsBackend.vault.ca.secretKey": "tls.crt", + + "global.acls.manageSystemACLs": "true", + + "global.tls.enabled": "true", + "global.tls.enableAutoEncrypt": "true", + "global.tls.caCert.secretName": "pki/cert/ca", + + "global.gossipEncryption.secretName": "consul/data/secret/gossip", + "global.gossipEncryption.secretKey": "gossip", + + "global.enterpriseLicense.secretName": "consul/data/secret/license", + "global.enterpriseLicense.secretKey": "license", + } + + serverHelmValues := map[string]string{ + "global.secretsBackend.vault.consulServerRole": "server", + "global.secretsBackend.vault.connectCA.address": serverClusterVault.Address(), + "global.secretsBackend.vault.connectCA.rootPKIPath": "connect_root", + "global.secretsBackend.vault.connectCA.intermediatePKIPath": "dc1/connect_inter", + + "global.acls.bootstrapToken.secretName": "consul/data/secret/bootstrap", + "global.acls.bootstrapToken.secretKey": "token", + "global.acls.partitionToken.secretName": "consul/data/secret/partition", + "global.acls.partitionToken.secretKey": "token", + + "server.exposeGossipAndRPCPorts": "true", + "server.serverCert.secretName": certPath, + + "server.extraVolumes[0].type": "secret", + "server.extraVolumes[0].name": vaultCASecretName, + "server.extraVolumes[0].load": "false", + } + + // On Kind, there are no load balancers but since all clusters + // share the same node network (docker bridge), we can use + // a NodePort service so that we can access node(s) in a different Kind cluster. + if cfg.UseKind { + serverHelmValues["global.adminPartitions.service.type"] = "NodePort" + serverHelmValues["global.adminPartitions.service.nodePort.https"] = "30000" + serverHelmValues["meshGateway.service.type"] = "NodePort" + serverHelmValues["meshGateway.service.nodePort"] = "30100" + } + + helpers.MergeMaps(serverHelmValues, commonHelmValues) + + logger.Log(t, "Installing Consul") + consulCluster := consul.NewHelmCluster(t, serverHelmValues, serverClusterCtx, cfg, consulReleaseName) + consulCluster.Create(t) + + partitionServiceName := fmt.Sprintf("%s-consul-partition", consulReleaseName) + partitionSvcAddress := k8s.ServiceHost(t, cfg, serverClusterCtx, partitionServiceName) + + k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, clientClusterCtx) + + // Move Vault CA secret from primary to secondary so that we can mount it to pods in the + // secondary cluster. + logger.Logf(t, "retrieving Vault CA secret %s from the primary cluster and applying to the secondary", vaultCASecretName) + vaultCASecret, err := serverClusterCtx.KubernetesClient(t).CoreV1().Secrets(ns).Get(context.Background(), vaultCASecretName, metav1.GetOptions{}) + vaultCASecret.ResourceVersion = "" + require.NoError(t, err) + _, err = clientClusterCtx.KubernetesClient(t).CoreV1().Secrets(ns).Create(context.Background(), vaultCASecret, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + clientClusterCtx.KubernetesClient(t).CoreV1().Secrets(ns).Delete(context.Background(), vaultCASecretName, metav1.DeleteOptions{}) + }) + + // Create client cluster. + clientHelmValues := map[string]string{ + "global.enabled": "false", + + "global.adminPartitions.name": secondaryPartition, + + "global.acls.bootstrapToken.secretName": "consul/data/secret/partition", + "global.acls.bootstrapToken.secretKey": "token", + + "global.secretsBackend.vault.agentAnnotations": fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName), + "global.secretsBackend.vault.adminPartitionsRole": "partition-init", + + "externalServers.enabled": "true", + "externalServers.hosts[0]": partitionSvcAddress, + "externalServers.tlsServerName": "server.dc1.consul", + "externalServers.k8sAuthMethodHost": k8sAuthMethodHost, + + "client.enabled": "true", + "client.exposeGossipPorts": "true", + "client.join[0]": partitionSvcAddress, + } + + if cfg.UseKind { + clientHelmValues["externalServers.httpsPort"] = "30000" + clientHelmValues["meshGateway.service.type"] = "NodePort" + clientHelmValues["meshGateway.service.nodePort"] = "30100" + } + + helpers.MergeMaps(clientHelmValues, commonHelmValues) + + // Install the consul cluster without servers in the client cluster kubernetes context. + clientConsulCluster := consul.NewHelmCluster(t, clientHelmValues, clientClusterCtx, cfg, consulReleaseName) + clientConsulCluster.Create(t) + + // Ensure consul clients are created. + agentPodList, err := clientClusterCtx.KubernetesClient(t).CoreV1().Pods(clientClusterCtx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "app=consul,component=client"}) + require.NoError(t, err) + require.NotEmpty(t, agentPodList.Items) + + output, err := k8s.RunKubectlAndGetOutputE(t, clientClusterCtx.KubectlOptions(t), "logs", agentPodList.Items[0].Name, "consul", "-n", clientClusterCtx.KubectlOptions(t).Namespace) + require.NoError(t, err) + require.Contains(t, output, "Partition: 'secondary'") +} diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go index f267e21e1a..b46ed52855 100644 --- a/acceptance/tests/vault/vault_test.go +++ b/acceptance/tests/vault/vault_test.go @@ -37,9 +37,16 @@ func TestVault(t *testing.T) { configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg) } - bootstrapToken := configureBootstrapTokenVaultSecret(t, vaultClient) + bootstrapToken := configureACLTokenVaultSecret(t, vaultClient, "bootstrap") - configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg, true) + serverPolicies := "gossip,connect-ca-dc1,server-cert-dc1,bootstrap-token" + if cfg.EnableEnterprise { + serverPolicies += ",license" + } + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "server", serverPolicies) + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "client", "gossip") + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "server-acl-init", "bootstrap-token") + configureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes") configurePKICA(t, vaultClient) certPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1") @@ -56,8 +63,8 @@ func TestVault(t *testing.T) { "controller.enabled": "true", "global.secretsBackend.vault.enabled": "true", - "global.secretsBackend.vault.consulServerRole": "consul-server", - "global.secretsBackend.vault.consulClientRole": "consul-client", + "global.secretsBackend.vault.consulServerRole": "server", + "global.secretsBackend.vault.consulClientRole": "client", "global.secretsBackend.vault.consulCARole": "consul-ca", "global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init", @@ -94,8 +101,8 @@ func TestVault(t *testing.T) { } if cfg.EnableEnterprise { - consulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/enterpriselicense" - consulHelmValues["global.enterpriseLicense.secretKey"] = "enterpriselicense" + consulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/license" + consulHelmValues["global.enterpriseLicense.secretKey"] = "license" } logger.Log(t, "Installing Consul") diff --git a/acceptance/tests/vault/vault_wan_fed_test.go b/acceptance/tests/vault/vault_wan_fed_test.go index 9b10f59338..182ed9cd4a 100644 --- a/acceptance/tests/vault/vault_wan_fed_test.go +++ b/acceptance/tests/vault/vault_wan_fed_test.go @@ -72,8 +72,6 @@ func TestVault_WANFederationViaGateways(t *testing.T) { configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg) } - configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg, true) - // Configure Vault Kubernetes auth method for the secondary datacenter. { // Create auth method service account and ClusterRoleBinding. The Vault server @@ -113,15 +111,29 @@ func TestVault_WANFederationViaGateways(t *testing.T) { secondaryVaultCluster.ConfigureAuthMethod(t, vaultClient, "kubernetes-dc2", k8sAuthMethodHost, authMethodRBACName, ns) } - configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "dc2", cfg, false) + commonServerPolicies := "gossip" + if cfg.EnableEnterprise { + commonServerPolicies += ",license" + } + primaryServerPolicies := commonServerPolicies + ",connect-ca-dc1,server-cert-dc1,bootstrap-token" + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "server", primaryServerPolicies) + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "client", "gossip") + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes", "server-acl-init", "bootstrap-token,replication-token") + configureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes") + + secondaryServerPolicies := commonServerPolicies + ",connect-ca-dc2,server-cert-dc2,replication-token" + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "server", secondaryServerPolicies) + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "client", "gossip") + configureKubernetesAuthRole(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "server-acl-init", "replication-token") + configureConsulCAKubernetesAuthRole(t, vaultClient, ns, "kubernetes-dc2") // Generate a CA and create PKI roles for the primary and secondary Consul servers. configurePKICA(t, vaultClient) primaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1") secondaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc2") - bootstrapToken := configureBootstrapTokenVaultSecret(t, vaultClient) - replicationToken := configureReplicationTokenVaultSecret(t, vaultClient) + bootstrapToken := configureACLTokenVaultSecret(t, vaultClient, "bootstrap") + replicationToken := configureACLTokenVaultSecret(t, vaultClient, "replication") // Create the Vault Policy for the Connect CA in both datacenters. createConnectCAPolicy(t, vaultClient, "dc1") @@ -131,10 +143,10 @@ func TestVault_WANFederationViaGateways(t *testing.T) { // secondary cluster. vaultCASecretName := vault.CASecretName(vaultReleaseName) logger.Logf(t, "retrieving Vault CA secret %s from the primary cluster and applying to the secondary", vaultCASecretName) - vaultCASecret, err := primaryCtx.KubernetesClient(t).CoreV1().Secrets(primaryCtx.KubectlOptions(t).Namespace).Get(context.Background(), vaultCASecretName, metav1.GetOptions{}) + vaultCASecret, err := primaryCtx.KubernetesClient(t).CoreV1().Secrets(ns).Get(context.Background(), vaultCASecretName, metav1.GetOptions{}) vaultCASecret.ResourceVersion = "" require.NoError(t, err) - _, err = secondaryCtx.KubernetesClient(t).CoreV1().Secrets(secondaryCtx.KubectlOptions(t).Namespace).Create(context.Background(), vaultCASecret, metav1.CreateOptions{}) + _, err = secondaryCtx.KubernetesClient(t).CoreV1().Secrets(ns).Create(context.Background(), vaultCASecret, metav1.CreateOptions{}) require.NoError(t, err) t.Cleanup(func() { secondaryCtx.KubernetesClient(t).CoreV1().Secrets(ns).Delete(context.Background(), vaultCASecretName, metav1.DeleteOptions{}) @@ -176,8 +188,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) { // Vault config. "global.secretsBackend.vault.enabled": "true", - "global.secretsBackend.vault.consulServerRole": "consul-server", - "global.secretsBackend.vault.consulClientRole": "consul-client", + "global.secretsBackend.vault.consulServerRole": "server", + "global.secretsBackend.vault.consulClientRole": "client", "global.secretsBackend.vault.consulCARole": "consul-ca", "global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init", "global.secretsBackend.vault.ca.secretName": vaultCASecretName, @@ -188,8 +200,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) { } if cfg.EnableEnterprise { - primaryConsulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/enterpriselicense" - primaryConsulHelmValues["global.enterpriseLicense.secretKey"] = "enterpriselicense" + primaryConsulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/license" + primaryConsulHelmValues["global.enterpriseLicense.secretKey"] = "license" } if cfg.UseKind { @@ -250,8 +262,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) { // Vault config. "global.secretsBackend.vault.enabled": "true", - "global.secretsBackend.vault.consulServerRole": "consul-server", - "global.secretsBackend.vault.consulClientRole": "consul-client", + "global.secretsBackend.vault.consulServerRole": "server", + "global.secretsBackend.vault.consulClientRole": "client", "global.secretsBackend.vault.consulCARole": "consul-ca", "global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init", "global.secretsBackend.vault.ca.secretName": vaultCASecretName, @@ -265,8 +277,8 @@ func TestVault_WANFederationViaGateways(t *testing.T) { } if cfg.EnableEnterprise { - secondaryConsulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/enterpriselicense" - secondaryConsulHelmValues["global.enterpriseLicense.secretKey"] = "enterpriselicense" + secondaryConsulHelmValues["global.enterpriseLicense.secretName"] = "consul/data/secret/license" + secondaryConsulHelmValues["global.enterpriseLicense.secretKey"] = "license" } if cfg.UseKind { diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl index e26a20eaa6..d533794e15 100644 --- a/charts/consul/templates/_helpers.tpl +++ b/charts/consul/templates/_helpers.tpl @@ -173,12 +173,11 @@ This template is for an init container. {{- if .Values.externalServers.tlsServerName }} -tls-server-name={{ .Values.externalServers.tlsServerName }} \ {{- end }} - {{- if not .Values.externalServers.useSystemRoots }} - -ca-file=/consul/tls/ca/tls.crt - {{- end }} {{- else }} -server-addr={{ template "consul.fullname" . }}-server \ -server-port=8501 \ + {{- end }} + {{- if or (not .Values.externalServers.enabled) (and .Values.externalServers.enabled (not .Values.externalServers.useSystemRoots)) }} {{- if .Values.global.secretsBackend.vault.enabled }} -ca-file=/vault/secrets/serverca.crt {{- else }} diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml index 7d5cdb2407..334949f840 100644 --- a/charts/consul/templates/client-daemonset.yaml +++ b/charts/consul/templates/client-daemonset.yaml @@ -68,7 +68,7 @@ spec: {{- if .Values.global.secretsBackend.vault.agentAnnotations }} {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }} {{- end }} - {{- if .Values.global.enterpriseLicense.secretName }} + {{- if and .Values.global.enterpriseLicense.secretName (not .Values.global.acls.manageSystemACLs) }} {{- with .Values.global.enterpriseLicense }} "vault.hashicorp.com/agent-inject-secret-enterpriselicense.txt": "{{ .secretName }}" "vault.hashicorp.com/agent-inject-template-enterpriselicense.txt": {{ template "consul.vaultSecretTemplate" . }} diff --git a/charts/consul/templates/partition-init-job.yaml b/charts/consul/templates/partition-init-job.yaml index 492160da0b..acc802b16a 100644 --- a/charts/consul/templates/partition-init-job.yaml +++ b/charts/consul/templates/partition-init-job.yaml @@ -2,6 +2,7 @@ {{- if (and .Values.global.adminPartitions.enabled (not $serverEnabled) (ne .Values.global.adminPartitions.name "default")) }} {{- template "consul.reservedNamesFailer" (list .Values.global.adminPartitions.name "global.adminPartitions.name") }} {{- if and (not .Values.externalServers.enabled) (ne .Values.global.adminPartitions.name "default") }}{{ fail "externalServers.enabled needs to be true and configured to create a non-default partition." }}{{ end -}} +{{- if and .Values.global.secretsBackend.vault.enabled .Values.global.acls.manageSystemACLs (not .Values.global.secretsBackend.vault.adminPartitionsRole) }}{{ fail "global.secretsBackend.vault.adminPartitionsRole is required when global.secretsBackend.vault.enabled and global.acls.manageSystemACLs are true." }}{{ end -}} apiVersion: batch/v1 kind: Job metadata: @@ -28,11 +29,35 @@ spec: component: partition-init annotations: "consul.hashicorp.com/connect-inject": "false" + {{- if (and .Values.global.secretsBackend.vault.enabled (or .Values.global.tls.enabled .Values.global.acls.manageSystemACLs)) }} + "vault.hashicorp.com/agent-pre-populate-only": "true" + "vault.hashicorp.com/agent-inject": "true" + {{- if .Values.global.acls.manageSystemACLs }} + "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.adminPartitionsRole }} + {{- if .Values.global.acls.bootstrapToken.secretName }} + {{- with .Values.global.acls.bootstrapToken }} + "vault.hashicorp.com/agent-inject-secret-bootstrap-token": "{{ .secretName }}" + "vault.hashicorp.com/agent-inject-template-bootstrap-token": {{ template "consul.vaultSecretTemplate" . }} + {{- end }} + {{- end }} + {{- else }} + "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }} + {{- end }} + "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }} + "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }} + {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }} + "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}" + "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}" + {{- end }} + {{- if .Values.global.secretsBackend.vault.agentAnnotations }} + {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }} + {{- end }} + {{- end }} spec: restartPolicy: Never serviceAccountName: {{ template "consul.fullname" . }}-partition-init {{- if .Values.global.tls.enabled }} - {{- if not .Values.externalServers.useSystemRoots }} + {{- if not (or .Values.externalServers.useSystemRoots .Values.global.secretsBackend.vault.enabled) }} volumes: - name: consul-ca-cert secret: @@ -55,14 +80,19 @@ spec: fieldRef: fieldPath: metadata.namespace {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }} + {{- if .Values.global.secretsBackend.vault.enabled }} + - name: CONSUL_HTTP_TOKEN_FILE + value: /vault/secrets/bootstrap-token + {{- else }} - name: CONSUL_HTTP_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.acls.bootstrapToken.secretName }} key: {{ .Values.global.acls.bootstrapToken.secretKey }} {{- end }} + {{- end }} {{- if .Values.global.tls.enabled }} - {{- if not .Values.externalServers.useSystemRoots }} + {{- if not (or .Values.externalServers.useSystemRoots .Values.global.secretsBackend.vault.enabled) }} volumeMounts: - name: consul-ca-cert mountPath: /consul/tls/ca @@ -86,8 +116,12 @@ spec: {{- if .Values.global.tls.enabled }} -use-https \ {{- if not .Values.externalServers.useSystemRoots }} + {{- if .Values.global.secretsBackend.vault.enabled }} + -ca-file=/vault/secrets/serverca.crt \ + {{- else }} -ca-file=/consul/tls/ca/tls.crt \ {{- end }} + {{- end }} {{- if .Values.externalServers.tlsServerName }} -tls-server-name={{ .Values.externalServers.tlsServerName }} \ {{- end }} diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml index cf137d388f..7b4d46577b 100644 --- a/charts/consul/templates/server-acl-init-job.yaml +++ b/charts/consul/templates/server-acl-init-job.yaml @@ -47,6 +47,12 @@ spec: "vault.hashicorp.com/agent-inject-template-bootstrap-token": {{ template "consul.vaultSecretTemplate" . }} {{- end }} {{- end }} + {{- if .Values.global.acls.partitionToken.secretName }} + {{- with .Values.global.acls.partitionToken }} + "vault.hashicorp.com/agent-inject-secret-partition-token": "{{ .secretName }}" + "vault.hashicorp.com/agent-inject-template-partition-token": {{ template "consul.vaultSecretTemplate" . }} + {{- end }} + {{- end }} {{- if .Values.global.tls.enabled }} "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }} "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }} @@ -269,6 +275,9 @@ spec: -acl-replication-token-file=/consul/acl/tokens/acl-replication-token \ {{- end }} {{- end }} + {{- if and .Values.global.secretsBackend.vault.enabled .Values.global.acls.partitionToken.secretName }} + -partition-token-file=/vault/secrets/partition-token \ + {{- end }} {{- if .Values.controller.enabled }} -controller=true \ diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats index b4d732feb9..57204ef2a9 100755 --- a/charts/consul/test/unit/client-daemonset.bats +++ b/charts/consul/test/unit/client-daemonset.bats @@ -2269,14 +2269,38 @@ rollingUpdate: local actual=$(echo $object | yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-enterpriselicense.txt"]' | tee /dev/stderr) [ "${actual}" = "path/to/secret" ] - local actual=$(echo $object | - yq -r '.annotations["vault.hashicorp.com/agent-inject-template-enterpriselicense.txt"]' | tee /dev/stderr) + local actual="$(echo $object | yq -r '.annotations["vault.hashicorp.com/agent-inject-template-enterpriselicense.txt"]' | tee /dev/stderr)" local expected=$'{{- with secret \"path/to/secret\" -}}\n{{- .Data.data.enterpriselicense -}}\n{{- end -}}' [ "${actual}" = "${expected}" ] } +@test "client/DaemonSet: vault enterprise license annotations are not set when ent license is set and ACLs are enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/client-daemonset.yaml \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=aclsrole' \ + --set 'global.enterpriseLicense.secretName=path/to/secret' \ + --set 'global.enterpriseLicense.secretKey=enterpriselicense' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.acls.bootstrapToken.secretName=boot' \ + --set 'global.acls.bootstrapToken.secretKey=token' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-enterpriselicense.txt"]' | tee /dev/stderr) + [ "${actual}" = "null" ] + + local actual="$(echo $object | + yq -r '.annotations["vault.hashicorp.com/agent-inject-template-enterpriselicense.txt"]' | tee /dev/stderr)" + [ "${actual}" = "null" ] +} + @test "client/DaemonSet: vault CONSUL_LICENSE_PATH is set to /vault/secrets/enterpriselicense.txt" { cd `chart_dir` local env=$(helm template \ diff --git a/charts/consul/test/unit/helpers.bats b/charts/consul/test/unit/helpers.bats index ee524a6842..181df6a34c 100644 --- a/charts/consul/test/unit/helpers.bats +++ b/charts/consul/test/unit/helpers.bats @@ -139,7 +139,7 @@ load _helpers # consul.getAutoEncryptClientCA helper since we need an existing template that calls # the consul.getAutoEncryptClientCA helper. -@test "helper/consul.getAutoEncryptClientCA: get-auto-encrypt-client-ca uses server's stateful set address by default" { +@test "helper/consul.getAutoEncryptClientCA: get-auto-encrypt-client-ca uses server's stateful set address by default and passes ca cert" { cd `chart_dir` local command=$(helm template \ -s templates/tests/test-runner.yaml \ @@ -217,10 +217,6 @@ load _helpers # check the default server port is 443 if not provided actual=$(echo $command | jq ' . | contains("-server-port=443")') [ "${actual}" = "true" ] - - # check server's CA cert - actual=$(echo $command | jq ' . | contains("-ca-file=/consul/tls/ca/tls.crt")') - [ "${actual}" = "true" ] } @test "helper/consul.getAutoEncryptClientCA: can pass cloud auto-join string to server address via externalServers.hosts" { @@ -285,7 +281,29 @@ load _helpers [ "${actual}" = "" ] } -@test "helper/consul.getAutoEncryptClientCA: uses the correct -ca-file when vault is enabled" { +@test "helper/consul.getAutoEncryptClientCA: uses the correct -ca-file when vault is enabled and external servers disabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/tests/test-runner.yaml \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'server.serverCert.secretName=pki_int/issue/test' \ + --set 'global.tls.caCert.secretName=pki_int/ca/pem' \ + . | tee /dev/stderr | + yq '.spec.initContainers[] | select(.name == "get-auto-encrypt-client-ca")' | tee /dev/stderr) + + actual=$(echo $object | jq '.command | join(" ") | contains("-ca-file=/vault/secrets/serverca.crt")') + [ "${actual}" = "true" ] + + actual=$(echo $object | jq '.volumeMounts[] | select(.name == "consul-ca-cert")') + [ "${actual}" = "" ] +} + +@test "helper/consul.getAutoEncryptClientCA: uses the correct -ca-file when vault and external servers is enabled" { cd `chart_dir` local object=$(helm template \ -s templates/tests/test-runner.yaml \ @@ -298,6 +316,8 @@ load _helpers --set 'server.serverCert.secretName=pki_int/issue/test' \ --set 'global.tls.caCert.secretName=pki_int/ca/pem' \ --set 'server.enabled=false' \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=consul.io' \ . | tee /dev/stderr | yq '.spec.initContainers[] | select(.name == "get-auto-encrypt-client-ca")' | tee /dev/stderr) diff --git a/charts/consul/test/unit/partition-init-job.bats b/charts/consul/test/unit/partition-init-job.bats index 78b91436f0..a907884afb 100644 --- a/charts/consul/test/unit/partition-init-job.bats +++ b/charts/consul/test/unit/partition-init-job.bats @@ -202,3 +202,323 @@ reservedNameTest() { [ "$status" -eq 1 ] [[ "$output" =~ "The name $name set for key global.adminPartitions.name is reserved by Consul for future use" ]] } + +#-------------------------------------------------------------------- +# Vault + +@test "partitionInit/Job: fails when vault and ACLs are enabled but adminPartitionsRole is not provided" { + cd `chart_dir` + run helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.acls.bootstrapToken.secretName=boot' \ + --set 'global.acls.bootstrapToken.secretKey=token' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=test' \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + . + [ "$status" -eq 1 ] + [[ "$output" =~ "global.secretsBackend.vault.adminPartitionsRole is required when global.secretsBackend.vault.enabled and global.acls.manageSystemACLs are true." ]] +} + +@test "partitionInit/Job: configures vault annotations when ACLs are enabled but TLS disabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.adminPartitionsRole=aprole' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=aclrole' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.acls.bootstrapToken.secretName=foo' \ + --set 'global.acls.bootstrapToken.secretKey=bar' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + # Check annotations + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-pre-populate-only"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr) + [ "${actual}" = "aprole" ] + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-inject-secret-bootstrap-token"') + [ "${actual}" = "foo" ] + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-inject-template-bootstrap-token"') + local expected=$'{{- with secret \"foo\" -}}\n{{- .Data.data.bar -}}\n{{- end -}}' + [ "${actual}" = "${expected}" ] + + # Check that the bootstrap token flag is set to the path of the Vault secret. + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="partition-init-job").env[] | select(.name=="CONSUL_HTTP_TOKEN_FILE").value') + [ "${actual}" = "/vault/secrets/bootstrap-token" ] + + # Check that no (secret) volumes are not attached + local actual=$(echo $object | jq -r '.spec.volumes') + [ "${actual}" = "null" ] + + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="partition-init-job").volumeMounts') + [ "${actual}" = "null" ] +} + +@test "partitionInit/Job: configures server CA to come from vault when vault and TLS are enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + # Check annotations + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-pre-populate-only"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr) + [ "${actual}" = "carole" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-secret-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = "foo" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-template-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = $'{{- with secret \"foo\" -}}\n{{- .Data.certificate -}}\n{{- end -}}' ] + + # Check that the consul-ca-cert volume is not attached + local actual=$(echo $object | jq -r '.spec.volumes') + [ "${actual}" = "null" ] + + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="partition-init-job").volumeMounts') + [ "${actual}" = "null" ] +} + +@test "partitionInit/Job: configures vault annotations when both ACLs and TLS are enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=aclrole' \ + --set 'global.secretsBackend.vault.adminPartitionsRole=aprole' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.acls.bootstrapToken.secretName=foo' \ + --set 'global.acls.bootstrapToken.secretKey=bar' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + # Check annotations + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-pre-populate-only"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr) + [ "${actual}" = "aprole" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-secret-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = "foo" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-template-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = $'{{- with secret \"foo\" -}}\n{{- .Data.certificate -}}\n{{- end -}}' ] + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-inject-secret-bootstrap-token"') + [ "${actual}" = "foo" ] + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-inject-template-bootstrap-token"') + local expected=$'{{- with secret \"foo\" -}}\n{{- .Data.data.bar -}}\n{{- end -}}' + [ "${actual}" = "${expected}" ] + + # Check that the bootstrap token flag is set to the path of the Vault secret. + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="partition-init-job").env[] | select(.name=="CONSUL_HTTP_TOKEN_FILE").value') + [ "${actual}" = "/vault/secrets/bootstrap-token" ] + + # Check that the consul-ca-cert volume is not attached + local actual=$(echo $object | jq -r '.spec.volumes') + [ "${actual}" = "null" ] + + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="partition-init-job").volumeMounts') + [ "${actual}" = "null" ] +} + +@test "partitionInit/Job: vault CA is not configured by default" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "partitionInit/Job: vault CA is not configured when secretName is set but secretKey is not" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretName=ca' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "partitionInit/Job: vault CA is not configured when secretKey is set but secretName is not" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "partitionInit/Job: vault CA is configured when both secretName and secretKey are set" { + cd `chart_dir` + local object=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretName=ca' \ + --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-extra-secret"') + [ "${actual}" = "ca" ] + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/ca-cert"') + [ "${actual}" = "/vault/custom/tls.crt" ] +} + +#-------------------------------------------------------------------- +# Vault agent annotations + +@test "partitionInit/Job: no vault agent annotations defined by default" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=aclrole' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | del(."consul.hashicorp.com/connect-inject") | del(."vault.hashicorp.com/agent-inject") | del(."vault.hashicorp.com/agent-pre-populate-only") | del(."vault.hashicorp.com/role") | del(."vault.hashicorp.com/agent-inject-secret-serverca.crt") | del(."vault.hashicorp.com/agent-inject-template-serverca.crt")' | tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "partitionInit/Job: vault agent annotations can be set" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/partition-init-job.yaml \ + --set 'global.enabled=false' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=bar" \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=aclrole' \ + --set 'global.secretsBackend.vault.agentAnnotations=foo: bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations.foo' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} diff --git a/charts/consul/test/unit/server-acl-init-job.bats b/charts/consul/test/unit/server-acl-init-job.bats index 9864ee7df4..0158dab7c6 100644 --- a/charts/consul/test/unit/server-acl-init-job.bats +++ b/charts/consul/test/unit/server-acl-init-job.bats @@ -633,14 +633,14 @@ load _helpers [ "${actual}" = "${expected}" ] # Check that the bootstrap token flag is set to the path of the Vault secret. - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").command | any(contains("-bootstrap-token-file=/vault/secrets/bootstrap-token"))') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").command | any(contains("-bootstrap-token-file=/vault/secrets/bootstrap-token"))') [ "${actual}" = "true" ] # Check that no (secret) volumes are not attached local actual=$(echo $object | jq -r '.spec.volumes') [ "${actual}" = "null" ] - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").volumeMounts') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").volumeMounts') [ "${actual}" = "null" ] } @@ -684,7 +684,7 @@ load _helpers local actual=$(echo $object | jq -r '.spec.volumes') [ "${actual}" = "null" ] - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").volumeMounts') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").volumeMounts') [ "${actual}" = "null" ] } @@ -800,9 +800,6 @@ load _helpers local object=$(helm template \ -s templates/server-acl-init-job.yaml \ --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=foo' \ --set 'global.secretsBackend.vault.consulServerRole=test' \ @@ -829,11 +826,11 @@ load _helpers local actual=$(echo $object | jq -r '.spec.volumes') [ "${actual}" = "null" ] - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").volumeMounts') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").volumeMounts') [ "${actual}" = "null" ] # Check that the replication token flag is set to the path of the Vault secret. - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").command | any(contains("-acl-replication-token-file=/vault/secrets/replication-token"))') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").command | any(contains("-acl-replication-token-file=/vault/secrets/replication-token"))') [ "${actual}" = "true" ] } @@ -842,9 +839,6 @@ load _helpers local object=$(helm template \ -s templates/server-acl-init-job.yaml \ --set 'global.acls.manageSystemACLs=true' \ - --set 'global.tls.enabled=true' \ - --set 'global.tls.enableAutoEncrypt=true' \ - --set 'global.tls.caCert.secretName=foo' \ --set 'global.secretsBackend.vault.enabled=true' \ --set 'global.secretsBackend.vault.consulClientRole=foo' \ --set 'global.secretsBackend.vault.consulServerRole=test' \ @@ -880,14 +874,60 @@ load _helpers local actual=$(echo $object | jq -r '.spec.volumes') [ "${actual}" = "null" ] - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").volumeMounts') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").volumeMounts') [ "${actual}" = "null" ] # Check that the replication and bootstrap token flags are set to the path of the Vault secret. - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").command | any(contains("-acl-replication-token-file=/vault/secrets/replication-token"))') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").command | any(contains("-acl-replication-token-file=/vault/secrets/replication-token"))') [ "${actual}" = "true" ] - local actual=$(echo $object | jq -r '.spec.containers[] | select(.name="post-install-job").command | any(contains("-bootstrap-token-file=/vault/secrets/bootstrap-token"))') + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").command | any(contains("-bootstrap-token-file=/vault/secrets/bootstrap-token"))') + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# Partition token in Vault + +@test "serverACLInit/Job: vault partition token can be provided" { + cd `chart_dir` + local object=$(helm template \ + -s templates/server-acl-init-job.yaml \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.manageSystemACLsRole=acl-role' \ + --set 'global.acls.bootstrapToken.secretName=/vault/boot' \ + --set 'global.acls.bootstrapToken.secretKey=token' \ + --set 'global.acls.partitionToken.secretName=/vault/secret' \ + --set 'global.acls.partitionToken.secretKey=token' \ + --set 'global.adminPartitions.enabled=true' \ + --set "global.adminPartitions.name=default" \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + # Check that the role is set. + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/role"') + [ "${actual}" = "acl-role" ] + + # Check Vault secret annotations. + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-inject-secret-partition-token"') + [ "${actual}" = "/vault/secret" ] + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-inject-template-partition-token"') + local expected=$'{{- with secret \"/vault/secret\" -}}\n{{- .Data.data.token -}}\n{{- end -}}' + [ "${actual}" = "${expected}" ] + + # Check that replication token Kubernetes secret volumes and volumeMounts are not attached. + local actual=$(echo $object | jq -r '.spec.volumes') + [ "${actual}" = "null" ] + + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").volumeMounts') + [ "${actual}" = "null" ] + + # Check that the replication token flag is set to the path of the Vault secret. + local actual=$(echo $object | jq -r '.spec.containers[] | select(.name=="post-install-job").command | any(contains("-partition-token-file=/vault/secrets/partition-token"))') [ "${actual}" = "true" ] } diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 85a7ed016c..8f63d97bdc 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -38,10 +38,12 @@ global: # must be installed in the default partition. Creation of Admin Partitions is only supported during installation. # Admin Partitions cannot be installed via a Helm upgrade operation. Only Helm installs are supported. enabled: false + # The name of the Admin Partition. The partition name cannot be modified once the partition has been installed. # Changing the partition name would require an un-install and a re-install with the updated name. # Must be "default" in the server cluster ie the Kubernetes cluster that the Consul server pods are deployed onto. name: "default" + # Partition service properties. service: type: LoadBalancer @@ -155,21 +157,32 @@ global: # - gossip encryption key defined by `global.gossipEncryption.secretName`. # To discover the service account name of the Consul client, run # ```shell-session - # $ helm template --show-only templates/client-serviceaccount.yaml charts/consul + # $ helm template --show-only templates/client-serviceaccount.yaml hashicorp/consul # ``` # and check the name of `metadata.name`. consulClientRole: "" # A Vault role to allow Kubernetes job that manages ACLs for this Helm chart (`server-acl-init`) - # to read and update Vault secrets for the Consul's bootstrap and replication tokens. + # to read and update Vault secrets for the Consul's bootstrap, replication or partition tokens. # This role must be bound the `server-acl-init`'s service account. # To discover the service account name of the `server-acl-init` job, run # ```shell-session - # $ helm template --show-only templates/server-acl-init-serviceaccount.yaml charts/consul + # $ helm template --show-only templates/server-acl-init-serviceaccount.yaml \ + # --set global.acls.manageSystemACLs=true hashicorp/consul # ``` # and check the name of `metadata.name`. manageSystemACLsRole: "" + # [Enterprise Only] A Vault role to allow Kubernetes job that creates a Consul partition for this Helm chart (`partition-init`) + # to read Vault secret for the partition ACL token. + # This role must be bound the `partition-init`'s service account. + # To discover the service account name of the `partition-init` job, run with Helm values for the client cluster: + # ```shell-session + # $ helm template --show-only templates/partition-init-serviceaccount.yaml -f client-cluster-values.yaml hashicorp/consul + # ``` + # and check the name of `metadata.name`. + adminPartitionsRole: "" + # This value defines additional annotations for # Vault agent on any pods where it'll be running. # This should be formatted as a multi-line string. @@ -397,6 +410,18 @@ global: # The key of the Kubernetes or Vault secret. secretKey: null + # partitionToken references a Vault secret containing the ACL token to be used in non-default partitions. + # This value should only be provided in the default partition and only when setting + # `global.secretsBackend.vault.enabled` to true. + # We will use the value of the secret stored in Vault to create an ACL token in Consul with the value of the + # secret as the secretID for the token. + # In non-default, partitions set this secret as the `bootstrapToken`. + partitionToken: + # The name of the path of the secret in Vault. + secretName: null + # The key of the Vault secret. + secretKey: null + # [Enterprise Only] This value refers to a Kubernetes secret that you have created # that contains your enterprise license. It is required if you are using an # enterprise binary. Defining it here applies it to your cluster once a leader diff --git a/control-plane/subcommand/server-acl-init/command.go b/control-plane/subcommand/server-acl-init/command.go index c487790ddd..05955cec40 100644 --- a/control-plane/subcommand/server-acl-init/command.go +++ b/control-plane/subcommand/server-acl-init/command.go @@ -72,8 +72,9 @@ type Command struct { flagACLReplicationTokenFile string // Flags to support partitions. - flagEnablePartitions bool // true if Admin Partitions are enabled - flagPartitionName string // name of the Admin Partition + flagEnablePartitions bool // true if Admin Partitions are enabled + flagPartitionName string // name of the Admin Partition + flagPartitionTokenFile string // Flags to support namespaces. flagEnableNamespaces bool // Use namespacing on all components @@ -174,6 +175,8 @@ func (c *Command) init() { "[Enterprise Only] Enables Admin Partitions") c.flags.StringVar(&c.flagPartitionName, "partition", "", "[Enterprise Only] Name of the Admin Partition") + c.flags.StringVar(&c.flagPartitionTokenFile, "partition-token-file", "", + "[Enterprise Only] Path to file containing ACL token to be used in non-default partitions.") c.flags.BoolVar(&c.flagEnableNamespaces, "enable-namespaces", false, "[Enterprise Only] Enables namespaces, in either a single Consul namespace or mirrored [Enterprise only feature]") @@ -249,34 +252,35 @@ func (c *Command) Run(args []string) int { c.UI.Error(err.Error()) return 1 } + var aclReplicationToken string if c.flagACLReplicationTokenFile != "" { - // Load the ACL replication token from file. - tokenBytes, err := ioutil.ReadFile(c.flagACLReplicationTokenFile) + var err error + aclReplicationToken, err = loadTokenFromFile(c.flagACLReplicationTokenFile) if err != nil { - c.UI.Error(fmt.Sprintf("Unable to read ACL replication token from file %q: %s", c.flagACLReplicationTokenFile, err)) + c.UI.Error(err.Error()) return 1 } - if len(tokenBytes) == 0 { - c.UI.Error(fmt.Sprintf("ACL replication token file %q is empty", c.flagACLReplicationTokenFile)) + } + + var partitionToken string + if c.flagPartitionTokenFile != "" { + var err error + partitionToken, err = loadTokenFromFile(c.flagPartitionTokenFile) + if err != nil { + c.UI.Error(err.Error()) return 1 } - aclReplicationToken = strings.TrimSpace(string(tokenBytes)) } var providedBootstrapToken string if c.flagBootstrapTokenFile != "" { - // Load the bootstrap token from file. - tokenBytes, err := ioutil.ReadFile(c.flagBootstrapTokenFile) + var err error + providedBootstrapToken, err = loadTokenFromFile(c.flagBootstrapTokenFile) if err != nil { - c.UI.Error(fmt.Sprintf("Unable to read bootstrap token from file %q: %s", c.flagBootstrapTokenFile, err)) - return 1 - } - if len(tokenBytes) == 0 { - c.UI.Error(fmt.Sprintf("Bootstrap token file %q is empty", c.flagBootstrapTokenFile)) + c.UI.Error(err.Error()) return 1 } - providedBootstrapToken = strings.TrimSpace(string(tokenBytes)) } var cancel context.CancelFunc @@ -370,7 +374,11 @@ func (c *Command) Run(args []string) int { if c.flagEnablePartitions && c.flagPartitionName == consulDefaultPartition && primary { // Partition token is local because only the Primary datacenter can have Admin Partitions. - err := c.createLocalACL("partitions", partitionRules, consulDC, primary, consulClient) + if c.flagPartitionTokenFile != "" { + err = c.createACLWithSecretID("partitions", partitionRules, consulDC, primary, consulClient, partitionToken, true) + } else { + err = c.createLocalACL("partitions", partitionRules, consulDC, primary, consulClient) + } if err != nil { c.log.Error(err.Error()) return 1 @@ -722,7 +730,7 @@ func (c *Command) Run(args []string) int { // Policy must be global because it replicates from the primary DC // and so the primary DC needs to be able to accept the token. if aclReplicationToken != "" { - err = c.createGlobalACLWithSecretID(common.ACLReplicationTokenName, rules, consulDC, primary, consulClient, aclReplicationToken) + err = c.createACLWithSecretID(common.ACLReplicationTokenName, rules, consulDC, primary, consulClient, aclReplicationToken, false) } else { err = c.createGlobalACL(common.ACLReplicationTokenName, rules, consulDC, primary, consulClient) } @@ -957,6 +965,18 @@ func (c *Command) validateFlags() error { return nil } +func loadTokenFromFile(tokenFile string) (string, error) { + // Load the bootstrap token from file. + tokenBytes, err := ioutil.ReadFile(tokenFile) + if err != nil { + return "", fmt.Errorf("unable to read token from file %q: %s", tokenFile, err) + } + if len(tokenBytes) == 0 { + return "", fmt.Errorf("token file %q is empty", tokenFile) + } + return strings.TrimSpace(string(tokenBytes)), nil +} + const ( consulDefaultNamespace = "default" consulDefaultPartition = "default" diff --git a/control-plane/subcommand/server-acl-init/command_ent_test.go b/control-plane/subcommand/server-acl-init/command_ent_test.go index a0a0025fc1..16db2c43e3 100644 --- a/control-plane/subcommand/server-acl-init/command_ent_test.go +++ b/control-plane/subcommand/server-acl-init/command_ent_test.go @@ -5,8 +5,10 @@ package serveraclinit import ( "context" "fmt" + "io/ioutil" "net/http" "net/http/httptest" + "os" "strings" "testing" @@ -18,6 +20,7 @@ import ( "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/mitchellh/cli" "github.com/stretchr/testify/require" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) @@ -1220,6 +1223,72 @@ func TestRun_NamespaceEnabled_ValidateLoginToken_SecondaryDatacenter(t *testing. } } +// Test that the partition token can be created when it's provided with a file. +func TestRun_PartitionTokenDefaultPartition_WithProvidedSecretID(t *testing.T) { + t.Parallel() + + k8s, testSvr := completeSetup(t) + defer testSvr.Stop() + setUpK8sServiceAccount(t, k8s, ns) + + partitionToken := "123e4567-e89b-12d3-a456-426614174000" + partitionTokenFile, err := ioutil.TempFile("", "partitiontoken") + require.NoError(t, err) + defer os.Remove(partitionTokenFile.Name()) + + partitionTokenFile.WriteString(partitionToken) + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + cmdArgs := []string{ + "-timeout=1m", + "-k8s-namespace=" + ns, + "-server-address", strings.Split(testSvr.HTTPAddr, ":")[0], + "-server-port", strings.Split(testSvr.HTTPAddr, ":")[1], + "-resource-prefix=" + resourcePrefix, + "-enable-partitions", + "-partition=default", + "-partition-token-file", partitionTokenFile.Name(), + } + + responseCode := cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + + // Check that this token is created. + consul, err := api.NewClient(&api.Config{ + Address: testSvr.HTTPAddr, + Token: partitionToken, + }) + require.NoError(t, err) + token, _, err := consul.ACL().TokenReadSelf(nil) + require.NoError(t, err) + + for _, policyLink := range token.Policies { + policy := policyExists(t, policyLink.Name, consul) + require.Equal(t, policy.Datacenters, []string{"dc1"}) + + // Test that the token was not created as a Kubernetes Secret. + _, err := k8s.CoreV1().Secrets(ns).Get(context.Background(), resourcePrefix+"-partitions-acl-token", metav1.GetOptions{}) + require.True(t, k8serrors.IsNotFound(err)) + } + + // Test that if the same command is run again, it doesn't error. + t.Run(t.Name()+"-retried", func(t *testing.T) { + ui = cli.NewMockUi() + cmd = Command{ + UI: ui, + clientset: k8s, + } + cmd.init() + responseCode = cmd.Run(cmdArgs) + require.Equal(t, 0, responseCode, ui.ErrorWriter.String()) + }) +} + // partitionedSetup is a helper function which creates a server and a consul agent that runs as // a client in the provided partitionName. The bootToken is the token used as the bootstrap token // for both the client and the server. The helper creates a server, then creates a partition with diff --git a/control-plane/subcommand/server-acl-init/command_test.go b/control-plane/subcommand/server-acl-init/command_test.go index 9ac9a6ab8a..d91b0a9437 100644 --- a/control-plane/subcommand/server-acl-init/command_test.go +++ b/control-plane/subcommand/server-acl-init/command_test.go @@ -59,11 +59,11 @@ func TestRun_FlagValidation(t *testing.T) { }, { Flags: []string{"-acl-replication-token-file=/notexist", "-server-address=localhost", "-resource-prefix=prefix"}, - ExpErr: "Unable to read ACL replication token from file \"/notexist\": open /notexist: no such file or directory", + ExpErr: "unable to read token from file \"/notexist\": open /notexist: no such file or directory", }, { Flags: []string{"-bootstrap-token-file=/notexist", "-server-address=localhost", "-resource-prefix=prefix"}, - ExpErr: "Unable to read bootstrap token from file \"/notexist\": open /notexist: no such file or directory", + ExpErr: "unable to read token from file \"/notexist\": open /notexist: no such file or directory", }, { Flags: []string{ @@ -1281,6 +1281,10 @@ func TestRun_NoLeader(t *testing.T) { "PUT", "/v1/acl/role", }, + { + "GET", + "/v1/acl/binding-rules", + }, { "PUT", "/v1/acl/binding-rule", @@ -1511,6 +1515,10 @@ func TestRun_ClientPolicyAndBindingRuleRetry(t *testing.T) { "PUT", "/v1/acl/role", }, + { + "GET", + "/v1/acl/binding-rules", + }, { "PUT", "/v1/acl/binding-rule", @@ -1674,6 +1682,10 @@ func TestRun_AlreadyBootstrapped(t *testing.T) { "PUT", "/v1/acl/role", }, + { + "GET", + "/v1/acl/binding-rules", + }, { "PUT", "/v1/acl/binding-rule", diff --git a/control-plane/subcommand/server-acl-init/create_or_update.go b/control-plane/subcommand/server-acl-init/create_or_update.go index a1934a8b01..085372827b 100644 --- a/control-plane/subcommand/server-acl-init/create_or_update.go +++ b/control-plane/subcommand/server-acl-init/create_or_update.go @@ -200,9 +200,9 @@ func (c *Command) createGlobalACL(name, rules, dc string, isPrimary bool, consul return c.createACL(name, rules, false, dc, isPrimary, consulClient, "") } -// createGlobalACLWithSecretID creates a global policy and acl token with provided secret ID. -func (c *Command) createGlobalACLWithSecretID(name, rules, dc string, isPrimary bool, consulClient *api.Client, secretID string) error { - return c.createACL(name, rules, false, dc, isPrimary, consulClient, secretID) +// createACLWithSecretID creates a global policy and acl token with provided secret ID. +func (c *Command) createACLWithSecretID(name, rules, dc string, isPrimary bool, consulClient *api.Client, secretID string, local bool) error { + return c.createACL(name, rules, local, dc, isPrimary, consulClient, secretID) } // createACL creates a policy with rules and name. If localToken is true then