Skip to content

Commit

Permalink
Run snapshot agent as a sidecar with consul servers (#1620)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ishustava
ishustava authored and Thomas Eckert committed Oct 27, 2022
1 parent daac0e9 commit f42b927
Show file tree
Hide file tree
Showing 24 changed files with 795 additions and 2,241 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@
BREAKING_CHANGES:
* Helm:
* Remove `global.consulSidecarContainer` from values file as there is no longer a consul sidecar. [[GH-1635](https://github.com/hashicorp/consul-k8s/pull/1635)]
* Consul snapshot-agent now runs as a sidecar with Consul servers. [[GH-1620](https://github.com/hashicorp/consul-k8s/pull/1620)]
This results in the following changes to Helm values:
* Move `client.snapshotAgent` values to `server.snapshotAgent`, with the exception of the following values:
* `client.snaphostAgent.replicas`
* `client.snaphostAgent.serviceAccount`
* Remove `global.secretsBackend.vault.consulSnapshotAgentRole` value. You should now use the `global.secretsBackend.vault.consulServerRole` for access to any Vault secrets.

FEATURES:
* Consul-dataplane:
Expand Down
7 changes: 2 additions & 5 deletions acceptance/tests/snapshot-agent/main_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package snapshotagent

import (
"fmt"
"os"
"testing"

Expand All @@ -11,8 +10,6 @@ import (
var suite testsuite.Suite

func TestMain(m *testing.M) {
fmt.Println("Skipping snapshot agent tests because it's not supported with agentless yet")
os.Exit(0)
//suite = testsuite.NewSuite(m)
//os.Exit(suite.Run())
suite = testsuite.NewSuite(m)
os.Exit(suite.Run())
}
123 changes: 58 additions & 65 deletions acceptance/tests/snapshot-agent/snapshot_agent_k8s_secret_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"strconv"
"testing"
"time"

Expand All @@ -15,7 +15,7 @@ import (
"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
"github.com/hashicorp/go-uuid"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand All @@ -31,83 +31,76 @@ func TestSnapshotAgent_K8sSecret(t *testing.T) {
if cfg.EnableCNI {
t.Skipf("skipping because -enable-cni is set and snapshot agent is already tested with regular tproxy")
}
ctx := suite.Environment().DefaultContext(t)
kubectlOptions := ctx.KubectlOptions(t)
ns := kubectlOptions.Namespace
releaseName := helpers.RandomName()

// Generate a bootstrap token
bootstrapToken, err := uuid.GenerateUUID()
require.NoError(t, err)

bsSecretName := fmt.Sprintf("%s-acl-bootstrap-token", releaseName)
bsSecretKey := "token"
saSecretName := fmt.Sprintf("%s-snapshot-agent-config", releaseName)
saSecretKey := "token"

// Create cluster
helmValues := map[string]string{
"global.tls.enabled": "true",
"global.gossipEncryption.autoGenerate": "true",
"global.acls.manageSystemACLs": "true",
"global.acls.bootstrapToken.secretName": bsSecretName,
"global.acls.bootstrapToken.secretKey": bsSecretKey,
"client.snapshotAgent.enabled": "true",
"client.snapshotAgent.configSecret.secretName": saSecretName,
"client.snapshotAgent.configSecret.secretKey": saSecretKey,
cases := map[string]struct {
secure bool
}{
"non-secure": {secure: false},
"secure": {secure: true},
}

// Get new cluster
consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), cfg, releaseName)
client := environment.KubernetesClientFromOptions(t, kubectlOptions)
for name, c := range cases {
t.Run(name, func(t *testing.T) {
ctx := suite.Environment().DefaultContext(t)
kubectlOptions := ctx.KubectlOptions(t)
ns := kubectlOptions.Namespace
releaseName := helpers.RandomName()

// Add bootstrap token secret
logger.Log(t, "Storing bootstrap token as a k8s secret")
consul.CreateK8sSecret(t, client, cfg, ns, bsSecretName, bsSecretKey, bootstrapToken)
saSecretName := fmt.Sprintf("%s-snapshot-agent-config", releaseName)
saSecretKey := "config"

// Add snapshot agent config secret
logger.Log(t, "Storing snapshot agent config as a k8s secret")
config := generateSnapshotAgentConfig(t, bootstrapToken)
logger.Logf(t, "Snapshot agent config: %s", config)
consul.CreateK8sSecret(t, client, cfg, ns, saSecretName, saSecretKey, config)
// Create cluster
helmValues := map[string]string{
"global.tls.enabled": strconv.FormatBool(c.secure),
"global.gossipEncryption.autoGenerate": strconv.FormatBool(c.secure),
"global.acls.manageSystemACLs": strconv.FormatBool(c.secure),
"server.snapshotAgent.enabled": "true",
"server.snapshotAgent.configSecret.secretName": saSecretName,
"server.snapshotAgent.configSecret.secretKey": saSecretKey,
"connectInject.enabled": "false",
"controller.enabled": "false",
}

// Create cluster
consulCluster.Create(t)
// ----------------------------------
// Get new cluster
consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), cfg, releaseName)
client := environment.KubernetesClientFromOptions(t, kubectlOptions)

// Validate that consul snapshot agent is running correctly and is generating snapshot files
logger.Log(t, "Confirming that Consul Snapshot Agent is generating snapshot files")
// Create k8s client from kubectl options.
// Add snapshot agent config secret
logger.Log(t, "Storing snapshot agent config as a k8s secret")
config := generateSnapshotAgentConfig(t)
logger.Logf(t, "Snapshot agent config: %s", config)
consul.CreateK8sSecret(t, client, cfg, ns, saSecretName, saSecretKey, config)

podList, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(),
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=client-snapshot-agent,release=%s", releaseName)})
require.NoError(t, err)
require.True(t, len(podList.Items) > 0)
// Create cluster
consulCluster.Create(t)
// ----------------------------------

// Validate that consul snapshot agent is running correctly and is generating snapshot files
logger.Log(t, "Confirming that Consul Snapshot Agent is generating snapshot files")
// Create k8s client from kubectl options.

// Wait for 10seconds to allow snapsot to write.
time.Sleep(10 * time.Second)
podList, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(),
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=server,release=%s", releaseName)})
require.NoError(t, err)
require.Len(t, podList.Items, 1, "expected to find only 1 consul server instance")

// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
hasSnapshots := false
for _, pod := range podList.Items {
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/")
logger.Logf(t, "Snapshot: \n%s", snapshotFileListOutput)
require.NoError(t, err)
if strings.Contains(snapshotFileListOutput, ".snap") {
logger.Logf(t, "Agent pod contains snapshot files")
hasSnapshots = true
break
} else {
logger.Logf(t, "Agent pod does not contain snapshot files")
}
// We need to give some extra time for ACLs to finish bootstrapping and for servers to come up.
timer := &retry.Timer{Timeout: 1 * time.Minute, Wait: 1 * time.Second}
retry.RunWith(timer, t, func(r *retry.R) {
// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
pod := podList.Items[0]
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/tmp")
require.NoError(r, err)
logger.Logf(t, "Snapshot: \n%s", snapshotFileListOutput)
require.Contains(r, snapshotFileListOutput, ".snap", "Agent pod does not contain snapshot files")
})
})
}
require.True(t, hasSnapshots, ".snap")
}

func generateSnapshotAgentConfig(t *testing.T, token string) string {
func generateSnapshotAgentConfig(t *testing.T) string {
config := map[string]interface{}{
"snapshot_agent": map[string]interface{}{
"token": token,
"log": map[string]interface{}{
"level": "INFO",
"enable_syslog": false,
Expand All @@ -124,7 +117,7 @@ func generateSnapshotAgentConfig(t *testing.T, token string) string {
"local_scratch_path": "",
},
"local_storage": map[string]interface{}{
"path": ".",
"path": "/tmp",
},
},
}
Expand Down
86 changes: 20 additions & 66 deletions acceptance/tests/snapshot-agent/snapshot_agent_vault_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package snapshotagent
import (
"context"
"fmt"
"strings"
"testing"
"time"

Expand All @@ -14,6 +13,7 @@ import (
"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
"github.com/hashicorp/consul/sdk/testutil/retry"
"github.com/hashicorp/go-uuid"
"github.com/hashicorp/go-version"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -54,13 +54,6 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// -------------------------
// PKI
// -------------------------
// Configure Service Mesh CA
connectCAPolicy := "connect-ca-dc1"
connectCARootPath := "connect_root"
connectCAIntermediatePath := "dc1/connect_inter"
// Configure Policy for Connect CA
vault.CreateConnectCARootAndIntermediatePKIPolicy(t, vaultClient, connectCAPolicy, connectCARootPath, connectCAIntermediatePath)

// Configure Server PKI
serverPKIConfig := &vault.PKIAndAuthRoleConfiguration{
BaseURL: "pki",
Expand Down Expand Up @@ -112,7 +105,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
bootstrapTokenSecret.SaveSecretAndAddReadPolicy(t, vaultClient)

// Snapshot Agent config
snapshotAgentConfig := generateSnapshotAgentConfig(t, bootstrapToken)
snapshotAgentConfig := generateSnapshotAgentConfig(t)
require.NoError(t, err)
snapshotAgentConfigSecret := &vault.KV2Secret{
Path: "consul/data/secret/snapshot-agent-config",
Expand All @@ -125,7 +118,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// -------------------------
// Additional Auth Roles
// -------------------------
serverPolicies := fmt.Sprintf("%s,%s,%s,%s", gossipSecret.PolicyName, connectCAPolicy, serverPKIConfig.PolicyName, bootstrapTokenSecret.PolicyName)
serverPolicies := fmt.Sprintf("%s,%s,%s,%s", gossipSecret.PolicyName, serverPKIConfig.PolicyName, bootstrapTokenSecret.PolicyName, snapshotAgentConfigSecret.PolicyName)
if cfg.EnableEnterprise {
serverPolicies += fmt.Sprintf(",%s", licenseSecret.PolicyName)
}
Expand All @@ -141,18 +134,6 @@ func TestSnapshotAgent_Vault(t *testing.T) {
}
srvAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

// client
consulClientRole := "client"
consulClientServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, "client")
clientAuthRoleConfig := &vault.KubernetesAuthRoleConfiguration{
ServiceAccountName: consulClientServiceAccountName,
KubernetesNamespace: ns,
AuthMethodPath: "kubernetes",
RoleName: consulClientRole,
PolicyNames: gossipSecret.PolicyName,
}
clientAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

// manageSystemACLs
manageSystemACLsRole := "server-acl-init"
manageSystemACLsServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, "server-acl-init")
Expand All @@ -175,42 +156,24 @@ func TestSnapshotAgent_Vault(t *testing.T) {
}
srvCAAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

// snapshot agent config
snapAgentRole := "snapshot-agent"
snapAgentServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, "snapshot-agent")
saAuthRoleConfig := &vault.KubernetesAuthRoleConfiguration{
ServiceAccountName: snapAgentServiceAccountName,
KubernetesNamespace: ns,
AuthMethodPath: "kubernetes",
RoleName: snapAgentRole,
PolicyNames: fmt.Sprintf("%s,%s", licenseSecret.PolicyName, snapshotAgentConfigSecret.PolicyName),
}
saAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

vaultCASecret := vault.CASecretName(vaultReleaseName)

consulHelmValues := map[string]string{
"server.extraVolumes[0].type": "secret",
"server.extraVolumes[0].name": vaultCASecret,
"server.extraVolumes[0].load": "false",

"connectInject.enabled": "true",
"connectInject.enabled": "false",
"connectInject.replicas": "1",
"controller.enabled": "true",
"controller.enabled": "false",

"global.secretsBackend.vault.enabled": "true",
"global.secretsBackend.vault.consulServerRole": consulServerRole,
"global.secretsBackend.vault.consulClientRole": consulClientRole,
"global.secretsBackend.vault.consulCARole": serverPKIConfig.RoleName,
"global.secretsBackend.vault.manageSystemACLsRole": manageSystemACLsRole,

"global.secretsBackend.vault.ca.secretName": vaultCASecret,
"global.secretsBackend.vault.ca.secretKey": "tls.crt",

"global.secretsBackend.vault.connectCA.address": vaultCluster.Address(),
"global.secretsBackend.vault.connectCA.rootPKIPath": connectCARootPath,
"global.secretsBackend.vault.connectCA.intermediatePKIPath": connectCAIntermediatePath,

"global.acls.manageSystemACLs": "true",
"global.acls.bootstrapToken.secretName": bootstrapTokenSecret.Path,
"global.acls.bootstrapToken.secretKey": bootstrapTokenSecret.Key,
Expand All @@ -220,10 +183,9 @@ func TestSnapshotAgent_Vault(t *testing.T) {
"global.tls.caCert.secretName": serverPKIConfig.CAPath,
"global.tls.enableAutoEncrypt": "true",

"client.snapshotAgent.enabled": "true",
"client.snapshotAgent.configSecret.secretName": snapshotAgentConfigSecret.Path,
"client.snapshotAgent.configSecret.secretKey": snapshotAgentConfigSecret.Key,
"global.secretsBackend.vault.consulSnapshotAgentRole": snapAgentRole,
"server.snapshotAgent.enabled": "true",
"server.snapshotAgent.configSecret.secretName": snapshotAgentConfigSecret.Path,
"server.snapshotAgent.configSecret.secretKey": snapshotAgentConfigSecret.Key,
}

if cfg.EnableEnterprise {
Expand All @@ -240,26 +202,18 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// Create k8s client from kubectl options.
client := environment.KubernetesClientFromOptions(t, kubectlOptions)
podList, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(),
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=client-snapshot-agent,release=%s", consulReleaseName)})
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=server,release=%s", consulReleaseName)})
require.NoError(t, err)
require.True(t, len(podList.Items) > 0)

// Wait for 10 seconds to allow snapshot to write.
time.Sleep(10 * time.Second)

// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
hasSnapshots := false
for _, pod := range podList.Items {
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/")
require.Len(t, podList.Items, 1, "expected to find only 1 consul server instance")

// We need to give some extra time for ACLs to finish bootstrapping and for servers to come up.
timer := &retry.Timer{Timeout: 1 * time.Minute, Wait: 1 * time.Second}
retry.RunWith(timer, t, func(r *retry.R) {
// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
pod := podList.Items[0]
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/tmp")
require.NoError(r, err)
logger.Logf(t, "Snapshot: \n%s", snapshotFileListOutput)
require.NoError(t, err)
if strings.Contains(snapshotFileListOutput, ".snap") {
logger.Logf(t, "Agent pod contains snapshot files")
hasSnapshots = true
break
} else {
logger.Logf(t, "Agent pod does not contain snapshot files")
}
}
require.True(t, hasSnapshots)
require.Contains(r, snapshotFileListOutput, ".snap", "Agent pod does not contain snapshot files")
})
}
Loading

0 comments on commit f42b927

Please sign in to comment.