Skip to content

Commit

Permalink
agentless: run snapshot-agent as a sidecar with consul servers
Browse files Browse the repository at this point in the history
  • Loading branch information
@ishustava
ishustava committed Oct 26, 2022
1 parent 933cc5e commit e4c31d7
Show file tree
Hide file tree
Showing 24 changed files with 777 additions and 2,220 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@
BREAKING_CHANGES:
* Helm:
* Remove `global.consulSidecarContainer` from values file as there is no longer a consul sidecar. [[GH-1635](https://github.com/hashicorp/consul-k8s/pull/1635)]
* Consul snapshot-agent now runs as a sidecar with Consul servers. [[GH-1620](https://github.com/hashicorp/consul-k8s/pull/1620)]
This results in the following changes to Helm values:
* Move `client.snapshotAgent` values to `server.snapshotAgent`, with the exception of the following values:
* `client.snaphostAgent.replicas`
* `client.snaphostAgent.serviceAccount`
* Remove `global.secretsBackend.vault.consulSnapshotAgentRole` value. You should now use the `global.secretsBackend.vault.consulServerRole` for access to any Vault secrets.

FEATURES:
* Consul-dataplane:
Expand Down
7 changes: 2 additions & 5 deletions acceptance/tests/snapshot-agent/main_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package snapshotagent

import (
"fmt"
"os"
"testing"

Expand All @@ -11,8 +10,6 @@ import (
var suite testsuite.Suite

func TestMain(m *testing.M) {
fmt.Println("Skipping snapshot agent tests because it's not supported with agentless yet")
os.Exit(0)
//suite = testsuite.NewSuite(m)
//os.Exit(suite.Run())
suite = testsuite.NewSuite(m)
os.Exit(suite.Run())
}
127 changes: 64 additions & 63 deletions acceptance/tests/snapshot-agent/snapshot_agent_k8s_secret_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"testing"
"time"
Expand All @@ -15,7 +16,6 @@ import (
"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
"github.com/hashicorp/go-uuid"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand All @@ -31,83 +31,84 @@ func TestSnapshotAgent_K8sSecret(t *testing.T) {
if cfg.EnableCNI {
t.Skipf("skipping because -enable-cni is set and snapshot agent is already tested with regular tproxy")
}
ctx := suite.Environment().DefaultContext(t)
kubectlOptions := ctx.KubectlOptions(t)
ns := kubectlOptions.Namespace
releaseName := helpers.RandomName()

// Generate a bootstrap token
bootstrapToken, err := uuid.GenerateUUID()
require.NoError(t, err)
cases := map[string]struct {
secure bool
}{
"non-secure": {secure: false},
//"secure": {secure: true},
}

bsSecretName := fmt.Sprintf("%s-acl-bootstrap-token", releaseName)
bsSecretKey := "token"
saSecretName := fmt.Sprintf("%s-snapshot-agent-config", releaseName)
saSecretKey := "token"
for name, c := range cases {
t.Run(name, func(t *testing.T) {
ctx := suite.Environment().DefaultContext(t)
kubectlOptions := ctx.KubectlOptions(t)
ns := kubectlOptions.Namespace
releaseName := helpers.RandomName()

// Create cluster
helmValues := map[string]string{
"global.tls.enabled": "true",
"global.gossipEncryption.autoGenerate": "true",
"global.acls.manageSystemACLs": "true",
"global.acls.bootstrapToken.secretName": bsSecretName,
"global.acls.bootstrapToken.secretKey": bsSecretKey,
"client.snapshotAgent.enabled": "true",
"client.snapshotAgent.configSecret.secretName": saSecretName,
"client.snapshotAgent.configSecret.secretKey": saSecretKey,
}
saSecretName := fmt.Sprintf("%s-snapshot-agent-config", releaseName)
saSecretKey := "config"

// Get new cluster
consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), cfg, releaseName)
client := environment.KubernetesClientFromOptions(t, kubectlOptions)
// Create cluster
helmValues := map[string]string{
"global.tls.enabled": strconv.FormatBool(c.secure),
"global.gossipEncryption.autoGenerate": strconv.FormatBool(c.secure),
"global.acls.manageSystemACLs": strconv.FormatBool(c.secure),
"server.snapshotAgent.enabled": "true",
"server.snapshotAgent.configSecret.secretName": saSecretName,
"server.snapshotAgent.configSecret.secretKey": saSecretKey,
"connectInject.enabled": "false",
"controller.enabled": "false",
}

// Add bootstrap token secret
logger.Log(t, "Storing bootstrap token as a k8s secret")
consul.CreateK8sSecret(t, client, cfg, ns, bsSecretName, bsSecretKey, bootstrapToken)
// Get new cluster
consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), cfg, releaseName)
client := environment.KubernetesClientFromOptions(t, kubectlOptions)

// Add snapshot agent config secret
logger.Log(t, "Storing snapshot agent config as a k8s secret")
config := generateSnapshotAgentConfig(t, bootstrapToken)
logger.Logf(t, "Snapshot agent config: %s", config)
consul.CreateK8sSecret(t, client, cfg, ns, saSecretName, saSecretKey, config)
// Add snapshot agent config secret
logger.Log(t, "Storing snapshot agent config as a k8s secret")
config := generateSnapshotAgentConfig(t)
logger.Logf(t, "Snapshot agent config: %s", config)
consul.CreateK8sSecret(t, client, cfg, ns, saSecretName, saSecretKey, config)

// Create cluster
consulCluster.Create(t)
// ----------------------------------
// Create cluster
consulCluster.Create(t)
// ----------------------------------

// Validate that consul snapshot agent is running correctly and is generating snapshot files
logger.Log(t, "Confirming that Consul Snapshot Agent is generating snapshot files")
// Create k8s client from kubectl options.
// Validate that consul snapshot agent is running correctly and is generating snapshot files
logger.Log(t, "Confirming that Consul Snapshot Agent is generating snapshot files")
// Create k8s client from kubectl options.

podList, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(),
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=client-snapshot-agent,release=%s", releaseName)})
require.NoError(t, err)
require.True(t, len(podList.Items) > 0)
podList, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(),
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=server,release=%s", releaseName)})
require.NoError(t, err)
require.True(t, len(podList.Items) > 0)

// Wait for 10seconds to allow snapsot to write.
time.Sleep(10 * time.Second)
// Wait for 10seconds to allow snapshot to write.
time.Sleep(10 * time.Second)

// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
hasSnapshots := false
for _, pod := range podList.Items {
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/")
logger.Logf(t, "Snapshot: \n%s", snapshotFileListOutput)
require.NoError(t, err)
if strings.Contains(snapshotFileListOutput, ".snap") {
logger.Logf(t, "Agent pod contains snapshot files")
hasSnapshots = true
break
} else {
logger.Logf(t, "Agent pod does not contain snapshot files")
}
// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
hasSnapshots := false
for _, pod := range podList.Items {
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/tmp")
logger.Logf(t, "Snapshot: \n%s", snapshotFileListOutput)
require.NoError(t, err)
if strings.Contains(snapshotFileListOutput, ".snap") {
logger.Logf(t, "Agent pod contains snapshot files")
hasSnapshots = true
break
} else {
logger.Logf(t, "Agent pod does not contain snapshot files")
}
}
require.True(t, hasSnapshots, ".snap")
})
}
require.True(t, hasSnapshots, ".snap")
}

func generateSnapshotAgentConfig(t *testing.T, token string) string {
func generateSnapshotAgentConfig(t *testing.T) string {
config := map[string]interface{}{
"snapshot_agent": map[string]interface{}{
"token": token,
"log": map[string]interface{}{
"level": "INFO",
"enable_syslog": false,
Expand All @@ -124,7 +125,7 @@ func generateSnapshotAgentConfig(t *testing.T, token string) string {
"local_scratch_path": "",
},
"local_storage": map[string]interface{}{
"path": ".",
"path": "/tmp",
},
},
}
Expand Down
56 changes: 9 additions & 47 deletions acceptance/tests/snapshot-agent/snapshot_agent_vault_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,6 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// -------------------------
// PKI
// -------------------------
// Configure Service Mesh CA
connectCAPolicy := "connect-ca-dc1"
connectCARootPath := "connect_root"
connectCAIntermediatePath := "dc1/connect_inter"
// Configure Policy for Connect CA
vault.CreateConnectCARootAndIntermediatePKIPolicy(t, vaultClient, connectCAPolicy, connectCARootPath, connectCAIntermediatePath)

// Configure Server PKI
serverPKIConfig := &vault.PKIAndAuthRoleConfiguration{
BaseURL: "pki",
Expand Down Expand Up @@ -112,7 +105,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
bootstrapTokenSecret.SaveSecretAndAddReadPolicy(t, vaultClient)

// Snapshot Agent config
snapshotAgentConfig := generateSnapshotAgentConfig(t, bootstrapToken)
snapshotAgentConfig := generateSnapshotAgentConfig(t)
require.NoError(t, err)
snapshotAgentConfigSecret := &vault.KV2Secret{
Path: "consul/data/secret/snapshot-agent-config",
Expand All @@ -125,7 +118,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// -------------------------
// Additional Auth Roles
// -------------------------
serverPolicies := fmt.Sprintf("%s,%s,%s,%s", gossipSecret.PolicyName, connectCAPolicy, serverPKIConfig.PolicyName, bootstrapTokenSecret.PolicyName)
serverPolicies := fmt.Sprintf("%s,%s,%s,%s", gossipSecret.PolicyName, serverPKIConfig.PolicyName, bootstrapTokenSecret.PolicyName, snapshotAgentConfigSecret.PolicyName)
if cfg.EnableEnterprise {
serverPolicies += fmt.Sprintf(",%s", licenseSecret.PolicyName)
}
Expand All @@ -141,18 +134,6 @@ func TestSnapshotAgent_Vault(t *testing.T) {
}
srvAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

// client
consulClientRole := "client"
consulClientServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, "client")
clientAuthRoleConfig := &vault.KubernetesAuthRoleConfiguration{
ServiceAccountName: consulClientServiceAccountName,
KubernetesNamespace: ns,
AuthMethodPath: "kubernetes",
RoleName: consulClientRole,
PolicyNames: gossipSecret.PolicyName,
}
clientAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

// manageSystemACLs
manageSystemACLsRole := "server-acl-init"
manageSystemACLsServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, "server-acl-init")
Expand All @@ -175,42 +156,24 @@ func TestSnapshotAgent_Vault(t *testing.T) {
}
srvCAAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

// snapshot agent config
snapAgentRole := "snapshot-agent"
snapAgentServiceAccountName := fmt.Sprintf("%s-consul-%s", consulReleaseName, "snapshot-agent")
saAuthRoleConfig := &vault.KubernetesAuthRoleConfiguration{
ServiceAccountName: snapAgentServiceAccountName,
KubernetesNamespace: ns,
AuthMethodPath: "kubernetes",
RoleName: snapAgentRole,
PolicyNames: fmt.Sprintf("%s,%s", licenseSecret.PolicyName, snapshotAgentConfigSecret.PolicyName),
}
saAuthRoleConfig.ConfigureK8SAuthRole(t, vaultClient)

vaultCASecret := vault.CASecretName(vaultReleaseName)

consulHelmValues := map[string]string{
"server.extraVolumes[0].type": "secret",
"server.extraVolumes[0].name": vaultCASecret,
"server.extraVolumes[0].load": "false",

"connectInject.enabled": "true",
"connectInject.enabled": "false",
"connectInject.replicas": "1",
"controller.enabled": "true",
"controller.enabled": "false",

"global.secretsBackend.vault.enabled": "true",
"global.secretsBackend.vault.consulServerRole": consulServerRole,
"global.secretsBackend.vault.consulClientRole": consulClientRole,
"global.secretsBackend.vault.consulCARole": serverPKIConfig.RoleName,
"global.secretsBackend.vault.manageSystemACLsRole": manageSystemACLsRole,

"global.secretsBackend.vault.ca.secretName": vaultCASecret,
"global.secretsBackend.vault.ca.secretKey": "tls.crt",

"global.secretsBackend.vault.connectCA.address": vaultCluster.Address(),
"global.secretsBackend.vault.connectCA.rootPKIPath": connectCARootPath,
"global.secretsBackend.vault.connectCA.intermediatePKIPath": connectCAIntermediatePath,

"global.acls.manageSystemACLs": "true",
"global.acls.bootstrapToken.secretName": bootstrapTokenSecret.Path,
"global.acls.bootstrapToken.secretKey": bootstrapTokenSecret.Key,
Expand All @@ -220,10 +183,9 @@ func TestSnapshotAgent_Vault(t *testing.T) {
"global.tls.caCert.secretName": serverPKIConfig.CAPath,
"global.tls.enableAutoEncrypt": "true",

"client.snapshotAgent.enabled": "true",
"client.snapshotAgent.configSecret.secretName": snapshotAgentConfigSecret.Path,
"client.snapshotAgent.configSecret.secretKey": snapshotAgentConfigSecret.Key,
"global.secretsBackend.vault.consulSnapshotAgentRole": snapAgentRole,
"server.snapshotAgent.enabled": "true",
"server.snapshotAgent.configSecret.secretName": snapshotAgentConfigSecret.Path,
"server.snapshotAgent.configSecret.secretKey": snapshotAgentConfigSecret.Key,
}

if cfg.EnableEnterprise {
Expand All @@ -240,7 +202,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// Create k8s client from kubectl options.
client := environment.KubernetesClientFromOptions(t, kubectlOptions)
podList, err := client.CoreV1().Pods(kubectlOptions.Namespace).List(context.Background(),
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=client-snapshot-agent,release=%s", consulReleaseName)})
metav1.ListOptions{LabelSelector: fmt.Sprintf("app=consul,component=server,release=%s", consulReleaseName)})
require.NoError(t, err)
require.True(t, len(podList.Items) > 0)

Expand All @@ -250,7 +212,7 @@ func TestSnapshotAgent_Vault(t *testing.T) {
// Loop through snapshot agents. Only one will be the leader and have the snapshot files.
hasSnapshots := false
for _, pod := range podList.Items {
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/")
snapshotFileListOutput, err := k8s.RunKubectlAndGetOutputWithLoggerE(t, kubectlOptions, terratestLogger.Discard, "exec", pod.Name, "-c", "consul-snapshot-agent", "--", "ls", "/tmp")
logger.Logf(t, "Snapshot: \n%s", snapshotFileListOutput)
require.NoError(t, err)
if strings.Contains(snapshotFileListOutput, ".snap") {
Expand Down
Loading

0 comments on commit e4c31d7

Please sign in to comment.