From 28128f6223177b333aecafab03a052b57f88aa69 Mon Sep 17 00:00:00 2001 From: NicoletaPopoviciu <87660255+NicoletaPopoviciu@users.noreply.github.com> Date: Wed, 29 Sep 2021 11:54:25 -0400 Subject: [PATCH 01/10] Enable adding extra containers to clients and servers via helm configuration. (#749) Co-authored-by: Iryna Shustava Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> --- CHANGELOG.md | 2 + charts/consul/templates/client-daemonset.yaml | 3 + .../consul/templates/server-statefulset.yaml | 3 + charts/consul/test/unit/client-daemonset.bats | 76 ++++++++++++++++++ .../consul/test/unit/server-statefulset.bats | 77 ++++++++++++++++++- charts/consul/values.yaml | 26 +++++++ 6 files changed, 186 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 744c065350..b4de5b5ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ IMPROVEMENTS: * Control Plane * Upgrade Docker image Alpine version from 3.13 to 3.14. [[GH-737](https://github.com/hashicorp/consul-k8s/pull/737)] +* Helm Chart + * Enable adding extra containers to server and client Pods. [[GH-749](https://github.com/hashicorp/consul-k8s/pull/749)] ## 0.34.1 (September 17, 2021) diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml index 58eb73ca9e..d2a2dcb583 100644 --- a/charts/consul/templates/client-daemonset.yaml +++ b/charts/consul/templates/client-daemonset.yaml @@ -362,6 +362,9 @@ spec: securityContext: {{- toYaml .Values.client.containerSecurityContext.client | nindent 12 }} {{- end }} + {{- if .Values.client.extraContainers }} + {{ toYaml .Values.client.extraContainers | nindent 8 }} + {{- end }} {{- if (or .Values.global.acls.manageSystemACLs (and .Values.global.tls.enabled (not .Values.global.tls.enableAutoEncrypt))) }} initContainers: {{- if .Values.global.acls.manageSystemACLs }} diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml index 09668f2859..05460ed217 100644 --- a/charts/consul/templates/server-statefulset.yaml +++ b/charts/consul/templates/server-statefulset.yaml @@ -351,6 +351,9 @@ spec: securityContext: {{- toYaml .Values.server.containerSecurityContext.server | nindent 12 }} {{- end }} + {{- if .Values.server.extraContainers }} + {{ toYaml .Values.server.extraContainers | nindent 8 }} + {{- end }} {{- if .Values.server.nodeSelector }} nodeSelector: {{ tpl .Values.server.nodeSelector . | indent 8 | trim }} diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats index d9bd845765..ce202bca3e 100755 --- a/charts/consul/test/unit/client-daemonset.bats +++ b/charts/consul/test/unit/client-daemonset.bats @@ -1419,3 +1419,79 @@ rollingUpdate: [ "$status" -eq 1 ] [[ "$output" =~ "global.adminPartitions.name has to be \"default\" in the server cluster" ]] } + +#-------------------------------------------------------------------- +# extraContainers + +@test "client/DaemonSet: extraContainers adds extra container" { + cd `chart_dir` + + # Test that it defines the extra container + local object=$(helm template \ + -s templates/client-daemonset.yaml \ + --set 'client.extraContainers[0].image=test-image' \ + --set 'client.extraContainers[0].name=test-container' \ + --set 'client.extraContainers[0].ports[0].name=test-port' \ + --set 'client.extraContainers[0].ports[0].containerPort=9410' \ + --set 'client.extraContainers[0].ports[0].protocol=TCP' \ + --set 'client.extraContainers[0].env[0].name=TEST_ENV' \ + --set 'client.extraContainers[0].env[0].value=test_env_value' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[] | select(.name == "test-container")' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.name' | tee /dev/stderr) + [ "${actual}" = "test-container" ] + + local actual=$(echo $object | + yq -r '.image' | tee /dev/stderr) + [ "${actual}" = "test-image" ] + + local actual=$(echo $object | + yq -r '.ports[0].name' | tee /dev/stderr) + [ "${actual}" = "test-port" ] + + local actual=$(echo $object | + yq -r '.ports[0].containerPort' | tee /dev/stderr) + [ "${actual}" = "9410" ] + + local actual=$(echo $object | + yq -r '.ports[0].protocol' | tee /dev/stderr) + [ "${actual}" = "TCP" ] + + local actual=$(echo $object | + yq -r '.env[0].name' | tee /dev/stderr) + [ "${actual}" = "TEST_ENV" ] + + local actual=$(echo $object | + yq -r '.env[0].value' | tee /dev/stderr) + [ "${actual}" = "test_env_value" ] + +} + +@test "client/DaemonSet: extraContainers supports adding two containers" { + cd `chart_dir` + + local object=$(helm template \ + -s templates/client-daemonset.yaml \ + --set 'client.extraContainers[0].image=test-image' \ + --set 'client.extraContainers[0].name=test-container' \ + --set 'client.extraContainers[1].image=test-image' \ + --set 'client.extraContainers[1].name=test-container-2' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers | length' | tee /dev/stderr) + + [ "${object}" = 3 ] + +} + +@test "client/DaemonSet: no extra client containers added by default" { + cd `chart_dir` + + local object=$(helm template \ + -s templates/client-daemonset.yaml \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers | length' | tee /dev/stderr) + + [ "${object}" = 1 ] +} diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats index 78dd6a873e..ac2be4c2c9 100755 --- a/charts/consul/test/unit/server-statefulset.bats +++ b/charts/consul/test/unit/server-statefulset.bats @@ -1308,7 +1308,6 @@ load _helpers [ "${actual}" = '{"name":"CONSUL_LICENSE_PATH","value":"/consul/license/bar"}' ] } - @test "server/StatefulSet: -recursor can be set by global.recursors" { cd `chart_dir` local actual=$(helm template \ @@ -1318,3 +1317,79 @@ load _helpers yq -r -c '.spec.template.spec.containers[0].command | join(" ") | contains("-recursor=\"1.2.3.4\"")' | tee /dev/stderr) [ "${actual}" = "true" ] } + +#-------------------------------------------------------------------- +# extraContainers + +@test "server/StatefulSet: adds extra container" { + cd `chart_dir` + + # Test that it defines the extra container + local object=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'server.extraContainers[0].image=test-image' \ + --set 'server.extraContainers[0].name=test-container' \ + --set 'server.extraContainers[0].ports[0].name=test-port' \ + --set 'server.extraContainers[0].ports[0].containerPort=9410' \ + --set 'server.extraContainers[0].ports[0].protocol=TCP' \ + --set 'server.extraContainers[0].env[0].name=TEST_ENV' \ + --set 'server.extraContainers[0].env[0].value=test_env_value' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[] | select(.name == "test-container")' | tee /dev/stderr) + + local actual=$(echo $object | + yq -r '.name' | tee /dev/stderr) + [ "${actual}" = "test-container" ] + + local actual=$(echo $object | + yq -r '.image' | tee /dev/stderr) + [ "${actual}" = "test-image" ] + + local actual=$(echo $object | + yq -r '.ports[0].name' | tee /dev/stderr) + [ "${actual}" = "test-port" ] + + local actual=$(echo $object | + yq -r '.ports[0].containerPort' | tee /dev/stderr) + [ "${actual}" = "9410" ] + + local actual=$(echo $object | + yq -r '.ports[0].protocol' | tee /dev/stderr) + [ "${actual}" = "TCP" ] + + local actual=$(echo $object | + yq -r '.env[0].name' | tee /dev/stderr) + [ "${actual}" = "TEST_ENV" ] + + local actual=$(echo $object | + yq -r '.env[0].value' | tee /dev/stderr) + [ "${actual}" = "test_env_value" ] + +} + +@test "server/StatefulSet: adds two extra containers" { + cd `chart_dir` + + local object=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'server.extraContainers[0].image=test-image' \ + --set 'server.extraContainers[0].name=test-container' \ + --set 'server.extraContainers[1].image=test-image' \ + --set 'server.extraContainers[1].name=test-container-2' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers | length' | tee /dev/stderr) + + [ "${object}" = 3 ] + +} + +@test "server/StatefulSet: no extra containers added by default" { + cd `chart_dir` + + local object=$(helm template \ + -s templates/server-statefulset.yaml \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers | length' | tee /dev/stderr) + + [ "${object}" = 1 ] +} diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index e1c0726deb..9105565572 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -596,6 +596,19 @@ server: # @type: array extraVolumes: [] + # A list of sidecar containers. + # Example: + # + # ```yaml + # extraContainers: + # - name: extra-container + # image: example-image:latest + # command: + # - ... + # ``` + # @type: array + extraContainers: [] + # This value defines the affinity (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) # for server pods. It defaults to allowing only a single server pod on each node, which # minimizes risk of the cluster becoming unusable if a node is lost. If you need @@ -925,6 +938,19 @@ client: # @type: array extraVolumes: [] + # A list of sidecar containers. + # Example: + # + # ```yaml + # extraContainers: + # - name: extra-container + # image: example-image:latest + # command: + # - ... + # ``` + # @type: array + extraContainers: [] + # Toleration Settings for Client pods # This should be a multi-line string matching the Toleration array # in a PodSpec. From 6305e749ddcedfe49159d70bcccee008c5b1f8a1 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Thu, 30 Sep 2021 11:43:27 -0600 Subject: [PATCH 02/10] acceptance-tests-framework: create ent image from values.yaml isntead of Chart.yaml (#752) --- .../acceptance/framework/config/config.go | 43 +++++++++++++------ .../framework/config/config_test.go | 39 +++++++++-------- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/charts/consul/test/acceptance/framework/config/config.go b/charts/consul/test/acceptance/framework/config/config.go index 4aab38f7d7..5346e248cc 100644 --- a/charts/consul/test/acceptance/framework/config/config.go +++ b/charts/consul/test/acceptance/framework/config/config.go @@ -1,7 +1,6 @@ package config import ( - "errors" "fmt" "io/ioutil" "path/filepath" @@ -91,38 +90,56 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { return helmValues, nil } -// entImage parses out consul version from Chart.yaml +type values struct { + Global globalValues `yaml:"global"` +} + +type globalValues struct { + Image string `yaml:"image"` +} + +// entImage parses out consul version from values.yaml // and sets global.image to the consul enterprise image with that version. func (t *TestConfig) entImage() (string, error) { if t.helmChartPath == "" { t.helmChartPath = HelmChartPath } - // Unmarshal Chart.yaml to get appVersion (i.e. Consul version) - chart, err := ioutil.ReadFile(filepath.Join(t.helmChartPath, "Chart.yaml")) + // Unmarshal values.yaml to current global.image value. + valuesContents, err := ioutil.ReadFile(filepath.Join(t.helmChartPath, "values.yaml")) if err != nil { return "", err } - var chartMap map[string]interface{} - err = yaml.Unmarshal(chart, &chartMap) + var v values + err = yaml.Unmarshal(valuesContents, &v) if err != nil { return "", err } - appVersion, ok := chartMap["appVersion"].(string) - if !ok { - return "", errors.New("unable to cast chartMap.appVersion to string") + // Check if the image contains digest instead of a tag. + // If it does, we want to use that image instead rather than + // trying to change the tag to an enterprise tag. + if strings.Contains(v.Global.Image, "@sha256") { + return v.Global.Image, nil } + + // Otherwise, assume that we have an image tag with a version in it. + consulImageSplits := strings.Split(v.Global.Image, ":") + if len(consulImageSplits) != 2 { + return "", fmt.Errorf("could not determine consul version from global.image: %s", v.Global.Image) + } + consulImageVersion := consulImageSplits[1] + var preRelease string // Handle versions like 1.9.0-rc1. - if strings.Contains(appVersion, "-") { - split := strings.Split(appVersion, "-") - appVersion = split[0] + if strings.Contains(consulImageVersion, "-") { + split := strings.Split(consulImageVersion, "-") + consulImageVersion = split[0] preRelease = fmt.Sprintf("-%s", split[1]) } - return fmt.Sprintf("hashicorp/consul-enterprise:%s-ent%s", appVersion, preRelease), nil + return fmt.Sprintf("hashicorp/consul-enterprise:%s-ent%s", consulImageVersion, preRelease), nil } // setIfNotEmpty sets key to val in map m if value is not empty. diff --git a/charts/consul/test/acceptance/framework/config/config_test.go b/charts/consul/test/acceptance/framework/config/config_test.go index 3e1679be88..df320687b8 100644 --- a/charts/consul/test/acceptance/framework/config/config_test.go +++ b/charts/consul/test/acceptance/framework/config/config_test.go @@ -116,39 +116,42 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) { func TestConfig_HelmValuesFromConfig_EntImage(t *testing.T) { tests := []struct { - appVersion string - expImage string - expErr string + consulImage string + expImage string + expErr string }{ { - appVersion: "1.9.0", - expImage: "hashicorp/consul-enterprise:1.9.0-ent", + consulImage: "hashicorp/consul:1.9.0", + expImage: "hashicorp/consul-enterprise:1.9.0-ent", }, { - appVersion: "1.8.5-rc1", - expImage: "hashicorp/consul-enterprise:1.8.5-ent-rc1", + consulImage: "hashicorp/consul:1.8.5-rc1", + expImage: "hashicorp/consul-enterprise:1.8.5-ent-rc1", }, { - appVersion: "1.7.0-beta3", - expImage: "hashicorp/consul-enterprise:1.7.0-ent-beta3", + consulImage: "hashicorp/consul:1.7.0-beta3", + expImage: "hashicorp/consul-enterprise:1.7.0-ent-beta3", }, { - appVersion: "1", - expErr: "unable to cast chartMap.appVersion to string", + consulImage: "invalid", + expErr: "could not determine consul version from global.image: invalid", + }, + { + consulImage: "hashicorp/consul@sha256:oioi2452345kjhlkh", + expImage: "hashicorp/consul@sha256:oioi2452345kjhlkh", }, } for _, tt := range tests { - t.Run(tt.appVersion, func(t *testing.T) { + t.Run(tt.consulImage, func(t *testing.T) { - // Write Chart.yaml to a temp dir which will then get parsed. - chartYAML := fmt.Sprintf(`apiVersion: v1 -name: consul -appVersion: %s -`, tt.appVersion) + // Write values.yaml to a temp dir which will then get parsed. + valuesYAML := fmt.Sprintf(`global: + image: %s +`, tt.consulImage) tmp, err := ioutil.TempDir("", "") require.NoError(t, err) defer os.RemoveAll(tmp) - require.NoError(t, ioutil.WriteFile(filepath.Join(tmp, "Chart.yaml"), []byte(chartYAML), 0644)) + require.NoError(t, ioutil.WriteFile(filepath.Join(tmp, "values.yaml"), []byte(valuesYAML), 0644)) cfg := TestConfig{ EnableEnterprise: true, From ff27e8a0004f08ffd38031999585259c0b13ce47 Mon Sep 17 00:00:00 2001 From: Luke Kysow <1034429+lkysow@users.noreply.github.com> Date: Thu, 30 Sep 2021 13:10:48 -0700 Subject: [PATCH 03/10] Add support for version command (#741) * Use correct name for binary * Add support for version command - Also refactor the version code out into top-level version pkg to match how the control-plane is factored --- CHANGELOG.md | 4 +- cli/README.md | 2 +- cli/cmd/version/version.go | 67 ++++++++------------- cli/commands.go | 8 +++ cli/main.go | 2 +- cli/version/version.go | 50 +++++++++++++++ control-plane/subcommand/version/command.go | 2 +- 7 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 cli/version/version.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b4de5b5ce4..9f7e305e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,9 @@ IMPROVEMENTS: * Control Plane * Upgrade Docker image Alpine version from 3.13 to 3.14. [[GH-737](https://github.com/hashicorp/consul-k8s/pull/737)] * Helm Chart - * Enable adding extra containers to server and client Pods. [[GH-749](https://github.com/hashicorp/consul-k8s/pull/749)] + * Enable adding extra containers to server and client Pods. [[GH-749](https://github.com/hashicorp/consul-k8s/pull/749)] +* CLI + * Add `version` command. [[GH-741](https://github.com/hashicorp/consul-k8s/pull/741)] ## 0.34.1 (September 17, 2021) diff --git a/cli/README.md b/cli/README.md index 1820b1cdd3..cad0f51b7e 100644 --- a/cli/README.md +++ b/cli/README.md @@ -124,4 +124,4 @@ Global Options: -kubeconfig= Path to kubeconfig file. This is aliased as "-c". -``` \ No newline at end of file +``` diff --git a/cli/cmd/version/version.go b/cli/cmd/version/version.go index 8ae086e89a..417baaff4a 100644 --- a/cli/cmd/version/version.go +++ b/cli/cmd/version/version.go @@ -2,49 +2,32 @@ package version import ( "fmt" - "strings" -) + "sync" -var ( - // The git commit that was compiled. These will be filled in by the compiler. - GitCommit string - GitDescribe string - - // The main version number that is being run at the moment. - // - // Version must conform to the format expected by - // github.com/hashicorp/go-version for tests to work. - Version = "0.0.1" - - // A pre-release marker for the version. If this is "" (empty string) - // then it means that it is a final release. Otherwise, this is a pre-release - // such as "dev" (in development), "beta", "rc1", etc. - VersionPrerelease = "" + "github.com/hashicorp/consul-k8s/cli/cmd/common" ) -// GetHumanVersion composes the parts of the version in a way that's suitable -// for displaying to humans. -func GetHumanVersion() string { - version := Version - if GitDescribe != "" { - version = GitDescribe - } - - release := VersionPrerelease - if GitDescribe == "" && release == "" { - release = "dev" - } - - if release != "" { - if !strings.HasSuffix(version, "-"+release) { - // if we tagged a prerelease version then the release is in the version already - version += fmt.Sprintf("-%s", release) - } - if GitCommit != "" { - version += fmt.Sprintf(" (%s)", GitCommit) - } - } - - // Strip off any single quotes added by the git information. - return strings.Replace(version, "'", "", -1) +type Command struct { + *common.BaseCommand + + Version string + once sync.Once +} + +func (c *Command) init() { + c.Init() +} + +func (c *Command) Run(_ []string) int { + c.once.Do(c.init) + c.UI.Output(fmt.Sprintf("consul-k8s %s", c.Version)) + return 0 +} + +func (c *Command) Synopsis() string { + return "Prints the version of the CLI." +} + +func (c *Command) Help() string { + return "Usage: consul version [options]\n" } diff --git a/cli/commands.go b/cli/commands.go index 2725c5d4bf..3fc4a7d936 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -6,6 +6,8 @@ import ( "github.com/hashicorp/consul-k8s/cli/cmd/common" "github.com/hashicorp/consul-k8s/cli/cmd/install" "github.com/hashicorp/consul-k8s/cli/cmd/uninstall" + cmdversion "github.com/hashicorp/consul-k8s/cli/cmd/version" + "github.com/hashicorp/consul-k8s/cli/version" "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" ) @@ -28,6 +30,12 @@ func initializeCommands(ctx context.Context, log hclog.Logger) (*common.BaseComm BaseCommand: baseCommand, }, nil }, + "version": func() (cli.Command, error) { + return &cmdversion.Command{ + BaseCommand: baseCommand, + Version: version.GetHumanVersion(), + }, nil + }, } return baseCommand, commands diff --git a/cli/main.go b/cli/main.go index 1285e7625c..0c3705f0df 100644 --- a/cli/main.go +++ b/cli/main.go @@ -6,7 +6,7 @@ import ( "os/signal" "syscall" - "github.com/hashicorp/consul-k8s/cli/cmd/version" + "github.com/hashicorp/consul-k8s/cli/version" "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" ) diff --git a/cli/version/version.go b/cli/version/version.go new file mode 100644 index 0000000000..6194e7ee7c --- /dev/null +++ b/cli/version/version.go @@ -0,0 +1,50 @@ +package version + +import ( + "fmt" + "strings" +) + +var ( + // The git commit that was compiled. These will be filled in by the compiler. + GitCommit string + GitDescribe string + + // The main version number that is being run at the moment. + // + // Version must conform to the format expected by + // github.com/hashicorp/go-version for tests to work. + Version = "0.34.1" + + // A pre-release marker for the version. If this is "" (empty string) + // then it means that it is a final release. Otherwise, this is a pre-release + // such as "dev" (in development), "beta", "rc1", etc. + VersionPrerelease = "dev" +) + +// GetHumanVersion composes the parts of the version in a way that's suitable +// for displaying to humans. +func GetHumanVersion() string { + version := Version + if GitDescribe != "" { + version = GitDescribe + } + + release := VersionPrerelease + if GitDescribe == "" && release == "" { + release = "dev" + } + + if release != "" { + if !strings.HasSuffix(version, "-"+release) { + // if we tagged a prerelease version then the release is in the version already + version += fmt.Sprintf("-%s", release) + } + if GitCommit != "" { + version += fmt.Sprintf(" (%s)", GitCommit) + } + } + + // Strip off any single quotes added by the git information. + return strings.Replace(version, "'", "", -1) +} diff --git a/control-plane/subcommand/version/command.go b/control-plane/subcommand/version/command.go index 3b53129762..58768a1f92 100644 --- a/control-plane/subcommand/version/command.go +++ b/control-plane/subcommand/version/command.go @@ -12,7 +12,7 @@ type Command struct { } func (c *Command) Run(_ []string) int { - c.UI.Output(fmt.Sprintf("consul-k8s %s", c.Version)) + c.UI.Output(fmt.Sprintf("consul-k8s-control-plane %s", c.Version)) return 0 } From 6254b1fbb88ba72b20b22da4216a2ff3e61a4fac Mon Sep 17 00:00:00 2001 From: Nitya Dhanushkodi Date: Thu, 30 Sep 2021 14:14:45 -0700 Subject: [PATCH 04/10] ci: fix installing helm prerequisite (#753) Grab helm binary from releases page rather than apt installing it. The apt install stopped working because of an expired certificate on baltocdn: https://app.circleci.com/pipelines/github/hashicorp/consul-k8s/2798/workflows/54156ea4-54c6-4622-a4ec-4a623a266cb5/jobs/18933 --- .circleci/config.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4da6ac895..7f76b6b469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,11 +45,10 @@ commands: chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl - curl https://baltocdn.com/helm/signing.asc | sudo apt-key add - - sudo apt-get install apt-transport-https --yes - echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list - sudo apt-get update - sudo apt-get install helm + wget https://get.helm.sh/helm-v3.7.0-linux-amd64.tar.gz + tar -zxvf helm-v3.7.0-linux-amd64.tar.gz + sudo mv linux-amd64/helm /usr/local/bin/helm + create-kind-clusters: parameters: version: From a376f18917e60242db517377a2d4ee258b0faf1b Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Fri, 1 Oct 2021 11:43:41 -0400 Subject: [PATCH 05/10] acceptance tests for admin partitions (#747) * Add acceptance tests for Admin Partitions - This tests is a multi cluster test that uses 2 Kubernetes clusters. Servers are installed on one of them and only agents in another. - It asserts that the agents in the agent-only cluster are created in the correct partition. - Updates have been made to the metrics test to account for partitions being emmitted by the envoy metrics. --- .../kustomization.yaml | 5 + .../cases/static-client-partition/patch.yaml | 10 ++ .../acceptance/tests/metrics/metrics_test.go | 8 +- .../acceptance/tests/partitions/main_test.go | 22 +++ .../tests/partitions/partitions_test.go | 146 ++++++++++++++++++ 5 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/kustomization.yaml create mode 100644 charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/patch.yaml create mode 100644 charts/consul/test/acceptance/tests/partitions/main_test.go create mode 100644 charts/consul/test/acceptance/tests/partitions/partitions_test.go diff --git a/charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/kustomization.yaml b/charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/kustomization.yaml new file mode 100644 index 0000000000..974fbd4fe1 --- /dev/null +++ b/charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/patch.yaml b/charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/patch.yaml new file mode 100644 index 0000000000..42857c3d2b --- /dev/null +++ b/charts/consul/test/acceptance/tests/fixtures/cases/static-client-partition/patch.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "consul.hashicorp.com/connect-service-upstreams": "static-server.default.secondary:1234" \ No newline at end of file diff --git a/charts/consul/test/acceptance/tests/metrics/metrics_test.go b/charts/consul/test/acceptance/tests/metrics/metrics_test.go index d4b3469d1d..44bec20f8b 100644 --- a/charts/consul/test/acceptance/tests/metrics/metrics_test.go +++ b/charts/consul/test/acceptance/tests/metrics/metrics_test.go @@ -75,13 +75,13 @@ func TestComponentMetrics(t *testing.T) { require.Contains(t, metricsOutput, `consul_acl_ResolveToken{quantile="0.5"}`) // Ingress Gateway Metrics - assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway",consul_source_namespace="default",consul_source_datacenter="dc1",envoy_cluster_name="local_agent"} 0`) + assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway"`) // Terminating Gateway Metrics - assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway",consul_source_namespace="default",consul_source_datacenter="dc1",envoy_cluster_name="local_agent"} 0`) + assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway"`) // Mesh Gateway Metrics - assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway",consul_source_namespace="default",consul_source_datacenter="dc1",envoy_cluster_name="local_agent"} 0`) + assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway"`) } // Test that merged service and envoy metrics are accessible from the @@ -124,7 +124,7 @@ func TestAppMetrics(t *testing.T) { metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+staticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP)) require.NoError(t, err) // This assertion represents the metrics from the envoy sidecar. - require.Contains(t, metricsOutput, `envoy_cluster_assignment_stale{local_cluster="server",consul_source_service="server",consul_source_namespace="default",consul_source_datacenter="dc1",envoy_cluster_name="local_agent"} 0`) + require.Contains(t, metricsOutput, `envoy_cluster_assignment_stale{local_cluster="server",consul_source_service="server"`) // This assertion represents the metrics from the application. require.Contains(t, metricsOutput, `service_started_total 1`) } diff --git a/charts/consul/test/acceptance/tests/partitions/main_test.go b/charts/consul/test/acceptance/tests/partitions/main_test.go new file mode 100644 index 0000000000..c26f293f10 --- /dev/null +++ b/charts/consul/test/acceptance/tests/partitions/main_test.go @@ -0,0 +1,22 @@ +package partitions + +import ( + "fmt" + "os" + "testing" + + testsuite "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/suite" +) + +var suite testsuite.Suite + +func TestMain(m *testing.M) { + suite = testsuite.NewSuite(m) + + if suite.Config().EnableMultiCluster { + os.Exit(suite.Run()) + } else { + fmt.Println("Skipping partitions tests because -enable-multi-cluster is not set") + os.Exit(0) + } +} diff --git a/charts/consul/test/acceptance/tests/partitions/partitions_test.go b/charts/consul/test/acceptance/tests/partitions/partitions_test.go new file mode 100644 index 0000000000..bb6c1553eb --- /dev/null +++ b/charts/consul/test/acceptance/tests/partitions/partitions_test.go @@ -0,0 +1,146 @@ +package partitions + +import ( + "context" + "fmt" + "testing" + + "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/environment" + "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/logger" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Test that Connect works in a default installation. +// i.e. without ACLs because TLS is required for setting up Admin Partitions. +func TestPartitions(t *testing.T) { + env := suite.Environment() + cfg := suite.Config() + + if !cfg.EnableEnterprise { + t.Skipf("skipping this test because -enable-enterprise is not set") + } + + if !cfg.UseKind { + t.Skipf("skipping this test because Admin Partition tests are only supported in Kind for now") + } + + primaryContext := env.DefaultContext(t) + secondaryContext := env.Context(t, environment.SecondaryContextName) + + ctx := context.Background() + + primaryHelmValues := map[string]string{ + "global.datacenter": "dc1", + "global.image": "hashicorp/consul-enterprise:1.11.0-ent-alpha", + + "global.adminPartitions.enabled": "true", + "global.enableConsulNamespaces": "true", + "global.tls.enabled": "true", + + "server.exposeGossipAndRPCPorts": "true", + + "connectInject.enabled": "true", + } + + if cfg.UseKind { + primaryHelmValues["global.adminPartitions.service.type"] = "NodePort" + primaryHelmValues["global.adminPartitions.service.nodePort.https"] = "30000" + } + + releaseName := helpers.RandomName() + + // Install the consul cluster with servers in the default kubernetes context. + primaryConsulCluster := consul.NewHelmCluster(t, primaryHelmValues, primaryContext, cfg, releaseName) + primaryConsulCluster.Create(t) + + // Get the TLS CA certificate and key secret from the primary cluster and apply it to secondary cluster + tlsCert := fmt.Sprintf("%s-consul-ca-cert", releaseName) + logger.Logf(t, "retrieving ca cert secret %s from the primary cluster and applying to the secondary", tlsCert) + caCertSecret, err := primaryContext.KubernetesClient(t).CoreV1().Secrets(primaryContext.KubectlOptions(t).Namespace).Get(ctx, tlsCert, metav1.GetOptions{}) + caCertSecret.ResourceVersion = "" + require.NoError(t, err) + _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(ctx, caCertSecret, metav1.CreateOptions{}) + require.NoError(t, err) + + tlsKey := fmt.Sprintf("%s-consul-ca-key", releaseName) + logger.Logf(t, "retrieving ca key secret %s from the primary cluster and applying to the secondary", tlsKey) + caKeySecret, err := primaryContext.KubernetesClient(t).CoreV1().Secrets(primaryContext.KubectlOptions(t).Namespace).Get(ctx, tlsKey, metav1.GetOptions{}) + caKeySecret.ResourceVersion = "" + require.NoError(t, err) + _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(ctx, caKeySecret, metav1.CreateOptions{}) + require.NoError(t, err) + + var partitionSvcIP string + if !cfg.UseKind { + // Get the IP of the partition service to configure the external server address in the values file for the workload cluster. + partitionServiceName := fmt.Sprintf("%s-partition-secret", releaseName) + logger.Logf(t, "retrieving partition service to determine external IP for servers") + partitionsSvc, err := primaryContext.KubernetesClient(t).CoreV1().Services(primaryContext.KubectlOptions(t).Namespace).Get(ctx, partitionServiceName, metav1.GetOptions{}) + require.NoError(t, err) + partitionSvcIP = partitionsSvc.Status.LoadBalancer.Ingress[0].IP + } else { + nodeList, err := primaryContext.KubernetesClient(t).CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + require.NoError(t, err) + // Get the address of the (only) node from the Kind cluster. + partitionSvcIP = nodeList.Items[0].Status.Addresses[0].Address + } + + // Create secondary cluster + secondaryHelmValues := map[string]string{ + "global.datacenter": "dc1", + "global.image": "hashicorp/consul-enterprise:1.11.0-ent-alpha", + "global.enabled": "false", + + "global.adminPartitions.enabled": "true", + "global.adminPartitions.name": "secondary", + "global.enableConsulNamespaces": "true", + + "global.tls.enabled": "true", + "global.tls.caCert.secretName": tlsCert, + "global.tls.caCert.secretKey": "tls.crt", + "global.tls.caKey.secretName": tlsKey, + "global.tls.caKey.secretKey": "tls.key", + + "externalServers.enabled": "true", + "externalServers.hosts[0]": partitionSvcIP, + "externalServers.tlsServerName": "server.dc1.consul", + + "client.enabled": "true", + "client.exposeGossipPorts": "true", + "client.join[0]": partitionSvcIP, + + "connectInject.enabled": "true", + } + + if cfg.UseKind { + secondaryHelmValues["externalServers.httpsPort"] = "30000" + } + + // Install the consul cluster without servers in the secondary kubernetes context. + secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName) + secondaryConsulCluster.Create(t) + + agentPodList, err := secondaryContext.KubernetesClient(t).CoreV1().Pods(secondaryContext.KubectlOptions(t).Namespace).List(ctx, metav1.ListOptions{LabelSelector: "app=consul,component=client"}) + require.NoError(t, err) + require.Len(t, agentPodList.Items, 1) + + output, err := k8s.RunKubectlAndGetOutputE(t, secondaryContext.KubectlOptions(t), "logs", agentPodList.Items[0].Name, "-n", secondaryContext.KubectlOptions(t).Namespace) + require.NoError(t, err) + require.Contains(t, output, "Partition: 'secondary'") + + // TODO: These can be enabled once mesh gateways are used for communication between services. Currently we cant setup a flat pod network on Kind. + // Check that we can connect services over the mesh gateways + + //logger.Log(t, "creating static-server in workload cluster") + //k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + + //logger.Log(t, "creating static-client in server cluster") + //k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-partition") + + //logger.Log(t, "checking that connection is successful") + //k8s.CheckStaticServerConnectionSuccessful(t, primaryContext.KubectlOptions(t), "http://localhost:1234") +} From 237e02ec5e1c71331ea4338f0963e126067b7995 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Fri, 1 Oct 2021 13:54:58 -0400 Subject: [PATCH 06/10] Automatically Generate and Use Gossip Encryption Key (#738) * Add global.gossipEncryption.autogenerate to values.yaml * Add description of key autogen * Add initial autogen-encryption-job yaml * Fix run condition on autogen * autogen is false by default * Set secretName/Key correctly, flesh out curling k8s a bit * Add some basic bats tests * Add tests for user set values for secretName and secretKey * Beef up comment for gossip encryption * Add notes * Update values.yaml to include autoGenerate * Add gossip-encryption-autogen-job.yaml * Port over autogen-encryption-job to gossip-encryption-autogen-job * Rename autogen-encryption-job.bats to gossip-encryption-autogen-job.bats * Remove change made to statefulset * Add check that secretName and secretKey are not set * Fix test failures * Set GOSSIP_KEY properly in server statefulset * Remove setting secretName and secretKey for autogen * Check if secret exists via 200 resp * Add gossip autogen to client daemonset * Send the key to secrets * Base64 encrypt consul key * Add podsecuritypolicy, role, rolebinding, and SA * Rename -gossip-encryption-autogen to -gossip-encryption-autogenerate * Rename *-autogen-* to *-autogeneration-* * Remove text about respecting user set secretName and secretKey for autogen * Rename gossip-encryption-autogen to gossip-encryption-autogeneration * Add some great tests! * Update changelog * Fix filename reference in job bats file * Update charts/consul/templates/gossip-encryption-autogeneration-job.yaml Co-authored-by: Iryna Shustava * Update charts/consul/test/unit/gossip-encryption-autogeneration-job.bats Co-authored-by: Iryna Shustava * Don't do the pre-check, but don't replace the current secret * Update charts/consul/test/unit/gossip-encryption-autogeneration-job.bats Co-authored-by: Iryna Shustava * Only give role create and get perms * Update charts/consul/test/unit/gossip-encryption-autogeneration-podsecurity.bats Co-authored-by: Iryna Shustava * Update charts/consul/values.yaml Co-authored-by: Iryna Shustava * Return kubectl command to what it was * Test that GOSSIP_KEY gets passed in on the encrypt flag * Autogen job does not run as root * Add check that gossip encryption is getting set * Fix test for gossip encryption in acceptance tests * Update charts/consul/test/acceptance/tests/basic/basic_test.go Co-authored-by: Ashwin Venkatesh * Rename s/autogeneration/autogenerate/ for gossip-encryption-autogeneration-* * Use correct filename in bats * Rename podsecuritypolicy test to be correct * Change v1 to metaV1 * Update charts/consul/test/acceptance/tests/basic/basic_test.go Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> * Test the other keyring value * Remove arbitrary test * Remove document separator * Remove temp dir from job * Remove get perm from autogenerate role * Add -encrypt flag test for client-daemonset * Change autogen to a feature in the CHANGELOG * Update comment on values.yaml * Update charts/consul/test/acceptance/tests/basic/basic_test.go Co-authored-by: Ashwin Venkatesh Co-authored-by: Iryna Shustava Co-authored-by: Ashwin Venkatesh Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com> --- CHANGELOG.md | 4 ++ charts/consul/templates/client-daemonset.yaml | 9 ++- .../gossip-encryption-autogenerate-job.yaml | 71 +++++++++++++++++++ ...yption-autogenerate-podsecuritypolicy.yaml | 39 ++++++++++ .../gossip-encryption-autogenerate-role.yaml | 30 ++++++++ ...p-encryption-autogenerate-rolebinding.yaml | 22 ++++++ ...ncryption-autogenerate-serviceaccount.yaml | 21 ++++++ .../consul/templates/server-statefulset.yaml | 9 ++- .../test/acceptance/tests/basic/basic_test.go | 28 +++++++- charts/consul/test/unit/client-daemonset.bats | 21 ++++++ .../gossip-encryption-autogenerate-job.bats | 63 ++++++++++++++++ ...yption-autogenerate-podsecuritypolicy.bats | 28 ++++++++ .../gossip-encryption-autogenerate-role.bats | 28 ++++++++ ...p-encryption-autogenerate-rolebinding.bats | 29 ++++++++ ...ncryption-autogenerate-serviceaccount.bats | 50 +++++++++++++ .../consul/test/unit/server-statefulset.bats | 22 ++++++ charts/consul/values.yaml | 27 +++---- 17 files changed, 478 insertions(+), 23 deletions(-) create mode 100644 charts/consul/templates/gossip-encryption-autogenerate-job.yaml create mode 100644 charts/consul/templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml create mode 100644 charts/consul/templates/gossip-encryption-autogenerate-role.yaml create mode 100644 charts/consul/templates/gossip-encryption-autogenerate-rolebinding.yaml create mode 100644 charts/consul/templates/gossip-encryption-autogenerate-serviceaccount.yaml create mode 100644 charts/consul/test/unit/gossip-encryption-autogenerate-job.bats create mode 100644 charts/consul/test/unit/gossip-encryption-autogenerate-podsecuritypolicy.bats create mode 100644 charts/consul/test/unit/gossip-encryption-autogenerate-role.bats create mode 100644 charts/consul/test/unit/gossip-encryption-autogenerate-rolebinding.bats create mode 100644 charts/consul/test/unit/gossip-encryption-autogenerate-serviceaccount.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f7e305e76..7bc08371b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## UNRELEASED +FEATURES: +* Helm Chart + * Add automatic generation of gossip encryption with `global.gossipEncryption.autoGenerate=true`. [[GH-738](https://github.com/hashicorp/consul-k8s/pull/738)] + IMPROVEMENTS: * Control Plane * Upgrade Docker image Alpine version from 3.13 to 3.14. [[GH-737](https://github.com/hashicorp/consul-k8s/pull/737)] diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml index d2a2dcb583..d4e1c05107 100644 --- a/charts/consul/templates/client-daemonset.yaml +++ b/charts/consul/templates/client-daemonset.yaml @@ -168,12 +168,17 @@ spec: fieldPath: status.podIP - name: CONSUL_DISABLE_PERM_MGMT value: "true" - {{- if (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} + {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }} - name: GOSSIP_KEY valueFrom: secretKeyRef: + {{- if .Values.global.gossipEncryption.autoGenerate }} + name: {{ template "consul.fullname" . }}-gossip-encryption-key + key: key + {{- else if (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} name: {{ .Values.global.gossipEncryption.secretName }} key: {{ .Values.global.gossipEncryption.secretKey }} + {{- end }} {{- end }} {{- if (and .Values.server.enterpriseLicense.secretName .Values.server.enterpriseLicense.secretKey .Values.server.enterpriseLicense.enableLicenseAutoload (not .Values.global.acls.manageSystemACLs)) }} - name: CONSUL_LICENSE_PATH @@ -252,7 +257,7 @@ spec: {{- end }} -datacenter={{ .Values.global.datacenter }} \ -data-dir=/consul/data \ - {{- if (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} + {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }} -encrypt="${GOSSIP_KEY}" \ {{- end }} {{- if .Values.client.join }} diff --git a/charts/consul/templates/gossip-encryption-autogenerate-job.yaml b/charts/consul/templates/gossip-encryption-autogenerate-job.yaml new file mode 100644 index 0000000000..ae5c0674d1 --- /dev/null +++ b/charts/consul/templates/gossip-encryption-autogenerate-job.yaml @@ -0,0 +1,71 @@ +{{- if .Values.global.gossipEncryption.autoGenerate }} +{{- if (or .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} + {{ fail "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." }} +{{ end }} +# automatically generate encryption key for gossip protocol and save it in Kubernetes secret +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + release: {{ .Release.Name }} + component: gossip-encryption-autogeneneration + annotations: + "consul.hashicorp.com/connect-inject": "false" + spec: + restartPolicy: Never + serviceAccountName: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 100 + fsGroup: 1000 + containers: + - name: gossip-encryption-autogen + image: "{{ .Values.global.image }}" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # We're using POST requests below to create secrets via Kubernetes API. + # Note that in the subsequent runs of the job, POST requests will + # return a 409 because these secrets would already exist; + # we are ignoring these response codes. + command: + - "/bin/sh" + - "-ec" + - | + secretName={{ template "consul.fullname" . }}-gossip-encryption-key + secretKey=key + keyValue=$(consul keygen | base64) + curl -s -X POST --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ + https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}/api/v1/namespaces/${NAMESPACE}/secrets \ + -H "Authorization: Bearer $( cat /var/run/secrets/kubernetes.io/serviceaccount/token )" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -d "{ \"kind\": \"Secret\", \"apiVersion\": \"v1\", \"metadata\": { \"name\": \"${secretName}\", \"namespace\": \"${NAMESPACE}\" }, \"type\": \"Opaque\", \"data\": { \"${secretKey}\": \"${keyValue}\" }}" > /dev/null + resources: + requests: + memory: "50Mi" + cpu: "50m" + limits: + memory: "50Mi" + cpu: "50m" +{{- end }} diff --git a/charts/consul/templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml b/charts/consul/templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml new file mode 100644 index 0000000000..6121fbbe30 --- /dev/null +++ b/charts/consul/templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml @@ -0,0 +1,39 @@ +{{- if .Values.global.gossipEncryption.autoGenerate }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation +spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'secret' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/consul/templates/gossip-encryption-autogenerate-role.yaml b/charts/consul/templates/gossip-encryption-autogenerate-role.yaml new file mode 100644 index 0000000000..5f9d354f38 --- /dev/null +++ b/charts/consul/templates/gossip-encryption-autogenerate-role.yaml @@ -0,0 +1,30 @@ +{{- if .Values.global.gossipEncryption.autoGenerate }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation +rules: +- apiGroups: [""] + resources: + - secrets + verbs: + - create +{{- if .Values.global.enablePodSecurityPolicies }} +- apiGroups: ["policy"] + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - {{ template "consul.fullname" . }}-gossip-encryption-autogenerate +{{- end }} +{{- end }} diff --git a/charts/consul/templates/gossip-encryption-autogenerate-rolebinding.yaml b/charts/consul/templates/gossip-encryption-autogenerate-rolebinding.yaml new file mode 100644 index 0000000000..caef0d221e --- /dev/null +++ b/charts/consul/templates/gossip-encryption-autogenerate-rolebinding.yaml @@ -0,0 +1,22 @@ +{{- if .Values.global.gossipEncryption.autoGenerate }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate +subjects: +- kind: ServiceAccount + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate +{{- end }} diff --git a/charts/consul/templates/gossip-encryption-autogenerate-serviceaccount.yaml b/charts/consul/templates/gossip-encryption-autogenerate-serviceaccount.yaml new file mode 100644 index 0000000000..a711f9a4c1 --- /dev/null +++ b/charts/consul/templates/gossip-encryption-autogenerate-serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- if .Values.global.gossipEncryption.autoGenerate }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "consul.fullname" . }}-gossip-encryption-autogenerate + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation +{{- with .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range . }} + - name: {{ .name }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml index 05460ed217..fe2dcff468 100644 --- a/charts/consul/templates/server-statefulset.yaml +++ b/charts/consul/templates/server-statefulset.yaml @@ -156,12 +156,17 @@ spec: fieldPath: metadata.namespace - name: CONSUL_DISABLE_PERM_MGMT value: "true" - {{- if (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} + {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }} - name: GOSSIP_KEY valueFrom: secretKeyRef: + {{- if .Values.global.gossipEncryption.autoGenerate }} + name: {{ template "consul.fullname" . }}-gossip-encryption-key + key: key + {{- else if (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} name: {{ .Values.global.gossipEncryption.secretName }} key: {{ .Values.global.gossipEncryption.secretKey }} + {{- end }} {{- end }} {{- if .Values.global.tls.enabled }} - name: CONSUL_HTTP_ADDR @@ -223,7 +228,7 @@ spec: -datacenter={{ .Values.global.datacenter }} \ -data-dir=/consul/data \ -domain={{ .Values.global.domain }} \ - {{- if (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey) }} + {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }} -encrypt="${GOSSIP_KEY}" \ {{- end }} {{- if .Values.server.connect }} diff --git a/charts/consul/test/acceptance/tests/basic/basic_test.go b/charts/consul/test/acceptance/tests/basic/basic_test.go index cc3cdc9877..95861e5f98 100644 --- a/charts/consul/test/acceptance/tests/basic/basic_test.go +++ b/charts/consul/test/acceptance/tests/basic/basic_test.go @@ -1,8 +1,10 @@ package basic import ( + "context" "fmt" "strconv" + "strings" "testing" "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/consul" @@ -10,6 +12,7 @@ import ( "github.com/hashicorp/consul-k8s/charts/consul/test/acceptance/framework/logger" "github.com/hashicorp/consul/api" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Test that the basic installation, i.e. just @@ -39,9 +42,10 @@ func TestBasicInstallation(t *testing.T) { t.Run(name, func(t *testing.T) { releaseName := helpers.RandomName() helmValues := map[string]string{ - "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), - "global.tls.enabled": strconv.FormatBool(c.secure), - "global.tls.enableAutoEncrypt": strconv.FormatBool(c.autoEncrypt), + "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), + "global.tls.enabled": strconv.FormatBool(c.secure), + "global.gossipEncryption.autoGenerate": strconv.FormatBool(c.secure), + "global.tls.enableAutoEncrypt": strconv.FormatBool(c.autoEncrypt), } consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), suite.Config(), releaseName) @@ -63,6 +67,24 @@ func TestBasicInstallation(t *testing.T) { kv, _, err := client.KV().Get(randomKey, nil) require.NoError(t, err) require.Equal(t, kv.Value, randomValue) + + // Check that autogenerated gossip encryption key is being used + if c.secure { + secretName := fmt.Sprintf("%s-consul-gossip-encryption-key", releaseName) + secretKey := "key" + + keyring, err := client.Operator().KeyringList(nil) + require.NoError(t, err) + + testContext := suite.Environment().DefaultContext(t) + secret, err := testContext.KubernetesClient(t).CoreV1().Secrets(testContext.KubectlOptions(t).Namespace).Get(context.Background(), secretName, metav1.GetOptions{}) + require.NoError(t, err) + gossipEncryptionKey := strings.TrimSpace(string(secret.Data[secretKey])) + + require.Len(t, keyring, 2) + require.Contains(t, keyring[0].Keys, gossipEncryptionKey) + require.Contains(t, keyring[1].Keys, gossipEncryptionKey) + } }) } } diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats index ce202bca3e..cbbbc10ef9 100755 --- a/charts/consul/test/unit/client-daemonset.bats +++ b/charts/consul/test/unit/client-daemonset.bats @@ -606,6 +606,27 @@ load _helpers [ "${actual}" = "" ] } +@test "client/DaemonSet: gossip encryption autogeneration properly sets secretName and secretKey" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-daemonset.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | .valueFrom.secretKeyRef | [.name=="RELEASE-NAME-consul-gossip-encryption-key", .key="key"] | all' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "client/DaemonSet: gossip encryption key is passed in via the -encrypt flag" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/client-daemonset.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[] | select(.name=="consul") | .command | any(contains("-encrypt=\"${GOSSIP_KEY}\""))' + | tee /dev/stderr) + [ "${actual}" = "true" ] +} + @test "client/DaemonSet: gossip encryption disabled in client DaemonSet when secretName is missing" { cd `chart_dir` local actual=$(helm template \ diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats new file mode 100644 index 0000000000..b78b9c231d --- /dev/null +++ b/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats @@ -0,0 +1,63 @@ +#!/usr/bin/env bats + +load _helpers + +@test "gossipEncryptionAutogenerate/Job: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-job.yaml \ + . +} + +@test "gossipEncryptionAutogenerate/Job: enabled with global.gossipEncryption.autoGenerate=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/gossip-encryption-autogenerate-job.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "gossipEncryptionAutogenerate/Job: disabled when global.gossipEncryption.autoGenerate=false" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-job.yaml \ + --set 'global.gossipEncryption.autoGenerate=false' \ + . +} + +@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretName and global.gossipEncryption.secretKey are set" { + cd `chart_dir` + run helm template \ + -s templates/gossip-encryption-autogenerate-job.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + --set 'global.gossipEncryption.secretName=name' \ + --set 'global.gossipEncryption.secretKey=key' \ + . + [ "$status" -eq 1 ] + [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]] +} + +@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretName is set" { + cd `chart_dir` + run helm template \ + -s templates/gossip-encryption-autogenerate-job.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + --set 'global.gossipEncryption.secretName=name' \ + . + [ "$status" -eq 1 ] + [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]] +} + +@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretKey is set" { + cd `chart_dir` + run helm template \ + -s templates/gossip-encryption-autogenerate-job.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + --set 'global.gossipEncryption.secretKey=key' \ + . + [ "$status" -eq 1 ] + [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]] +} + diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-podsecuritypolicy.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-podsecuritypolicy.bats new file mode 100644 index 0000000000..810147bed3 --- /dev/null +++ b/charts/consul/test/unit/gossip-encryption-autogenerate-podsecuritypolicy.bats @@ -0,0 +1,28 @@ +#!/usr/bin/env bats + +load _helpers + +@test "gossipEncryptionAutogenerate/PodSecurityPolicy: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml \ + . +} + +@test "gossipEncryptionAutogenerate/PodSecurityPolicy: disabled with global.gossipEncryption.autoGenerate=false" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml \ + --set 'global.gossipEncryption.autoGenerate=false' \ + . +} + +@test "gossipEncryptionAutogenerate/PodSecurityPolicy: enabled with global.gossipEncryption.autoGenerate=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/gossip-encryption-autogenerate-podsecuritypolicy.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq -s 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-role.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-role.bats new file mode 100644 index 0000000000..7707a872f0 --- /dev/null +++ b/charts/consul/test/unit/gossip-encryption-autogenerate-role.bats @@ -0,0 +1,28 @@ +#!/usr/bin/env bats + +load _helpers + +@test "gossipEncryptionAutogenerate/Role: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-role.yaml \ + . +} + +@test "gossipEncryptionAutogenerate/Role: disabled with global.gossipEncryption.autoGenerate=false" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-role.yaml \ + --set 'global.gossipEncryption.autoGenerate=false' \ + . +} + +@test "gossipEncryptionAutogenerate/Role: enabled when global.gossipEncryption.autoGenerate=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/gossip-encryption-autogenerate-role.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-rolebinding.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-rolebinding.bats new file mode 100644 index 0000000000..9847beaa24 --- /dev/null +++ b/charts/consul/test/unit/gossip-encryption-autogenerate-rolebinding.bats @@ -0,0 +1,29 @@ + +#!/usr/bin/env bats + +load _helpers + +@test "gossipEncryptionAutogenerate/RoleBinding: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-rolebinding.yaml \ + . +} + +@test "gossipEncryptionAutogenerate/RoleBinding: disabled with global.gossipEncryption.autoGenerate=false" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-rolebinding.yaml \ + --set 'global.gossipEncryption.autoGenerate=false' \ + . +} + +@test "gossipEncryptionAutogenerate/RoleBinding: enabled with global.gossipEncryption.autoGenerate=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/gossip-encryption-autogenerate-rolebinding.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-serviceaccount.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-serviceaccount.bats new file mode 100644 index 0000000000..782d1b1ad6 --- /dev/null +++ b/charts/consul/test/unit/gossip-encryption-autogenerate-serviceaccount.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats + +load _helpers + +@test "gossipEncryptionAutogenerate/ServiceAccount: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-serviceaccount.yaml \ + . +} + +@test "gossipEncryptionAutogenerate/ServiceAccount: disabled with global.gossipEncryption.autoGenerate=false" { + cd `chart_dir` + assert_empty helm template \ + -s templates/gossip-encryption-autogenerate-serviceaccount.yaml \ + --set 'global.gossipEncryption.autoGenerate=false' \ + . +} + +@test "gossipEncryptionAutogenerate/ServiceAccount: enabled with global.gossipEncryption.autoGenerate=true" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/gossip-encryption-autogenerate-serviceaccount.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# global.imagePullSecrets + +@test "gossipEncryptionAutogenerate/ServiceAccount: can set image pull secrets" { + cd `chart_dir` + local object=$(helm template \ + -s templates/gossip-encryption-autogenerate-serviceaccount.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + --set 'global.imagePullSecrets[0].name=my-secret' \ + --set 'global.imagePullSecrets[1].name=my-secret2' \ + . | tee /dev/stderr) + + local actual=$(echo "$object" | + yq -r '.imagePullSecrets[0].name' | tee /dev/stderr) + [ "${actual}" = "my-secret" ] + + local actual=$(echo "$object" | + yq -r '.imagePullSecrets[1].name' | tee /dev/stderr) + [ "${actual}" = "my-secret2" ] +} + diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats index ac2be4c2c9..a7e48a3951 100755 --- a/charts/consul/test/unit/server-statefulset.bats +++ b/charts/consul/test/unit/server-statefulset.bats @@ -842,6 +842,28 @@ load _helpers [ "${actual}" = "" ] } +@test "server/StatefulSet: gossip encryption autogeneration properly sets secretName and secretKey" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | .valueFrom.secretKeyRef | [.name=="RELEASE-NAME-consul-gossip-encryption-key", .key="key"] | all' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "server/StatefulSet: gossip encryption key is passed via the -encrypt flag" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'global.gossipEncryption.autoGenerate=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[] | select(.name=="consul") | .command | any(contains("-encrypt=\"${GOSSIP_KEY}\""))' + | tee /dev/stderr) + [ "${actual}" = "true" ] +} + + @test "server/StatefulSet: gossip encryption disabled in server StatefulSet when secretName is missing" { cd `chart_dir` local actual=$(helm template \ diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 9105565572..a0d4472a74 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -117,26 +117,21 @@ global: # created by this chart. See https://kubernetes.io/docs/concepts/policy/pod-security-policy/. enablePodSecurityPolicies: false - # Configures which Kubernetes secret to retrieve Consul's - # gossip encryption key from (see `-encrypt` (https://consul.io/docs/agent/options#_encrypt)). If secretName or - # secretKey are not set, gossip encryption will not be enabled. The secret must - # be in the same namespace that Consul is installed into. + # Configures Consul's gossip encryption key, set as a Kubernetes secret + # (see `-encrypt` (https://consul.io/docs/agent/options#_encrypt)). + # By default, gossip encryption is not enabled. The gossip encryption key may be set automatically or manually. + # The recommended method is to automatically generate the key. + # To automatically generate and set a gossip encryption key, set autoGenerate to true. + # Values for secretName and secretKey should not be set if autoGenerate is true. + # To manually generate a gossip encryption key, set secretName and secretKey and use Consul to generate + # a Kubernetes secret referencing these values. # - # The secret can be created by running: - # - # ```shell - # $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen) # ``` - # - # To reference, use: - # - # ```yaml - # global: - # gossipEncryption: - # secretName: consul-gossip-encryption-key - # secretKey: key + # $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen) # ``` gossipEncryption: + # Automatically generate a gossip encryption key and save it to a Kubernetes secret. + autoGenerate: false # secretName is the name of the Kubernetes secret that holds the gossip # encryption key. The secret must be in the same namespace that Consul is installed into. secretName: "" From 2443bff0ee4ddb8690ff9ac6ebf9e6ea8e9b97ad Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Tue, 5 Oct 2021 15:53:51 -0400 Subject: [PATCH 07/10] Fix Helm test that I broke (#755) * Atoning for my sins * Fix the issue in client-daemonset bats too --- charts/consul/test/unit/client-daemonset.bats | 2 +- charts/consul/test/unit/server-statefulset.bats | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats index cbbbc10ef9..e0560c8c60 100755 --- a/charts/consul/test/unit/client-daemonset.bats +++ b/charts/consul/test/unit/client-daemonset.bats @@ -622,7 +622,7 @@ load _helpers -s templates/client-daemonset.yaml \ --set 'global.gossipEncryption.autoGenerate=true' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[] | select(.name=="consul") | .command | any(contains("-encrypt=\"${GOSSIP_KEY}\""))' + yq '.spec.template.spec.containers[] | select(.name=="consul") | .command | any(contains("-encrypt=\"${GOSSIP_KEY}\""))' \ | tee /dev/stderr) [ "${actual}" = "true" ] } diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats index a7e48a3951..6418cef313 100755 --- a/charts/consul/test/unit/server-statefulset.bats +++ b/charts/consul/test/unit/server-statefulset.bats @@ -858,7 +858,7 @@ load _helpers -s templates/server-statefulset.yaml \ --set 'global.gossipEncryption.autoGenerate=true' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[] | select(.name=="consul") | .command | any(contains("-encrypt=\"${GOSSIP_KEY}\""))' + yq '.spec.template.spec.containers[] | select(.name=="consul") | .command | any(contains("-encrypt=\"${GOSSIP_KEY}\""))' \ | tee /dev/stderr) [ "${actual}" = "true" ] } From 57fc0c92d211f4bc81247de7455e7902b4417c50 Mon Sep 17 00:00:00 2001 From: Mark <47016163+mark-vw@users.noreply.github.com> Date: Tue, 5 Oct 2021 15:47:13 -0700 Subject: [PATCH 08/10] add resource config for mesh gw service-init (#758) * Add resource settings for mesh gw service-init --- .../templates/mesh-gateway-deployment.yaml | 10 ++--- .../test/unit/mesh-gateway-deployment.bats | 44 +++++++++++++++++++ charts/consul/values.yaml | 12 +++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml index de3f0ff689..4d87baf678 100644 --- a/charts/consul/templates/mesh-gateway-deployment.yaml +++ b/charts/consul/templates/mesh-gateway-deployment.yaml @@ -225,13 +225,9 @@ spec: mountPath: /consul/tls/ca readOnly: true {{- end }} - resources: - requests: - memory: "50Mi" - cpu: "50m" - limits: - memory: "50Mi" - cpu: "50m" + {{- if .Values.meshGateway.initServiceInitContainer.resources }} + resources: {{ toYaml .Values.meshGateway.initServiceInitContainer.resources | nindent 12 }} + {{- end }} containers: - name: mesh-gateway image: {{ .Values.global.imageEnvoy | quote }} diff --git a/charts/consul/test/unit/mesh-gateway-deployment.bats b/charts/consul/test/unit/mesh-gateway-deployment.bats index 4e12efa89d..aab58a8413 100755 --- a/charts/consul/test/unit/mesh-gateway-deployment.bats +++ b/charts/consul/test/unit/mesh-gateway-deployment.bats @@ -437,6 +437,50 @@ key2: value2' \ [ "${actual}" = "cpu2" ] } +#-------------------------------------------------------------------- +# service-init container resources + +@test "meshGateway/Deployment: init service-init container has default resources" { + cd `chart_dir` + local actual=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers[1].resources' | tee /dev/stderr) + + [ $(echo "${actual}" | yq -r '.requests.memory') = "50Mi" ] + [ $(echo "${actual}" | yq -r '.requests.cpu') = "50m" ] + [ $(echo "${actual}" | yq -r '.limits.memory') = "50Mi" ] + [ $(echo "${actual}" | yq -r '.limits.cpu') = "50m" ] +} + +@test "meshGateway/Deployment: init service-init container resources can be set" { + cd `chart_dir` + local object=$(helm template \ + -s templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.initServiceInitContainer.resources.requests.memory=memory' \ + --set 'meshGateway.initServiceInitContainer.resources.requests.cpu=cpu' \ + --set 'meshGateway.initServiceInitContainer.resources.limits.memory=memory2' \ + --set 'meshGateway.initServiceInitContainer.resources.limits.cpu=cpu2' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers[1].resources' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.requests.memory' | tee /dev/stderr) + [ "${actual}" = "memory" ] + + local actual=$(echo $object | yq -r '.requests.cpu' | tee /dev/stderr) + [ "${actual}" = "cpu" ] + + local actual=$(echo $object | yq -r '.limits.memory' | tee /dev/stderr) + [ "${actual}" = "memory2" ] + + local actual=$(echo $object | yq -r '.limits.cpu' | tee /dev/stderr) + [ "${actual}" = "cpu2" ] +} + #-------------------------------------------------------------------- # consul sidecar resources diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index a0d4472a74..49defa4f03 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -2004,6 +2004,18 @@ meshGateway: memory: "150Mi" cpu: "50m" + # Resource settings for the `service-init` init container. + # @recurse: false + # @type: map + initServiceInitContainer: + resources: + requests: + memory: "50Mi" + cpu: "50m" + limits: + memory: "50Mi" + cpu: "50m" + # By default, we set an anti-affinity so that two gateway pods won't be # on the same node. NOTE: Gateways require that Consul client agents are # also running on the nodes alongside each gateway pod. From 18de67cb62e60cb4385686dd9c36dc6016e17697 Mon Sep 17 00:00:00 2001 From: Luke Kysow <1034429+lkysow@users.noreply.github.com> Date: Wed, 6 Oct 2021 09:26:35 -0700 Subject: [PATCH 09/10] Add changelog entry for GH-758 (#762) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc08371b9..ec5596255b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ FEATURES: * Helm Chart * Add automatic generation of gossip encryption with `global.gossipEncryption.autoGenerate=true`. [[GH-738](https://github.com/hashicorp/consul-k8s/pull/738)] + * Add support for configuring resources for mesh gateway `service-init` container. [[GH-758](https://github.com/hashicorp/consul-k8s/pull/758)] IMPROVEMENTS: * Control Plane From 0792f7dfd8480aa02f80e3dd45fa7bc67051f9eb Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Wed, 6 Oct 2021 17:40:13 -0600 Subject: [PATCH 10/10] Acceptance tests fixes (#765) * Pin to a previous version of the tf azure provider because of the issue with AKS API. * Bump all k8s minor versions by 1. The oldest is now 1.18 and the newest is 1.22. Also swap GKE and EKS versions because GKE only supports 1.19+ * Update to the latest version of the EKS module which fixes an issue with kubeconfig file permissions --- .circleci/config.yml | 25 ++++++++----------- README.md | 2 +- charts/consul/test/terraform/aks/main.tf | 4 +-- charts/consul/test/terraform/eks/main.tf | 8 +++--- charts/consul/test/terraform/eks/variables.tf | 4 +-- charts/consul/test/terraform/gke/main.tf | 2 +- 6 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7f76b6b469..c309f2cc64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -449,7 +449,7 @@ jobs: - checkout - install-prereqs - create-kind-clusters: - version: "v1.20.7" + version: "v1.21.2" - restore_cache: keys: - consul-helm-modcache-v2-{{ checksum "charts/consul/test/acceptance/go.mod" }} @@ -481,7 +481,7 @@ jobs: - checkout - install-prereqs - create-kind-clusters: - version: "v1.20.7" + version: "v1.21.2" - restore_cache: keys: - consul-helm-modcache-v2-{{ checksum "charts/consul/test/acceptance/go.mod" }} @@ -561,7 +561,7 @@ jobs: ######################## # ACCEPTANCE TESTS ######################## - acceptance-gke-1-17: + acceptance-gke-1-19: environment: - TEST_RESULTS: /tmp/test-results docker: @@ -627,7 +627,7 @@ jobs: fail_only: true failure_message: "GKE acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}" - acceptance-aks-1-19: + acceptance-aks-1-20: environment: - TEST_RESULTS: /tmp/test-results docker: @@ -717,11 +717,6 @@ jobs: echo "export primary_kubeconfig=$primary_kubeconfig" >> $BASH_ENV echo "export secondary_kubeconfig=$secondary_kubeconfig" >> $BASH_ENV - # Change file permissions of the kubecofig files to avoid warnings by helm. - # TODO: remove when https://github.com/terraform-aws-modules/terraform-aws-eks/pull/1114 is merged. - chmod 600 "$primary_kubeconfig" - chmod 600 "$secondary_kubeconfig" - # Restore go module cache if there is one - restore_cache: keys: @@ -801,7 +796,7 @@ jobs: fail_only: true failure_message: "OpenShift acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}" - acceptance-kind-1-21: + acceptance-kind-1-22: environment: - TEST_RESULTS: /tmp/test-results machine: @@ -811,7 +806,7 @@ jobs: - checkout - install-prereqs - create-kind-clusters: - version: "v1.21.1" + version: "v1.22.1" - restore_cache: keys: - consul-helm-modcache-v2-{{ checksum "charts/consul/test/acceptance/go.mod" }} @@ -832,7 +827,7 @@ jobs: path: /tmp/test-results - slack/status: fail_only: true - failure_message: "Acceptance tests against Kind with Kubernetes v1.21 failed. Check the logs at: ${CIRCLE_BUILD_URL}" + failure_message: "Acceptance tests against Kind with Kubernetes v1.22 failed. Check the logs at: ${CIRCLE_BUILD_URL}" update-helm-charts-index: docker: @@ -947,16 +942,16 @@ workflows: # - acceptance-openshift: # requires: # - cleanup-azure-resources - - acceptance-gke-1-17: + - acceptance-gke-1-19: requires: - cleanup-gcp-resources - acceptance-eks-1-18: requires: - cleanup-eks-resources - - acceptance-aks-1-19: + - acceptance-aks-1-20: requires: - cleanup-azure-resources - - acceptance-kind-1-21 + - acceptance-kind-1-22 # update-helm-charts-index: <-- Disable until we can figure out a release workflow. We currently don't want it to trigger on a tag because the tag is pushed by the eco-releases pipeline. # jobs: # - helm-chart-pipeline-approval: diff --git a/README.md b/README.md index 34c00c052a..cc5b1985c2 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ use Consul with Kubernetes, please see the ### Prerequisites * **Helm 3.0+** (Helm 2 is not supported) - * **Kubernetes 1.17+** - This is the earliest version of Kubernetes tested. + * **Kubernetes 1.18+** - This is the earliest version of Kubernetes tested. It is possible that this chart works with earlier versions but it is untested. diff --git a/charts/consul/test/terraform/aks/main.tf b/charts/consul/test/terraform/aks/main.tf index 0406afd0b2..774b94b68c 100644 --- a/charts/consul/test/terraform/aks/main.tf +++ b/charts/consul/test/terraform/aks/main.tf @@ -1,5 +1,5 @@ provider "azurerm" { - version = "~> 2.0" + version = "2.78.0" features {} } @@ -24,7 +24,7 @@ resource "azurerm_kubernetes_cluster" "default" { location = azurerm_resource_group.default[count.index].location resource_group_name = azurerm_resource_group.default[count.index].name dns_prefix = "consul-k8s-${random_id.suffix[count.index].dec}" - kubernetes_version = "1.19.13" + kubernetes_version = "1.20.9" default_node_pool { name = "default" diff --git a/charts/consul/test/terraform/eks/main.tf b/charts/consul/test/terraform/eks/main.tf index b16646af33..3aa79db821 100644 --- a/charts/consul/test/terraform/eks/main.tf +++ b/charts/consul/test/terraform/eks/main.tf @@ -51,7 +51,7 @@ module "eks" { count = var.cluster_count source = "terraform-aws-modules/eks/aws" - version = "12.2.0" + version = "17.20.0" cluster_name = "consul-k8s-${random_id.suffix[count.index].dec}" cluster_version = "1.18" @@ -69,9 +69,9 @@ module "eks" { } } - manage_aws_auth = false - write_kubeconfig = true - config_output_path = pathexpand("~/.kube/consul-k8s-${random_id.suffix[count.index].dec}") + manage_aws_auth = false + write_kubeconfig = true + kubeconfig_output_path = pathexpand("~/.kube/consul-k8s-${random_id.suffix[count.index].dec}") tags = var.tags } diff --git a/charts/consul/test/terraform/eks/variables.tf b/charts/consul/test/terraform/eks/variables.tf index a401312a45..cab209fda0 100644 --- a/charts/consul/test/terraform/eks/variables.tf +++ b/charts/consul/test/terraform/eks/variables.tf @@ -14,7 +14,7 @@ variable "role_arn" { } variable "tags" { - type = map - default = {} + type = map + default = {} description = "Tags to attach to the created resources." } diff --git a/charts/consul/test/terraform/gke/main.tf b/charts/consul/test/terraform/gke/main.tf index a6c294e2e6..8f86ad1290 100644 --- a/charts/consul/test/terraform/gke/main.tf +++ b/charts/consul/test/terraform/gke/main.tf @@ -10,7 +10,7 @@ resource "random_id" "suffix" { data "google_container_engine_versions" "main" { location = var.zone - version_prefix = "1.17." + version_prefix = "1.19." } resource "google_container_cluster" "cluster" {