From 799b116caaf7f7a3aacd0e9930a44d875ab63f63 Mon Sep 17 00:00:00 2001 From: Paul Glass Date: Mon, 24 Jul 2023 11:06:15 -0500 Subject: [PATCH] Support running with restricted PSA enforcement enabled (part 1) (#2572) Support restricted PSA enforcement in a basic setup. This is enough to get a basic setup with ACLs and TLS working and an acceptance test passing (but does not update every component). On OpenShift, we have the option to set the security context or not. If the security context is unset, then it is set automatically by OpenShift SCCs. However, we prefer to set the security context to avoid useless warnings on OpenShift and to reduce the config difference between OpenShift and plain Kube. By default, OpenShift namespaces have the audit and warn PSA labels set to restricted, so we receive pod security warnings when deploying Consul to OpenShift even though the pods will be able to run. Helm chart changes: * Add a helper to the helm chart to define a "restricted" container security context (when pod security policies are not enabled) * Update the following container securityContexts to use the "restricted" settings (not exhaustive) - gateway-cleanup-job.yaml - gateway-resources-job.yaml - gossip-encryption-autogenerate-job.yaml - server-acl-init-cleanup-job.yaml - only if `.Values.server.containerSecurityContext.server.acl-init` is unset - server-acl-init-job.yaml - only if `.Values.server.containerSecurityContext.server.acl-init` is unset - server-statefulset.yaml: - the locality-init container receives the restricted context - the consul container receives the restricted context only if `.Values.server.containerSecurityContext.server` is unset - tls-init-cleanup-job.yaml - only if `.Values.server.containerSecurityContext.server.tls-init` is unset - tls-init-job.yaml - only if `.Values.server.containerSecurityContext.server.tls-init` is unset - webhook-cert-manager-deployment.yaml Acceptance test changes: * When `-enable-openshift` and `-enable-cni` are set, configure the CNI settings correctly for OpenShift. * Add the `-enable-restricted-psa-enforcement` test flag. When this is set, the tests assume the Consul namespace has restricted PSA enforcement enabled. The tests will deploy the CNI (if enabled) into the `kube-system` namespace. Compatible test cases will deploy applications outside of the Consul namespace. * Update the ConnectHelper to configure the NetworkAttachmentDefinition required to be compatible with the CNI on OpenShift. * Add fixtures for static-client and static-server for OpenShift. This is necessary because the deployment configs must reference the network attachment definition when using the CNI on OpenShift. * Update tests in the `acceptance/tests/connect` directory to either run or skip based on -enable-cni and -enable-openshift --- .changelog/2572.txt | 3 + acceptance/framework/config/config.go | 22 +++- acceptance/framework/config/config_test.go | 1 + .../framework/connhelper/connect_helper.go | 114 ++++++++++++++---- .../framework/consul/helm_cluster_test.go | 6 + .../framework/environment/environment.go | 10 ++ acceptance/framework/flags/flags.go | 14 ++- .../connect/connect_external_servers_test.go | 2 + .../connect/connect_inject_namespaces_test.go | 2 + .../tests/connect/connect_inject_test.go | 30 +++-- .../connect/connect_proxy_lifecycle_test.go | 1 + .../tests/connect/permissive_mtls_test.go | 1 + .../bases/openshift/network-attachment.yaml | 17 +++ .../kustomization.yaml | 8 ++ .../static-client-openshift-inject/patch.yaml | 14 +++ .../kustomization.yaml | 8 ++ .../static-client-openshift-tproxy/patch.yaml | 18 +++ .../kustomization.yaml | 8 ++ .../cases/static-server-openshift/patch.yaml | 42 +++++++ charts/consul/templates/_helpers.tpl | 25 +++- .../templates/connect-inject-deployment.yaml | 1 + .../consul/templates/gateway-cleanup-job.yaml | 1 + .../templates/gateway-resources-job.yaml | 1 + .../gossip-encryption-autogenerate-job.yaml | 1 + .../server-acl-init-cleanup-job.yaml | 3 + .../consul/templates/server-acl-init-job.yaml | 3 + .../consul/templates/server-statefulset.yaml | 5 +- .../templates/tls-init-cleanup-job.yaml | 3 + charts/consul/templates/tls-init-job.yaml | 3 + .../webhook-cert-manager-deployment.yaml | 1 + .../consul/test/unit/server-statefulset.bats | 70 ++++++++++- 31 files changed, 398 insertions(+), 40 deletions(-) create mode 100644 .changelog/2572.txt create mode 100644 acceptance/tests/fixtures/bases/openshift/network-attachment.yaml create mode 100644 acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml diff --git a/.changelog/2572.txt b/.changelog/2572.txt new file mode 100644 index 0000000000..4bc6c4ba50 --- /dev/null +++ b/.changelog/2572.txt @@ -0,0 +1,3 @@ +```release-note:improvement +helm: set container securityContexts to match the `restricted` Pod Security Standards policy to support running Consul in a namespace with restricted PSA enforcement enabled +``` diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index eada42af20..83114ad4a2 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "testing" "github.com/hashicorp/go-version" "gopkg.in/yaml.v2" @@ -73,7 +74,8 @@ type TestConfig struct { EnablePodSecurityPolicies bool - EnableCNI bool + EnableCNI bool + EnableRestrictedPSAEnforcement bool EnableTransparentProxy bool @@ -134,10 +136,22 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { if t.EnableCNI { setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true") + setIfNotEmpty(helmValues, "connectInject.cni.logLevel", "debug") // GKE is currently the only cloud provider that uses a different CNI bin dir. if t.UseGKE { setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin") } + if t.EnableOpenshift { + setIfNotEmpty(helmValues, "connectInject.cni.multus", "true") + setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/var/lib/cni/bin") + setIfNotEmpty(helmValues, "connectInject.cni.cniNetDir", "/etc/kubernetes/cni/net.d") + } + + if t.EnableRestrictedPSAEnforcement { + // The CNI requires privilege, so when restricted PSA enforcement is enabled on the Consul + // namespace it must be run in a different privileged namespace. + setIfNotEmpty(helmValues, "connectInject.cni.namespace", "kube-system") + } } setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy)) @@ -219,6 +233,12 @@ func (t *TestConfig) entImage() (string, error) { return fmt.Sprintf("hashicorp/consul-enterprise:%s%s-ent", consulImageVersion, preRelease), nil } +func (c *TestConfig) SkipWhenOpenshiftAndCNI(t *testing.T) { + if c.EnableOpenshift && c.EnableCNI { + t.Skip("skipping because -enable-cni and -enable-openshift are set and this test doesn't deploy apps correctly") + } +} + // setIfNotEmpty sets key to val in map m if value is not empty. func setIfNotEmpty(m map[string]string, key, val string) { if val != "" { diff --git a/acceptance/framework/config/config_test.go b/acceptance/framework/config/config_test.go index 96f0f0e7eb..df981e26fa 100644 --- a/acceptance/framework/config/config_test.go +++ b/acceptance/framework/config/config_test.go @@ -116,6 +116,7 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) { }, map[string]string{ "connectInject.cni.enabled": "true", + "connectInject.cni.logLevel": "debug", "connectInject.transparentProxy.defaultEnabled": "false", }, }, diff --git a/acceptance/framework/connhelper/connect_helper.go b/acceptance/framework/connhelper/connect_helper.go index 40c02fb091..02f88bbb22 100644 --- a/acceptance/framework/connhelper/connect_helper.go +++ b/acceptance/framework/connhelper/connect_helper.go @@ -6,9 +6,11 @@ package connhelper import ( "context" "strconv" + "strings" "testing" "time" + terratestK8s "github.com/gruntwork-io/terratest/modules/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/config" "github.com/hashicorp/consul-k8s/acceptance/framework/consul" "github.com/hashicorp/consul-k8s/acceptance/framework/environment" @@ -44,7 +46,12 @@ type ConnectHelper struct { // ReleaseName is the name of the Consul cluster. ReleaseName string + // Ctx is used to deploy Consul Ctx environment.TestContext + // UseAppNamespace is used top optionally deploy applications into a separate namespace. + // If unset, the namespace associated with Ctx is used. + UseAppNamespace bool + Cfg *config.TestConfig // consulCluster is the cluster to use for the test. @@ -82,6 +89,14 @@ func (c *ConnectHelper) Upgrade(t *testing.T) { c.consulCluster.Upgrade(t, c.helmValues()) } +func (c *ConnectHelper) KubectlOptsForApp(t *testing.T) *terratestK8s.KubectlOptions { + opts := c.Ctx.KubectlOptions(t) + if !c.UseAppNamespace { + return opts + } + return c.Ctx.KubectlOptionsForNamespace(opts.Namespace + "-apps") +} + // DeployClientAndServer deploys a client and server pod to the Kubernetes // cluster which will be used to test service mesh connectivity. If the Secure // flag is true, a pre-check is done to ensure that the ACL tokens for the test @@ -108,23 +123,46 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") - if c.Cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + c.setupAppNamespace(t) + + opts := c.KubectlOptsForApp(t) + if c.Cfg.EnableCNI && c.Cfg.EnableOpenshift { + // On OpenShift with the CNI, we need to create a network attachment definition in the namespace + // where the applications are running, and the app deployment configs need to reference that network + // attachment definition. + + // TODO: A base fixture is the wrong place for these files + k8s.KubectlApply(t, opts, "../fixtures/bases/openshift/") + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + k8s.KubectlDelete(t, opts, "../fixtures/bases/openshift/") + }) + + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift") + if c.Cfg.EnableTransparentProxy { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy") + } else { + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject") + } } else { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + if c.Cfg.EnableTransparentProxy { + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + } else { + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + } } - // Check that both static-server and static-client have been injected and // now have 2 containers. retry.RunWith( &retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond}, t, func(r *retry.R) { for _, labelSelector := range []string{"app=static-server", "app=static-client"} { - podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: labelSelector, - FieldSelector: `status.phase=Running`, - }) + podList, err := c.Ctx.KubernetesClient(t).CoreV1(). + Pods(opts.Namespace). + List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelector, + FieldSelector: `status.phase=Running`, + }) require.NoError(r, err) require.Len(r, podList.Items, 1) require.Len(r, podList.Items[0].Spec.Containers, 2) @@ -132,27 +170,58 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { }) } +// setupAppNamespace creates a namespace where applications are deployed. This +// does nothing if UseAppNamespace is not set. The app namespace is relevant +// when testing with restricted PSA enforcement enabled. +func (c *ConnectHelper) setupAppNamespace(t *testing.T) { + if !c.UseAppNamespace { + return + } + opts := c.KubectlOptsForApp(t) + // If we are deploying apps in another namespace, create the namespace. + + _, err := k8s.RunKubectlAndGetOutputE(t, opts, "create", "ns", opts.Namespace) + if err != nil && strings.Contains(err.Error(), "AlreadyExists") { + return + } + require.NoError(t, err) + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + k8s.RunKubectl(t, opts, "delete", "ns", opts.Namespace) + }) + + if c.Cfg.EnableRestrictedPSAEnforcement { + // Allow anything to run in the app namespace. + k8s.RunKubectl(t, opts, "label", "--overwrite", "ns", opts.Namespace, + "pod-security.kubernetes.io/enforce=privileged", + "pod-security.kubernetes.io/enforce-version=v1.24", + ) + } + +} + // CreateResolverRedirect creates a resolver that redirects to a static-server, a corresponding k8s service, // and intentions. This helper is primarly used to ensure that the virtual-ips are persisted to consul properly. func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) { logger.Log(t, "creating resolver redirect") - options := c.Ctx.KubectlOptions(t) + opts := c.KubectlOptsForApp(t) + c.setupAppNamespace(t) kustomizeDir := "../fixtures/cases/resolver-redirect-virtualip" - k8s.KubectlApplyK(t, options, kustomizeDir) + k8s.KubectlApplyK(t, opts, kustomizeDir) helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { - k8s.KubectlDeleteK(t, options, kustomizeDir) - }) + k8s.KubectlDeleteK(t, opts, kustomizeDir) + } } // TestConnectionFailureWithoutIntention ensures the connection to the static // server fails when no intentions are configured. func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) { logger.Log(t, "checking that the connection is not successful because there's no intention") + opts := c.KubectlOptsForApp(t) if c.Cfg.EnableTransparentProxy { - k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server") + k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://static-server") } else { - k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234") + k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://localhost:1234") } } @@ -177,11 +246,12 @@ func (c *ConnectHelper) CreateIntention(t *testing.T) { // static-client pod once the intention is set. func (c *ConnectHelper) TestConnectionSuccess(t *testing.T) { logger.Log(t, "checking that connection is successful") + opts := c.KubectlOptsForApp(t) if c.Cfg.EnableTransparentProxy { // todo: add an assertion that the traffic is going through the proxy - k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server") + k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://static-server") } else { - k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234") + k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://localhost:1234") } } @@ -192,8 +262,10 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) { // Test that kubernetes readiness status is synced to Consul. // Create a file called "unhealthy" at "/tmp/" so that the readiness probe // of the static-server pod fails. + opts := c.KubectlOptsForApp(t) + logger.Log(t, "testing k8s -> consul health checks sync by making the static-server unhealthy") - k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy") + k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy") // The readiness probe should take a moment to be reflected in Consul, // CheckStaticServerConnection will retry until Consul marks the service @@ -205,20 +277,20 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) { // other tests. logger.Log(t, "checking that connection is unsuccessful") if c.Cfg.EnableTransparentProxy { - k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{ + k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{ "curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", "curl: (7) Failed to connect to static-server port 80: Connection refused", }, "", "http://static-server") } else { - k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{ + k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{ "curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", }, "", "http://localhost:1234") } // Return the static-server to a "healthy state". - k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy") + k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy") } // helmValues uses the Secure and AutoEncrypt fields to set values for the Helm diff --git a/acceptance/framework/consul/helm_cluster_test.go b/acceptance/framework/consul/helm_cluster_test.go index 9c44006d43..011ca23e0f 100644 --- a/acceptance/framework/consul/helm_cluster_test.go +++ b/acceptance/framework/consul/helm_cluster_test.go @@ -8,6 +8,7 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/config" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" "github.com/stretchr/testify/require" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -80,9 +81,14 @@ func (c *ctx) Name() string { func (c *ctx) KubectlOptions(_ *testing.T) *k8s.KubectlOptions { return &k8s.KubectlOptions{} } +func (c *ctx) KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions { + return &k8s.KubectlOptions{} +} func (c *ctx) KubernetesClient(_ *testing.T) kubernetes.Interface { return fake.NewSimpleClientset() } func (c *ctx) ControllerRuntimeClient(_ *testing.T) client.Client { return runtimefake.NewClientBuilder().Build() } + +var _ environment.TestContext = (*ctx)(nil) diff --git a/acceptance/framework/environment/environment.go b/acceptance/framework/environment/environment.go index 9150f4ff03..7014a3c05f 100644 --- a/acceptance/framework/environment/environment.go +++ b/acceptance/framework/environment/environment.go @@ -35,6 +35,8 @@ type TestEnvironment interface { // for example, information about a specific Kubernetes cluster. type TestContext interface { KubectlOptions(t *testing.T) *k8s.KubectlOptions + // TODO: I don't love this. + KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions KubernetesClient(t *testing.T) kubernetes.Interface ControllerRuntimeClient(t *testing.T) client.Client } @@ -138,6 +140,14 @@ func (k kubernetesContext) KubectlOptions(t *testing.T) *k8s.KubectlOptions { return k.options } +func (k kubernetesContext) KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions { + return &k8s.KubectlOptions{ + ContextName: k.kubeContextName, + ConfigPath: k.pathToKubeConfig, + Namespace: ns, + } +} + // KubernetesClientFromOptions takes KubectlOptions and returns Kubernetes API client. func KubernetesClientFromOptions(t *testing.T, options *k8s.KubectlOptions) kubernetes.Interface { configPath, err := options.GetConfigPath(t) diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index e6a29de100..de413bbba5 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -27,7 +27,8 @@ type TestFlags struct { flagEnablePodSecurityPolicies bool - flagEnableCNI bool + flagEnableCNI bool + flagEnableRestrictedPSAEnforcement bool flagEnableTransparentProxy bool @@ -115,6 +116,13 @@ func (t *TestFlags) init() { flag.BoolVar(&t.flagEnableCNI, "enable-cni", false, "If true, the test suite will run tests with consul-cni plugin enabled. "+ "In general, this will only run against tests that are mesh related (connect, mesh-gateway, peering, etc") + flag.BoolVar(&t.flagEnableRestrictedPSAEnforcement, "enable-restricted-psa-enforcement", false, + "If true, this indicates that Consul is being run in a namespace with restricted PSA enforcement enabled. "+ + "The tests do not configure Consul's namespace with PSA enforcement enabled. This must configured before tests are run. "+ + "The CNI and test applications need more privilege than is allowed in a restricted namespace. "+ + "When set, the CNI will be deployed into the kube-system namespace, and in supported test cases, applications "+ + "are deployed, by default, into a namespace named '-apps' instead of being deployed into the "+ + "Consul namespace.") flag.BoolVar(&t.flagEnableTransparentProxy, "enable-transparent-proxy", false, "If true, the test suite will run tests with transparent proxy enabled. "+ @@ -172,6 +180,7 @@ func (t *TestFlags) Validate() error { if t.flagEnableEnterprise && t.flagEnterpriseLicense == "" { return errors.New("-enable-enterprise provided without setting env var CONSUL_ENT_LICENSE with consul license") } + return nil } @@ -195,7 +204,8 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { EnablePodSecurityPolicies: t.flagEnablePodSecurityPolicies, - EnableCNI: t.flagEnableCNI, + EnableCNI: t.flagEnableCNI, + EnableRestrictedPSAEnforcement: t.flagEnableRestrictedPSAEnforcement, EnableTransparentProxy: t.flagEnableTransparentProxy, diff --git a/acceptance/tests/connect/connect_external_servers_test.go b/acceptance/tests/connect/connect_external_servers_test.go index a7b0f656bf..c95d791773 100644 --- a/acceptance/tests/connect/connect_external_servers_test.go +++ b/acceptance/tests/connect/connect_external_servers_test.go @@ -30,6 +30,8 @@ func TestConnectInject_ExternalServers(t *testing.T) { caseName := fmt.Sprintf("secure: %t", secure) t.Run(caseName, func(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) serverHelmValues := map[string]string{ diff --git a/acceptance/tests/connect/connect_inject_namespaces_test.go b/acceptance/tests/connect/connect_inject_namespaces_test.go index f848594cd2..04021ec391 100644 --- a/acceptance/tests/connect/connect_inject_namespaces_test.go +++ b/acceptance/tests/connect/connect_inject_namespaces_test.go @@ -34,6 +34,7 @@ func TestConnectInjectNamespaces(t *testing.T) { if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) cases := []struct { name string @@ -246,6 +247,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) { if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) consulDestNS := "consul-dest" cases := []struct { diff --git a/acceptance/tests/connect/connect_inject_test.go b/acceptance/tests/connect/connect_inject_test.go index 199bbf0f1b..0704b16e54 100644 --- a/acceptance/tests/connect/connect_inject_test.go +++ b/acceptance/tests/connect/connect_inject_test.go @@ -38,11 +38,12 @@ func TestConnectInject(t *testing.T) { releaseName := helpers.RandomName() connHelper := connhelper.ConnectHelper{ - ClusterKind: consul.Helm, - Secure: c.secure, - ReleaseName: releaseName, - Ctx: ctx, - Cfg: cfg, + ClusterKind: consul.Helm, + Secure: c.secure, + ReleaseName: releaseName, + Ctx: ctx, + UseAppNamespace: cfg.EnableRestrictedPSAEnforcement, + Cfg: cfg, } connHelper.Setup(t) @@ -71,11 +72,12 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { releaseName := helpers.RandomName() connHelper := connhelper.ConnectHelper{ - ClusterKind: consul.Helm, - Secure: true, - ReleaseName: releaseName, - Ctx: ctx, - Cfg: cfg, + ClusterKind: consul.Helm, + Secure: true, + ReleaseName: releaseName, + Ctx: ctx, + UseAppNamespace: cfg.EnableRestrictedPSAEnforcement, + Cfg: cfg, } connHelper.Setup(t) @@ -84,7 +86,8 @@ func TestConnectInject_VirtualIPFailover(t *testing.T) { connHelper.CreateResolverRedirect(t) connHelper.DeployClientAndServer(t) - k8s.CheckStaticServerConnectionSuccessful(t, connHelper.Ctx.KubectlOptions(t), "static-client", "http://resolver-redirect") + opts := connHelper.KubectlOptsForApp(t) + k8s.CheckStaticServerConnectionSuccessful(t, opts, "static-client", "http://resolver-redirect") } // Test the endpoints controller cleans up force-killed pods. @@ -93,6 +96,9 @@ func TestConnectInject_CleanupKilledPods(t *testing.T) { name := fmt.Sprintf("secure: %t", secure) t.Run(name, func(t *testing.T) { cfg := suite.Config() + + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ @@ -161,6 +167,8 @@ func TestConnectInject_MultiportServices(t *testing.T) { name := fmt.Sprintf("secure: %t", secure) t.Run(name, func(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) + ctx := suite.Environment().DefaultContext(t) helmValues := map[string]string{ diff --git a/acceptance/tests/connect/connect_proxy_lifecycle_test.go b/acceptance/tests/connect/connect_proxy_lifecycle_test.go index d2f6a0c383..7487847a0a 100644 --- a/acceptance/tests/connect/connect_proxy_lifecycle_test.go +++ b/acceptance/tests/connect/connect_proxy_lifecycle_test.go @@ -34,6 +34,7 @@ const ( // Test the endpoints controller cleans up force-killed pods. func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { cfg := suite.Config() + cfg.SkipWhenOpenshiftAndCNI(t) for _, testCfg := range []LifecycleShutdownConfig{ {secure: false, helmValues: map[string]string{ diff --git a/acceptance/tests/connect/permissive_mtls_test.go b/acceptance/tests/connect/permissive_mtls_test.go index f0a55779ee..310d879d06 100644 --- a/acceptance/tests/connect/permissive_mtls_test.go +++ b/acceptance/tests/connect/permissive_mtls_test.go @@ -20,6 +20,7 @@ func TestConnectInject_PermissiveMTLS(t *testing.T) { if !cfg.EnableTransparentProxy { t.Skipf("skipping this because -enable-transparent-proxy is not set") } + cfg.SkipWhenOpenshiftAndCNI(t) ctx := suite.Environment().DefaultContext(t) diff --git a/acceptance/tests/fixtures/bases/openshift/network-attachment.yaml b/acceptance/tests/fixtures/bases/openshift/network-attachment.yaml new file mode 100644 index 0000000000..4b3f7948ee --- /dev/null +++ b/acceptance/tests/fixtures/bases/openshift/network-attachment.yaml @@ -0,0 +1,17 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: consul-cni +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "consul-cni", + "cni_bin_dir": "/var/lib/cni/bin", + "cni_net_dir": "/etc/kubernetes/cni/net.d", + "kubeconfig": "ZZZ-consul-cni-kubeconfig", + "log_level": "debug", + "multus": true, + "name": "consul-cni", + "type": "consul-cni" + }' + diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml new file mode 100644 index 0000000000..4d4a53b87f --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-inject/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml new file mode 100644 index 0000000000..8cc6d10411 --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-inject/patch.yaml @@ -0,0 +1,14 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "consul.hashicorp.com/connect-service-upstreams": "static-server:1234" + "k8s.v1.cni.cncf.io/networks": '[{ "name":"consul-cni" }]' diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml new file mode 100644 index 0000000000..4d4a53b87f --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml new file mode 100644 index 0000000000..3b9c91fcc0 --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-client-openshift-tproxy/patch.yaml @@ -0,0 +1,18 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# When using the CNI on OpenShift, we need to specify the +# network attachment definition for the pods to use. This assumes +# that one named 'consul-cni' was created by the acceptance tests. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "k8s.v1.cni.cncf.io/networks": '[{ "name":"consul-cni" }]' + diff --git a/acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml b/acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml new file mode 100644 index 0000000000..bc50c78adf --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-server-openshift/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../bases/static-server + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml b/acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml new file mode 100644 index 0000000000..8e2ed857f3 --- /dev/null +++ b/acceptance/tests/fixtures/cases/static-server-openshift/patch.yaml @@ -0,0 +1,42 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "k8s.v1.cni.cncf.io/networks": '[{ "name":"consul-cni" }]' + spec: + containers: + - name: static-server + image: docker.mirror.hashicorp.services/kschoche/http-echo:latest + args: + - -text="hello world" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + livenessProbe: + httpGet: + port: 8080 + initialDelaySeconds: 1 + failureThreshold: 1 + periodSeconds: 1 + startupProbe: + httpGet: + port: 8080 + initialDelaySeconds: 1 + failureThreshold: 30 + periodSeconds: 1 + readinessProbe: + exec: + command: ['sh', '-c', 'test ! -f /tmp/unhealthy'] + initialDelaySeconds: 1 + failureThreshold: 1 + periodSeconds: 1 + serviceAccountName: static-server diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl index 1b866888c0..18f57b188c 100644 --- a/charts/consul/templates/_helpers.tpl +++ b/charts/consul/templates/_helpers.tpl @@ -15,6 +15,29 @@ as well as the global.name setting. {{- end -}} {{- end -}} +{{- define "consul.restrictedSecurityContext" -}} +{{- if not .Values.global.enablePodSecurityPolicies -}} +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault +{{- if not .Values.global.openshift.enabled -}} +{{/* +We must set runAsUser or else the root user will be used in some cases and +containers will fail to start due to runAsNonRoot above (e.g. +tls-init-cleanup). On OpenShift, runAsUser is automatically. We pick user 100 +because it is a non-root user id that exists in the consul, consul-dataplane, +and consul-k8s-control-plane images. +*/}} + runAsUser: 100 +{{- end -}} +{{- end -}} +{{- end -}} + {{- define "consul.vaultSecretTemplate" -}} | {{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} @@ -422,4 +445,4 @@ Usage: {{ template "consul.validateTelemetryCollectorCloud" . }} {{- if or (and .Values.telemetryCollector.cloud.clientSecret.secretName .Values.telemetryCollector.cloud.clientSecret.secretKey .Values.telemetryCollector.cloud.clientId.secretName .Values.telemetryCollector.cloud.clientId.secretKey (not .Values.global.cloud.resourceId.secretKey)) }} {{fail "When telemetryCollector has clientId and clientSecret .global.cloud.resourceId.secretKey must be set"}} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index 14c3961b4e..e726c9ecc9 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -94,6 +94,7 @@ spec: - containerPort: 8080 name: webhook-server protocol: TCP + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/gateway-cleanup-job.yaml b/charts/consul/templates/gateway-cleanup-job.yaml index 8731aaed81..a987c3b591 100644 --- a/charts/consul/templates/gateway-cleanup-job.yaml +++ b/charts/consul/templates/gateway-cleanup-job.yaml @@ -40,6 +40,7 @@ spec: containers: - name: gateway-cleanup image: {{ .Values.global.imageK8S }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} command: - consul-k8s-control-plane args: diff --git a/charts/consul/templates/gateway-resources-job.yaml b/charts/consul/templates/gateway-resources-job.yaml index 5fcd96cad3..3a29f75e66 100644 --- a/charts/consul/templates/gateway-resources-job.yaml +++ b/charts/consul/templates/gateway-resources-job.yaml @@ -40,6 +40,7 @@ spec: containers: - name: gateway-resources image: {{ .Values.global.imageK8S }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} command: - consul-k8s-control-plane args: diff --git a/charts/consul/templates/gossip-encryption-autogenerate-job.yaml b/charts/consul/templates/gossip-encryption-autogenerate-job.yaml index 9d296478a1..240bfe3f9c 100644 --- a/charts/consul/templates/gossip-encryption-autogenerate-job.yaml +++ b/charts/consul/templates/gossip-encryption-autogenerate-job.yaml @@ -48,6 +48,7 @@ spec: containers: - name: gossip-encryption-autogen image: "{{ .Values.global.imageK8S }}" + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} command: - "/bin/sh" - "-ec" diff --git a/charts/consul/templates/server-acl-init-cleanup-job.yaml b/charts/consul/templates/server-acl-init-cleanup-job.yaml index 4d0aa8f05d..c9f6763bd8 100644 --- a/charts/consul/templates/server-acl-init-cleanup-job.yaml +++ b/charts/consul/templates/server-acl-init-cleanup-job.yaml @@ -60,6 +60,9 @@ spec: containers: - name: server-acl-init-cleanup image: {{ .Values.global.imageK8S }} + {{- if not .Values.server.containerSecurityContext.aclInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} + {{- end }} command: - consul-k8s-control-plane args: diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml index 6625e23b38..c3d4a710e8 100644 --- a/charts/consul/templates/server-acl-init-job.yaml +++ b/charts/consul/templates/server-acl-init-job.yaml @@ -129,6 +129,9 @@ spec: containers: - name: server-acl-init-job image: {{ .Values.global.imageK8S }} + {{- if not .Values.server.containerSecurityContext.aclInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml index 0bde9b881a..04c84df71b 100644 --- a/charts/consul/templates/server-statefulset.yaml +++ b/charts/consul/templates/server-statefulset.yaml @@ -238,6 +238,7 @@ spec: volumeMounts: - name: extra-config mountPath: /consul/extra-config + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} containers: - name: consul image: "{{ default .Values.global.image .Values.server.image }}" @@ -526,9 +527,11 @@ spec: {{- toYaml .Values.server.resources | nindent 12 }} {{- end }} {{- end }} - {{- if not .Values.global.openshift.enabled }} + {{- if .Values.server.containerSecurityContext.server }} securityContext: {{- toYaml .Values.server.containerSecurityContext.server | nindent 12 }} + {{- else }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} {{- end }} {{- if .Values.server.extraContainers }} {{ toYaml .Values.server.extraContainers | nindent 8 }} diff --git a/charts/consul/templates/tls-init-cleanup-job.yaml b/charts/consul/templates/tls-init-cleanup-job.yaml index 69b5a30849..2254a38ed2 100644 --- a/charts/consul/templates/tls-init-cleanup-job.yaml +++ b/charts/consul/templates/tls-init-cleanup-job.yaml @@ -48,6 +48,9 @@ spec: containers: - name: tls-init-cleanup image: "{{ .Values.global.image }}" + {{- if not .Values.server.containerSecurityContext.tlsInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/tls-init-job.yaml b/charts/consul/templates/tls-init-job.yaml index 5839f07dbf..12d3acbad8 100644 --- a/charts/consul/templates/tls-init-job.yaml +++ b/charts/consul/templates/tls-init-job.yaml @@ -63,6 +63,9 @@ spec: containers: - name: tls-init image: "{{ .Values.global.imageK8S }}" + {{- if not .Values.server.containerSecurityContext.tlsInit }} + {{- include "consul.restrictedSecurityContext" . | nindent 10 }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/consul/templates/webhook-cert-manager-deployment.yaml b/charts/consul/templates/webhook-cert-manager-deployment.yaml index dd93c039d2..7ba25b330c 100644 --- a/charts/consul/templates/webhook-cert-manager-deployment.yaml +++ b/charts/consul/templates/webhook-cert-manager-deployment.yaml @@ -51,6 +51,7 @@ spec: -deployment-namespace={{ .Release.Namespace }} image: {{ .Values.global.imageK8S }} name: webhook-cert-manager + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} resources: limits: cpu: 100m diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats index 108fd9bbf8..a60884d20c 100755 --- a/charts/consul/test/unit/server-statefulset.bats +++ b/charts/consul/test/unit/server-statefulset.bats @@ -829,9 +829,9 @@ load _helpers } #-------------------------------------------------------------------- -# global.openshift.enabled & client.containerSecurityContext +# global.openshift.enabled && server.containerSecurityContext -@test "server/StatefulSet: container level securityContexts are not set when global.openshift.enabled=true" { +@test "server/StatefulSet: Can set container level securityContexts when global.openshift.enabled=true" { cd `chart_dir` local manifest=$(helm template \ -s templates/server-statefulset.yaml \ @@ -839,8 +839,72 @@ load _helpers --set 'server.containerSecurityContext.server.privileged=false' \ . | tee /dev/stderr) + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.containers | map(select(.name == "consul")) | .[0].securityContext.privileged') + [ "${actual}" = "false" ] +} + +#-------------------------------------------------------------------- +# global.openshift.enabled + +@test "server/StatefulSet: restricted container securityContexts are set when global.openshift.enabled=true" { + cd `chart_dir` + local manifest=$(helm template \ + -s templates/server-statefulset.yaml \ + --set 'global.openshift.enabled=true' \ + . | tee /dev/stderr) + + local expected=$(echo '{ + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "seccompProfile": { + "type": "RuntimeDefault" + } + }') + + # Check consul container local actual=$(echo "$manifest" | yq -r '.spec.template.spec.containers | map(select(.name == "consul")) | .[0].securityContext') - [ "${actual}" = "null" ] + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] + + # Check locality-init container + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.initContainers | map(select(.name == "locality-init")) | .[0].securityContext') + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] +} + +#-------------------------------------------------------------------- +# global.openshift.enabled = false + +@test "server/StatefulSet: restricted container securityContexts are set by default" { + cd `chart_dir` + local manifest=$(helm template \ + -s templates/server-statefulset.yaml \ + . | tee /dev/stderr) + + local expected=$(echo '{ + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "seccompProfile": { + "type": "RuntimeDefault" + }, + "runAsUser": 100 + }') + + # Check consul container + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.containers | map(select(.name == "consul")) | .[0].securityContext') + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] + + # Check locality-init container + local actual=$(echo "$manifest" | yq -r '.spec.template.spec.initContainers | map(select(.name == "locality-init")) | .[0].securityContext') + local equal=$(jq -n --argjson a "$actual" --argjson b "$expected" '$a == $b') + [ "$equal" == "true" ] } #--------------------------------------------------------------------