Support running with restricted PSA enforcement enabled (part 1)

.changelog/2572.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+helm: set container securityContexts to match the `restricted` Pod Security Standards policy to support running Consul in a namespace with restricted PSA enforcement enabled
+```

acceptance/framework/config/config.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"testing"
 
 	"github.com/hashicorp/go-version"
 	"gopkg.in/yaml.v2"
@@ -73,7 +74,8 @@ type TestConfig struct {
 
 	EnablePodSecurityPolicies bool
 
-	EnableCNI bool
+	EnableCNI                      bool
+	EnableRestrictedPSAEnforcement bool
 
 	EnableTransparentProxy bool
 
@@ -135,10 +137,22 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) {
 
 	if t.EnableCNI {
 		setIfNotEmpty(helmValues, "connectInject.cni.enabled", "true")
+		setIfNotEmpty(helmValues, "connectInject.cni.logLevel", "debug")
 		// GKE is currently the only cloud provider that uses a different CNI bin dir.
 		if t.UseGKE {
 			setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/home/kubernetes/bin")
 		}
+		if t.EnableOpenshift {
+			setIfNotEmpty(helmValues, "connectInject.cni.multus", "true")
+			setIfNotEmpty(helmValues, "connectInject.cni.cniBinDir", "/var/lib/cni/bin")
+			setIfNotEmpty(helmValues, "connectInject.cni.cniNetDir", "/etc/kubernetes/cni/net.d")
+		}
+
+		if t.EnableRestrictedPSAEnforcement {
+			// The CNI requires privilege, so when restricted PSA enforcement is enabled on the Consul
+			// namespace it must be run in a different privileged namespace.
+			setIfNotEmpty(helmValues, "connectInject.cni.namespace", "kube-system")
+		}
 	}
 
 	setIfNotEmpty(helmValues, "connectInject.transparentProxy.defaultEnabled", strconv.FormatBool(t.EnableTransparentProxy))
@@ -220,6 +234,12 @@ func (t *TestConfig) entImage() (string, error) {
 	return fmt.Sprintf("hashicorp/consul-enterprise:%s%s-ent", consulImageVersion, preRelease), nil
 }
 
+func (c *TestConfig) SkipWhenOpenshiftAndCNI(t *testing.T) {
+	if c.EnableOpenshift && c.EnableCNI {
+		t.Skip("skipping because -enable-cni and -enable-openshift are set and this test doesn't deploy apps correctly")
+	}
+}
+
 // setIfNotEmpty sets key to val in map m if value is not empty.
 func setIfNotEmpty(m map[string]string, key, val string) {
 	if val != "" {

acceptance/framework/config/config_test.go

@@ -116,6 +116,7 @@ func TestConfig_HelmValuesFromConfig(t *testing.T) {
 			},
 			map[string]string{
 				"connectInject.cni.enabled":                     "true",
+				"connectInject.cni.logLevel":                    "debug",
 				"connectInject.transparentProxy.defaultEnabled": "false",
 			},
 		},

acceptance/framework/connhelper/connect_helper.go

@@ -6,9 +6,11 @@ package connhelper
 import (
 	"context"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
+	terratestK8s "github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
@@ -44,7 +46,12 @@ type ConnectHelper struct {
 	// ReleaseName is the name of the Consul cluster.
 	ReleaseName string
 
+	// Ctx is used to deploy Consul
 	Ctx environment.TestContext
+	// UseAppNamespace is used top optionally deploy applications into a separate namespace.
+	// If unset, the namespace associated with Ctx is used.
+	UseAppNamespace bool
+
 	Cfg *config.TestConfig
 
 	// consulCluster is the cluster to use for the test.
@@ -82,6 +89,14 @@ func (c *ConnectHelper) Upgrade(t *testing.T) {
 	c.consulCluster.Upgrade(t, c.helmValues())
 }
 
+func (c *ConnectHelper) KubectlOptsForApp(t *testing.T) *terratestK8s.KubectlOptions {
+	opts := c.Ctx.KubectlOptions(t)
+	if !c.UseAppNamespace {
+		return opts
+	}
+	return c.Ctx.KubectlOptionsForNamespace(opts.Namespace + "-apps")
+}
+
 // DeployClientAndServer deploys a client and server pod to the Kubernetes
 // cluster which will be used to test service mesh connectivity. If the Secure
 // flag is true, a pre-check is done to ensure that the ACL tokens for the test
@@ -108,51 +123,105 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) {
 
 	logger.Log(t, "creating static-server and static-client deployments")
 
-	k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
-	if c.Cfg.EnableTransparentProxy {
-		k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy")
+	c.setupAppNamespace(t)
+
+	opts := c.KubectlOptsForApp(t)
+	if c.Cfg.EnableCNI && c.Cfg.EnableOpenshift {
+		// On OpenShift with the CNI, we need to create a network attachment definition in the namespace
+		// where the applications are running, and the app deployment configs need to reference that network
+		// attachment definition.
+
+		// TODO: A base fixture is the wrong place for these files
+		k8s.KubectlApply(t, opts, "../fixtures/bases/openshift/")
+		helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, func() {
+			k8s.KubectlDelete(t, opts, "../fixtures/bases/openshift/")
+		})
+
+		k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift")
+		if c.Cfg.EnableTransparentProxy {
+			k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy")
+		} else {
+			k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject")
+		}
 	} else {
-		k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
+		k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
+		if c.Cfg.EnableTransparentProxy {
+			k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy")
+		} else {
+			k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
+		}
 	}
-
 	// Check that both static-server and static-client have been injected and
 	// now have 2 containers.
 	retry.RunWith(
 		&retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond}, t,
 		func(r *retry.R) {
 			for _, labelSelector := range []string{"app=static-server", "app=static-client"} {
-				podList, err := c.Ctx.KubernetesClient(t).CoreV1().Pods(c.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{
-					LabelSelector: labelSelector,
-					FieldSelector: `status.phase=Running`,
-				})
+				podList, err := c.Ctx.KubernetesClient(t).CoreV1().
+					Pods(opts.Namespace).
+					List(context.Background(), metav1.ListOptions{
+						LabelSelector: labelSelector,
+						FieldSelector: `status.phase=Running`,
+					})
 				require.NoError(r, err)
 				require.Len(r, podList.Items, 1)
 				require.Len(r, podList.Items[0].Spec.Containers, 2)
 			}
 		})
 }
 
+// setupAppNamespace creates a namespace where applications are deployed. This
+// does nothing if UseAppNamespace is not set. The app namespace is relevant
+// when testing with restricted PSA enforcement enabled.
+func (c *ConnectHelper) setupAppNamespace(t *testing.T) {
+	if !c.UseAppNamespace {
+		return
+	}
+	opts := c.KubectlOptsForApp(t)
+	// If we are deploying apps in another namespace, create the namespace.
+
+	_, err := k8s.RunKubectlAndGetOutputE(t, opts, "create", "ns", opts.Namespace)
+	if err != nil && strings.Contains(err.Error(), "AlreadyExists") {
+		return
+	}
+	require.NoError(t, err)
+	helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, func() {
+		k8s.RunKubectl(t, opts, "delete", "ns", opts.Namespace)
+	})
+
+	if c.Cfg.EnableRestrictedPSAEnforcement {
+		// Allow anything to run in the app namespace.
+		k8s.RunKubectl(t, opts, "label", "--overwrite", "ns", opts.Namespace,
+			"pod-security.kubernetes.io/enforce=privileged",
+			"pod-security.kubernetes.io/enforce-version=v1.24",
+		)
+	}
+
+}
+
 // CreateResolverRedirect creates a resolver that redirects to a static-server, a corresponding k8s service,
 // and intentions. This helper is primarly used to ensure that the virtual-ips are persisted to consul properly.
 func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) {
 	logger.Log(t, "creating resolver redirect")
-	options := c.Ctx.KubectlOptions(t)
+	opts := c.KubectlOptsForApp(t)
+	c.setupAppNamespace(t)
 	kustomizeDir := "../fixtures/cases/resolver-redirect-virtualip"
-	k8s.KubectlApplyK(t, options, kustomizeDir)
+	k8s.KubectlApplyK(t, opts, kustomizeDir)
 
 	helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, func() {
-		k8s.KubectlDeleteK(t, options, kustomizeDir)
+		k8s.KubectlDeleteK(t, opts, kustomizeDir)
 	})
 }
 
 // TestConnectionFailureWithoutIntention ensures the connection to the static
 // server fails when no intentions are configured.
 func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) {
 	logger.Log(t, "checking that the connection is not successful because there's no intention")
+	opts := c.KubectlOptsForApp(t)
 	if c.Cfg.EnableTransparentProxy {
-		k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server")
+		k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://static-server")
 	} else {
-		k8s.CheckStaticServerConnectionFailing(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234")
+		k8s.CheckStaticServerConnectionFailing(t, opts, StaticClientName, "http://localhost:1234")
 	}
 }
 
@@ -177,11 +246,12 @@ func (c *ConnectHelper) CreateIntention(t *testing.T) {
 // static-client pod once the intention is set.
 func (c *ConnectHelper) TestConnectionSuccess(t *testing.T) {
 	logger.Log(t, "checking that connection is successful")
+	opts := c.KubectlOptsForApp(t)
 	if c.Cfg.EnableTransparentProxy {
 		// todo: add an assertion that the traffic is going through the proxy
-		k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://static-server")
+		k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://static-server")
 	} else {
-		k8s.CheckStaticServerConnectionSuccessful(t, c.Ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234")
+		k8s.CheckStaticServerConnectionSuccessful(t, opts, StaticClientName, "http://localhost:1234")
 	}
 }
 
@@ -192,8 +262,10 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) {
 	// Test that kubernetes readiness status is synced to Consul.
 	// Create a file called "unhealthy" at "/tmp/" so that the readiness probe
 	// of the static-server pod fails.
+	opts := c.KubectlOptsForApp(t)
+
 	logger.Log(t, "testing k8s -> consul health checks sync by making the static-server unhealthy")
-	k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy")
+	k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "touch", "/tmp/unhealthy")
 
 	// The readiness probe should take a moment to be reflected in Consul,
 	// CheckStaticServerConnection will retry until Consul marks the service
@@ -205,20 +277,20 @@ func (c *ConnectHelper) TestConnectionFailureWhenUnhealthy(t *testing.T) {
 	// other tests.
 	logger.Log(t, "checking that connection is unsuccessful")
 	if c.Cfg.EnableTransparentProxy {
-		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{
+		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{
 			"curl: (56) Recv failure: Connection reset by peer",
 			"curl: (52) Empty reply from server",
 			"curl: (7) Failed to connect to static-server port 80: Connection refused",
 		}, "", "http://static-server")
 	} else {
-		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, c.Ctx.KubectlOptions(t), StaticClientName, false, []string{
+		k8s.CheckStaticServerConnectionMultipleFailureMessages(t, opts, StaticClientName, false, []string{
 			"curl: (56) Recv failure: Connection reset by peer",
 			"curl: (52) Empty reply from server",
 		}, "", "http://localhost:1234")
 	}
 
 	// Return the static-server to a "healthy state".
-	k8s.RunKubectl(t, c.Ctx.KubectlOptions(t), "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy")
+	k8s.RunKubectl(t, opts, "exec", "deploy/"+StaticServerName, "--", "rm", "/tmp/unhealthy")
 }
 
 // helmValues uses the Secure and AutoEncrypt fields to set values for the Helm

acceptance/framework/consul/helm_cluster_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
 	"github.com/stretchr/testify/require"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
@@ -80,9 +81,14 @@ func (c *ctx) Name() string {
 func (c *ctx) KubectlOptions(_ *testing.T) *k8s.KubectlOptions {
 	return &k8s.KubectlOptions{}
 }
+func (c *ctx) KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions {
+	return &k8s.KubectlOptions{}
+}
 func (c *ctx) KubernetesClient(_ *testing.T) kubernetes.Interface {
 	return fake.NewSimpleClientset()
 }
 func (c *ctx) ControllerRuntimeClient(_ *testing.T) client.Client {
 	return runtimefake.NewClientBuilder().Build()
 }
+
+var _ environment.TestContext = (*ctx)(nil)

acceptance/framework/environment/environment.go

@@ -35,6 +35,8 @@ type TestEnvironment interface {
 // for example, information about a specific Kubernetes cluster.
 type TestContext interface {
 	KubectlOptions(t *testing.T) *k8s.KubectlOptions
+	// TODO: I don't love this.
+	KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions
 	KubernetesClient(t *testing.T) kubernetes.Interface
 	ControllerRuntimeClient(t *testing.T) client.Client
 }
@@ -138,6 +140,14 @@ func (k kubernetesContext) KubectlOptions(t *testing.T) *k8s.KubectlOptions {
 	return k.options
 }
 
+func (k kubernetesContext) KubectlOptionsForNamespace(ns string) *k8s.KubectlOptions {
+	return &k8s.KubectlOptions{
+		ContextName: k.kubeContextName,
+		ConfigPath:  k.pathToKubeConfig,
+		Namespace:   ns,
+	}
+}
+
 // KubernetesClientFromOptions takes KubectlOptions and returns Kubernetes API client.
 func KubernetesClientFromOptions(t *testing.T, options *k8s.KubectlOptions) kubernetes.Interface {
 	configPath, err := options.GetConfigPath(t)

acceptance/framework/flags/flags.go

@@ -27,7 +27,8 @@ type TestFlags struct {
 
 	flagEnablePodSecurityPolicies bool
 
-	flagEnableCNI bool
+	flagEnableCNI                      bool
+	flagEnableRestrictedPSAEnforcement bool
 
 	flagEnableTransparentProxy bool
 
@@ -116,6 +117,13 @@ func (t *TestFlags) init() {
 	flag.BoolVar(&t.flagEnableCNI, "enable-cni", false,
 		"If true, the test suite will run tests with consul-cni plugin enabled. "+
 			"In general, this will only run against tests that are mesh related (connect, mesh-gateway, peering, etc")
+	flag.BoolVar(&t.flagEnableRestrictedPSAEnforcement, "enable-restricted-psa-enforcement", false,
+		"If true, this indicates that Consul is being run in a namespace with restricted PSA enforcement enabled. "+
+			"The tests do not configure Consul's namespace with PSA enforcement enabled. This must configured before tests are run. "+
+			"The CNI and test applications need more privilege than is allowed in a restricted namespace. "+
+			"When set, the CNI will be deployed into the kube-system namespace, and in supported test cases, applications "+
+			"are deployed, by default, into a namespace named '<consul-namespace>-apps' instead of being deployed into the "+
+			"Consul namespace.")
 
 	flag.BoolVar(&t.flagEnableTransparentProxy, "enable-transparent-proxy", false,
 		"If true, the test suite will run tests with transparent proxy enabled. "+
@@ -176,6 +184,7 @@ func (t *TestFlags) Validate() error {
 	if t.flagEnableEnterprise && t.flagEnterpriseLicense == "" {
 		return errors.New("-enable-enterprise provided without setting env var CONSUL_ENT_LICENSE with consul license")
 	}
+
 	return nil
 }
 
@@ -200,7 +209,8 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig {
 
 		EnablePodSecurityPolicies: t.flagEnablePodSecurityPolicies,
 
-		EnableCNI: t.flagEnableCNI,
+		EnableCNI:                      t.flagEnableCNI,
+		EnableRestrictedPSAEnforcement: t.flagEnableRestrictedPSAEnforcement,
 
 		EnableTransparentProxy: t.flagEnableTransparentProxy,
 

acceptance/tests/connect/connect_external_servers_test.go

@@ -30,6 +30,8 @@ func TestConnectInject_ExternalServers(t *testing.T) {
 		caseName := fmt.Sprintf("secure: %t", secure)
 		t.Run(caseName, func(t *testing.T) {
 			cfg := suite.Config()
+			cfg.SkipWhenOpenshiftAndCNI(t)
+
 			ctx := suite.Environment().DefaultContext(t)
 
 			serverHelmValues := map[string]string{

acceptance/tests/connect/connect_inject_namespaces_test.go

@@ -34,6 +34,7 @@ func TestConnectInjectNamespaces(t *testing.T) {
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}
+	cfg.SkipWhenOpenshiftAndCNI(t)
 
 	cases := []struct {
 		name                 string
@@ -246,6 +247,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) {
 	if !cfg.EnableEnterprise {
 		t.Skipf("skipping this test because -enable-enterprise is not set")
 	}
+	cfg.SkipWhenOpenshiftAndCNI(t)
 
 	consulDestNS := "consul-dest"
 	cases := []struct {