PodSecurityPolicy switch (#218)

* move pkg/installer/util -> pkg/util

* PodSecurityPolicy admission plugin

* new config option features.enabled_psp
* regenerate kubeadm config every upgrade

* Fixed panic: runtime error: index out of range

* errors.Wrap() missed errors

* Permissive PSP for kube-system pods

* drop if -f /etc/kubernetes/kubelet.conf before kubeadm upgrade

* make defaultAdmissionPlugins a []string

* Improve naming

config.yaml.dist

@@ -23,6 +23,12 @@ network:
 provider:
   name: '' # Kubernetes cloud provider, like "aws"
 
+features:
+  # enables PodSecurityPolicy admission plugin in API server, as well as creates
+  # default `privileged` PodSecurityPolicy, plus RBAC rules to authorize
+  # `kube-system` namespace pods to `use` it.
+  enable_pod_security_policy: false
+
 backup:
   # Ark supported provider, see https://heptio.github.io/ark/v0.10.0/support-matrix
   provider: '' # currently only aws is supported, empty provider disable ark backups

pkg/certificate/certificate.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/kubermatic/kubeone/pkg/installer/util"
+	"github.com/kubermatic/kubeone/pkg/util"
 
 	"k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/cert/triple"

pkg/cmd/kubeconfig.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
-	"github.com/kubermatic/kubeone/pkg/installer/util"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 type kubeconfigOptions struct {

pkg/cmd/upgrade.go

@@ -26,7 +26,6 @@ func upgradeCmd(rootFlags *pflag.FlagSet) *cobra.Command {
 
 This command takes KubeOne manifest which contains information about hosts and how the cluster should be provisioned.
 It's possible to source information about hosts from Terraform output, using the '--tfjson' flag.`,
-		Hidden:  true,
 		Args:    cobra.ExactArgs(1),
 		Example: `kubeone upgrade mycluster.yaml -t terraformoutput.json`,
 		RunE: func(_ *cobra.Command, args []string) error {

pkg/config/cluster.go

@@ -22,6 +22,7 @@ type Cluster struct {
 	Workers           []WorkerConfig          `json:"workers"`
 	Backup            BackupConfig            `json:"backup"`
 	MachineController MachineControllerConfig `json:"machine_controller"`
+	Features          Features                `json:"features"`
 }
 
 // DefaultAndValidate checks if the cluster config makes sense.
@@ -460,6 +461,11 @@ func (m *MachineControllerConfig) DefaultAndValidate() error {
 	return nil
 }
 
+// Features switches
+type Features struct {
+	EnablePodSecurityPolicy bool `json:"enable_pod_security_policy"`
+}
+
 func boolPtr(val bool) *bool {
 	return &val
 }

pkg/features/activate.go

@@ -0,0 +1,29 @@
+package features
+
+import (
+	"github.com/pkg/errors"
+
+	kubeadmv1beta1 "github.com/kubermatic/kubeone/pkg/apis/kubeadm/v1beta1"
+	"github.com/kubermatic/kubeone/pkg/config"
+	"github.com/kubermatic/kubeone/pkg/util"
+)
+
+// Activate configured features.
+// Installing CRDs, creating policies and so on
+func Activate(ctx *util.Context) error {
+	if err := installKubeSystemPSP(ctx.Cluster.Features.EnablePodSecurityPolicy, ctx); err != nil {
+		return errors.Wrap(err, "failed to install PodSecurityPolicy")
+	}
+
+	return nil
+}
+
+// UpdateKubeadmClusterConfiguration update additional config options in the kubeadm's
+// v1beta1.ClusterConfiguration according to enabled features
+func UpdateKubeadmClusterConfiguration(featuresCfg config.Features, clusterConfig *kubeadmv1beta1.ClusterConfiguration) {
+	if clusterConfig.APIServer.ExtraArgs == nil {
+		clusterConfig.APIServer.ExtraArgs = make(map[string]string)
+	}
+
+	activateKubeadmPSP(featuresCfg.EnablePodSecurityPolicy, clusterConfig)
+}

pkg/features/psp.go

@@ -0,0 +1,164 @@
+package features
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	kubeadmv1beta1 "github.com/kubermatic/kubeone/pkg/apis/kubeadm/v1beta1"
+	"github.com/kubermatic/kubeone/pkg/templates"
+	"github.com/kubermatic/kubeone/pkg/util"
+
+	corev1 "k8s.io/api/core/v1"
+	policybeta1 "k8s.io/api/policy/v1beta1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	pspAdmissionPlugin            = "PodSecurityPolicy"
+	apiServerAdmissionPluginsFlag = "enable-admission-plugins"
+	pspRoleNamespace              = "kube-system"
+)
+
+var (
+	defaultAdmissionPlugins = []string{
+		"NamespaceLifecycle",
+		"LimitRanger",
+		"ServiceAccount",
+		"PersistentVolumeClaimResize",
+		"DefaultStorageClass",
+		"DefaultTolerationSeconds",
+		"MutatingAdmissionWebhook",
+		"ValidatingAdmissionWebhook",
+		"ResourceQuota",
+		"Priority",
+	}
+)
+
+func activateKubeadmPSP(activate bool, clusterConfig *kubeadmv1beta1.ClusterConfiguration) {
+	if !activate {
+		return
+	}
+
+	if _, ok := clusterConfig.APIServer.ExtraArgs[apiServerAdmissionPluginsFlag]; ok {
+		clusterConfig.APIServer.ExtraArgs[apiServerAdmissionPluginsFlag] += "," + pspAdmissionPlugin
+	} else {
+		clusterConfig.APIServer.ExtraArgs[apiServerAdmissionPluginsFlag] = strings.Join(append(defaultAdmissionPlugins, pspAdmissionPlugin), ",")
+	}
+}
+
+func installKubeSystemPSP(activate bool, ctx *util.Context) error {
+	if !activate {
+		return nil
+	}
+
+	rbacClient := ctx.Clientset.RbacV1()
+
+	err := templates.EnsurePodSecurityPolicy(ctx.Clientset.PolicyV1beta1().PodSecurityPolicies(), privilegedPSP())
+	if err != nil {
+		return errors.Wrap(err, "failed to ensure PodSecurityPolicy")
+	}
+
+	err = templates.EnsureClusterRole(rbacClient.ClusterRoles(), privilegedPSPClusterRole())
+	if err != nil {
+		return errors.Wrap(err, "failed to ensure PodSecurityPolicy cluster role")
+	}
+
+	err = templates.EnsureRoleBinding(rbacClient.RoleBindings(pspRoleNamespace), privilegedPSPRoleBinding())
+	if err != nil {
+		return errors.Wrap(err, "failed to ensure PodSecurityPolicy role binding")
+	}
+
+	return nil
+}
+
+func privilegedPSP() *policybeta1.PodSecurityPolicy {
+	return &policybeta1.PodSecurityPolicy{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "policy/v1beta1",
+			Kind:       "PodSecurityPolicy",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "privileged",
+		},
+		Spec: policybeta1.PodSecurityPolicySpec{
+			Privileged:               true,
+			HostNetwork:              true,
+			HostIPC:                  true,
+			HostPID:                  true,
+			AllowPrivilegeEscalation: boolPtr(true),
+			AllowedCapabilities:      []corev1.Capability{"*"},
+			Volumes:                  []policybeta1.FSType{policybeta1.All},
+			HostPorts: []policybeta1.HostPortRange{
+				{Min: 0, Max: 65535},
+			},
+			RunAsUser: policybeta1.RunAsUserStrategyOptions{
+				Rule: policybeta1.RunAsUserStrategyRunAsAny,
+			},
+			SELinux: policybeta1.SELinuxStrategyOptions{
+				Rule: policybeta1.SELinuxStrategyRunAsAny,
+			},
+			SupplementalGroups: policybeta1.SupplementalGroupsStrategyOptions{
+				Rule: policybeta1.SupplementalGroupsStrategyRunAsAny,
+			},
+			FSGroup: policybeta1.FSGroupStrategyOptions{
+				Rule: policybeta1.FSGroupStrategyRunAsAny,
+			},
+		},
+	}
+}
+
+func privilegedPSPClusterRole() *rbacv1.ClusterRole {
+	return &rbacv1.ClusterRole{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "rbac.authorization.k8s.io/v1",
+			Kind:       "ClusterRole",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "privileged-psp",
+		},
+		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups:     []string{"policy"},
+				Resources:     []string{"podsecuritypolicies"},
+				Verbs:         []string{"use"},
+				ResourceNames: []string{"privileged"},
+			},
+		},
+	}
+}
+
+func privilegedPSPRoleBinding() *rbacv1.RoleBinding {
+	return &rbacv1.RoleBinding{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "rbac.authorization.k8s.io/v1",
+			Kind:       "RoleBinding",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "privileged-psp",
+			Namespace: pspRoleNamespace,
+		},
+		RoleRef: rbacv1.RoleRef{
+			Name:     "privileged-psp",
+			Kind:     "ClusterRole",
+			APIGroup: rbacv1.GroupName,
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				APIGroup: rbacv1.GroupName,
+				Kind:     "Group",
+				Name:     "system:nodes",
+			},
+			{
+				APIGroup: rbacv1.GroupName,
+				Kind:     "Group",
+				Name:     "system:serviceaccounts:" + pspRoleNamespace,
+			},
+		},
+	}
+}
+
+func boolPtr(b bool) *bool {
+	return &b
+}

pkg/installer/installation/ark.go

@@ -1,8 +1,8 @@
 package installation
 
 import (
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/templates/ark"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func deployArk(ctx *util.Context) error {

pkg/installer/installation/certs.go

@@ -4,8 +4,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/kubermatic/kubeone/pkg/config"
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/ssh"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func downloadCA(ctx *util.Context) error {

pkg/installer/installation/cni.go

@@ -1,8 +1,8 @@
 package installation
 
 import (
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/templates/canal"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func applyCanalCNI(ctx *util.Context) error {

pkg/installer/installation/controlplane.go

@@ -4,8 +4,8 @@ import (
 	"strconv"
 
 	"github.com/kubermatic/kubeone/pkg/config"
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/ssh"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func joinControlplaneNode(ctx *util.Context) error {

pkg/installer/installation/install.go

@@ -3,7 +3,8 @@ package installation
 import (
 	"github.com/pkg/errors"
 
-	"github.com/kubermatic/kubeone/pkg/installer/util"
+	"github.com/kubermatic/kubeone/pkg/features"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 // Install performs all the steps required to install Kubernetes on
@@ -23,6 +24,7 @@ func Install(ctx *util.Context) error {
 		{fn: joinControlplaneNode, errMsg: "unable to join other masters a cluster"},
 		{fn: copyKubeconfig, errMsg: "unable to copy kubeconfig to home directory"},
 		{fn: util.BuildKubernetesClientset, errMsg: "unable to build kubernetes clientset"},
+		{fn: features.Activate, errMsg: "unable to activate features"},
 		{fn: applyCanalCNI, errMsg: "failed to install cni plugin canal"},
 		{fn: installMachineController, errMsg: "failed to install machine-controller"},
 		{fn: createWorkerMachines, errMsg: "failed to create worker machines"},

pkg/installer/installation/kubeadm_config.go

@@ -6,9 +6,9 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/kubermatic/kubeone/pkg/config"
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/ssh"
 	"github.com/kubermatic/kubeone/pkg/templates/kubeadm"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func generateKubeadm(ctx *util.Context) error {
@@ -27,10 +27,5 @@ func generateKubeadm(ctx *util.Context) error {
 }
 
 func generateKubeadmOnNode(ctx *util.Context, _ *config.HostConfig, conn ssh.Connection) error {
-	err := ctx.Configuration.UploadTo(conn, ctx.WorkDir)
-	if err != nil {
-		return errors.Wrap(err, "failed to upload")
-	}
-
-	return nil
+	return errors.Wrap(ctx.Configuration.UploadTo(conn, ctx.WorkDir), "failed to upload")
 }

pkg/installer/installation/kubeadm_leader.go

@@ -4,8 +4,8 @@ import (
 	"strconv"
 
 	"github.com/kubermatic/kubeone/pkg/config"
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/ssh"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 const (

pkg/installer/installation/kubeconfig.go

@@ -2,8 +2,8 @@ package installation
 
 import (
 	"github.com/kubermatic/kubeone/pkg/config"
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/ssh"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func copyKubeconfig(ctx *util.Context) error {

pkg/installer/installation/machine_controller.go

@@ -1,8 +1,8 @@
 package installation
 
 import (
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/templates/machinecontroller"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 func installMachineController(ctx *util.Context) error {

pkg/installer/installation/prerequisites.go

@@ -4,8 +4,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/kubermatic/kubeone/pkg/config"
-	"github.com/kubermatic/kubeone/pkg/installer/util"
 	"github.com/kubermatic/kubeone/pkg/ssh"
+	"github.com/kubermatic/kubeone/pkg/util"
 )
 
 const dockerVersion = "18.09.2"