Custom volume plugin dir

docs/configuration.md

@@ -89,6 +89,7 @@ spec:
       vxlanVNI: 4096
       mtu: 1450
       wireguard: false
+      flexVolumeDriverPath: /usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds
   podSecurityPolicy:
     defaultPolicy: 00-k0s-privileged
   workerProfiles: []
@@ -173,6 +174,7 @@ Using type `etcd` will make k0s to create and manage an elastic etcd cluster wit
 - `vxlanVNI`: The virtual network ID to use for VXLAN. (default: `4096`)
 - `mtu`: MTU to use for overlay network (default `1450`)
 - `wireguard`: enable wireguard based encryption (default `false`). Your host system must be wireguard ready. See https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic for details.
+- `flexVolumeDriverPath`: The host path to use for Calicos flex-volume-driver (default: `/usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds`). This should only need to be changed if the default path is unwriteable. See https://github.com/projectcalico/calico/issues/2712 for details. This option should ideally be paired with a custom volumePluginDir in the profile used on your worker nodes.
 
 ### `spec.podSecurityPolicy`
 
@@ -211,6 +213,16 @@ Example:
              innerKey: innerValue
 ```
 
+
+Custom volumePluginDir:
+
+```
+  workerProfiles:
+    - name: custom-role
+      values:
+         volumePluginDir: /var/libexec/k0s/kubelet-plugins/volume/exec
+```
+
 ### `images`
 Each node under the `images` key has the same structure
 ```

docs/troubleshooting.md

@@ -49,4 +49,30 @@ As Etcd is not fully supported on ARM architecture it also means that k0s contro
 
 Once we enable [cloud provider support](cloud-providers.md) on kubelet on worker nodes, kubelet will automatically add a taint `node.cloudprovider.kubernetes.io/uninitialized` for the node. This tain will prevent normal workloads to be scheduled on the node untill the cloud provider controller actually runs second initialization on the node and removes the taint. This means that these nodes are not schedulable untill the cloud provider controller is actually succesfully running on the cluster.
 
-For troubleshooting your specific cloud provider see its documentation.
+For troubleshooting your specific cloud provider see its documentation.
+
+## k0s not working with read only `/usr`
+
+By default k0s does not run on nodes where `/usr` is read only.
+
+This can be fixed by changing the default path for `volumePluginDir` in your k0s config. You will need to change to values, one for the kubelet itself, and one for Calico.
+
+Here is a snippet of an example config with the default values changed:
+
+```yaml
+spec:
+  controllerManager:
+    extraArgs:
+      flex-volume-plugin-dir: "/etc/kubernetes/kubelet-plugins/volume/exec"
+  network:
+    calico:
+      flexVolumeDriverPath: /etc/k0s/kubelet-plugins/volume/exec/nodeagent~uds
+  workerProfiles:
+    - name: coreos
+      values:
+        volumePluginDir: /etc/k0s/kubelet-plugins/volume/exec/
+```
+
+With this config you can start your server as usual. Any workers will need to be started with
+
+`k0s worker --profile coreos [TOKEN]`

pkg/apis/v1beta1/calico.go

@@ -17,21 +17,23 @@ package v1beta1
 
 // Calico defines the calico related config options
 type Calico struct {
-	Mode            string `yaml:"mode"`
-	VxlanPort       int    `yaml:"vxlanPort"`
-	VxlanVNI        int    `yaml:"vxlanVNI"`
-	MTU             int    `yaml:"mtu"`
-	EnableWireguard bool   `yaml:"wireguard"`
+	Mode                 string `yaml:"mode"`
+	VxlanPort            int    `yaml:"vxlanPort"`
+	VxlanVNI             int    `yaml:"vxlanVNI"`
+	MTU                  int    `yaml:"mtu"`
+	EnableWireguard      bool   `yaml:"wireguard"`
+	FlexVolumeDriverPath string `yaml:"flexVolumeDriverPath"`
 }
 
 // DefaultCalico returns sane defaults for calico
 func DefaultCalico() *Calico {
 	return &Calico{
-		Mode:            "vxlan",
-		VxlanPort:       4789,
-		VxlanVNI:        4096,
-		MTU:             1450,
-		EnableWireguard: false,
+		Mode:                 "vxlan",
+		VxlanPort:            4789,
+		VxlanVNI:             4096,
+		MTU:                  1450,
+		EnableWireguard:      false,
+		FlexVolumeDriverPath: "/usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds",
 	}
 }
 
@@ -42,6 +44,7 @@ func (c *Calico) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	c.VxlanVNI = 4096
 	c.MTU = 1450
 	c.EnableWireguard = false
+	c.FlexVolumeDriverPath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds"
 
 	type ycalico Calico
 	yc := (*ycalico)(c)

pkg/component/server/calico.go

@@ -45,12 +45,13 @@ type manifestsSaver interface {
 }
 
 type calicoConfig struct {
-	MTU             int
-	Mode            string
-	VxlanPort       int
-	VxlanVNI        int
-	ClusterCIDR     string
-	EnableWireguard bool
+	MTU                  int
+	Mode                 string
+	VxlanPort            int
+	VxlanVNI             int
+	ClusterCIDR          string
+	EnableWireguard      bool
+	FlexVolumeDriverPath string
 
 	CalicoCNIImage             string
 	CalicoFlexVolumeImage      string
@@ -189,12 +190,15 @@ func (c *Calico) processConfigChanges(previousConfig calicoConfig) *calicoConfig
 
 func (c *Calico) getConfig() (calicoConfig, error) {
 	config := calicoConfig{
-		MTU:                        c.clusterConf.Spec.Network.Calico.MTU,
-		Mode:                       c.clusterConf.Spec.Network.Calico.Mode,
-		VxlanPort:                  c.clusterConf.Spec.Network.Calico.VxlanPort,
-		VxlanVNI:                   c.clusterConf.Spec.Network.Calico.VxlanVNI,
-		EnableWireguard:            c.clusterConf.Spec.Network.Calico.EnableWireguard,
-		ClusterCIDR:                c.clusterConf.Spec.Network.PodCIDR,
+		MTU:                  c.clusterConf.Spec.Network.Calico.MTU,
+		Mode:                 c.clusterConf.Spec.Network.Calico.Mode,
+		VxlanPort:            c.clusterConf.Spec.Network.Calico.VxlanPort,
+		VxlanVNI:             c.clusterConf.Spec.Network.Calico.VxlanVNI,
+		EnableWireguard:      c.clusterConf.Spec.Network.Calico.EnableWireguard,
+		FlexVolumeDriverPath: c.clusterConf.Spec.Network.Calico.FlexVolumeDriverPath,
+
+		ClusterCIDR: c.clusterConf.Spec.Network.PodCIDR,
+
 		CalicoCNIImage:             c.clusterConf.Images.Calico.CNI.URI(),
 		CalicoFlexVolumeImage:      c.clusterConf.Images.Calico.FlexVolume.URI(),
 		CalicoNodeImage:            c.clusterConf.Images.Calico.Node.URI(),

pkg/component/server/calico_test.go

@@ -97,6 +97,53 @@ func TestCalicoManifests(t *testing.T) {
 		}
 		require.False(t, found, "Must not have FELIX_WIREGUARDENABLED env setting if config spec has no wireguard enabled")
 	})
+
+	t.Run("flex_volume_driver_path_set_from_config", func(t *testing.T) {
+		cfg := v1beta1.DefaultClusterConfig()
+		cfg.Spec.Network.Calico.FlexVolumeDriverPath = "/etc/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds"
+		saver := inMemorySaver{}
+		calico, err := NewCalico(cfg, saver)
+		require.NoError(t, err)
+
+		_ = calico.processConfigChanges(calicoConfig{})
+
+		daemonSetManifestRaw, foundRaw := saver["calico-DaemonSet-calico-node.yaml"]
+		require.True(t, foundRaw, "must have daemon set for calico")
+		spec := daemonSetContainersEnv{}
+		require.NoError(t, yaml.Unmarshal(daemonSetManifestRaw, &spec))
+		found := false
+		for _, volume := range spec.Spec.Template.Spec.Volumes {
+			if volume.Name != "flexvol-driver-host" {
+				continue
+			}
+			found = true
+			require.Equal(t, "/etc/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds", volume.HostPath.Path)
+		}
+		require.True(t, found, "Must have flexvol-driver-host volume")
+	})
+
+	t.Run("flex_volume_driver_path_set_from_default", func(t *testing.T) {
+		cfg := v1beta1.DefaultClusterConfig()
+		saver := inMemorySaver{}
+		calico, err := NewCalico(cfg, saver)
+		require.NoError(t, err)
+
+		_ = calico.processConfigChanges(calicoConfig{})
+
+		daemonSetManifestRaw, foundRaw := saver["calico-DaemonSet-calico-node.yaml"]
+		require.True(t, foundRaw, "must have daemon set for calico")
+		spec := daemonSetContainersEnv{}
+		require.NoError(t, yaml.Unmarshal(daemonSetManifestRaw, &spec))
+		found := false
+		for _, volume := range spec.Spec.Template.Spec.Volumes {
+			if volume.Name != "flexvol-driver-host" {
+				continue
+			}
+			found = true
+			require.Equal(t, "/usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds", volume.HostPath.Path)
+		}
+		require.True(t, found, "Must have flexvol-driver-host volume")
+	})
 }
 
 // this structure is needed only for unit tests and basocally it describes some fields that are needed to be parsed out of the daemon set manifest
@@ -112,6 +159,13 @@ type daemonSetContainersEnv struct {
 						ValueFrom interface{} `yaml:"valueFrom"`
 					} `yaml:"env"`
 				} `yaml:"containers"`
+				Volumes []struct {
+					Name     string `yaml:"name"`
+					HostPath struct {
+						Type string `yaml:"type"`
+						Path string `yaml:"path"`
+					} `yaml:"hostPath"`
+				} `yaml:"volumes"`
 			} `yaml:"spec"`
 		} `yaml:"template"`
 	} `yaml:"spec"`

pkg/component/server/kubeletconfig.go

@@ -82,14 +82,15 @@ func (k *KubeletConfig) Run() error {
 func (k *KubeletConfig) run(dnsAddress string) (*bytes.Buffer, error) {
 	manifest := bytes.NewBuffer([]byte{})
 	clientCAFile := filepath.Join(k.k0sVars.CertRootDir, "ca.crt")
-	defaultProfile := getDefaultProfile(dnsAddress, clientCAFile)
+	volumePluginDir := k.k0sVars.KubeletVolumePluginDir
+	defaultProfile := getDefaultProfile(dnsAddress, clientCAFile, volumePluginDir)
 
 	if err := k.writeConfigMapWithProfile(manifest, "default", defaultProfile); err != nil {
 		return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err)
 	}
 	configMapNames := []string{formatProfileName("default")}
 	for _, profile := range k.clusterSpec.WorkerProfiles {
-		profileConfig := getDefaultProfile(dnsAddress, clientCAFile)
+		profileConfig := getDefaultProfile(dnsAddress, clientCAFile, volumePluginDir)
 		merged, err := mergeProfiles(&profileConfig, profile.Values)
 		if err != nil {
 			return nil, fmt.Errorf("can't merge profile `%s` with default profile: %v", profile.Name, err)
@@ -161,7 +162,7 @@ func (k *KubeletConfig) writeRbacRoleBindings(w io.Writer, configMapNames []stri
 	return tw.WriteToBuffer(w)
 }
 
-func getDefaultProfile(dnsAddress string, clientCAFile string) unstructuredYamlObject {
+func getDefaultProfile(dnsAddress string, clientCAFile string, volumePluginDir string) unstructuredYamlObject {
 	// the motivation to keep it like this instead of the yaml template:
 	// - it's easier to merge programatically defined structure
 	// - apart from map[string]interface there is no good way to define free-form mapping
@@ -201,6 +202,7 @@ func getDefaultProfile(dnsAddress string, clientCAFile string) unstructuredYamlO
 			"TLS_RSA_WITH_AES_128_GCM_SHA256",
 		},
 		"volumeStatsAggPeriod": "0s",
+		"volumePluginDir":      volumePluginDir,
 		"failSwapOn":           false,
 	}
 }

pkg/component/server/kubeletconfig_test.go

@@ -31,6 +31,7 @@ var k0sVars = constant.GetConfig("")
 
 func Test_KubeletConfig(t *testing.T) {
 	dnsAddr := "dns.local"
+	volumePluginDir := k0sVars.KubeletVolumePluginDir
 	clientCAFile := filepath.Join(k0sVars.CertRootDir, "ca.crt")
 
 	t.Run("default_profile_only", func(t *testing.T) {
@@ -78,12 +79,12 @@ func Test_KubeletConfig(t *testing.T) {
 			assert.NoError(t, yaml.Unmarshal([]byte(manifestYamls[2]), &profileYYY))
 
 			// manually apple the same changes to default config and check that there is no diff
-			defaultProfileKubeletConfig := getDefaultProfile(dnsAddr, clientCAFile)
+			defaultProfileKubeletConfig := getDefaultProfile(dnsAddr, clientCAFile, volumePluginDir)
 			defaultProfileKubeletConfig["authentication"].(map[string]interface{})["anonymous"].(map[string]interface{})["enabled"] = false
 			defaultWithChangesXXX, err := yaml.Marshal(defaultProfileKubeletConfig)
 			assert.NoError(t, err)
 
-			defaultProfileKubeletConfig = getDefaultProfile(dnsAddr, clientCAFile)
+			defaultProfileKubeletConfig = getDefaultProfile(dnsAddr, clientCAFile, volumePluginDir)
 			defaultProfileKubeletConfig["authentication"].(map[string]interface{})["webhook"].(map[string]interface{})["cacheTTL"] = "15s"
 			defaultWithChangesYYY, err := yaml.Marshal(defaultProfileKubeletConfig)
 

pkg/component/worker/kubelet.go

@@ -64,11 +64,6 @@ func (k *Kubelet) Init() error {
 		return errors.Wrapf(err, "failed to create %s", k.dataDir)
 	}
 
-	err = util.InitDirectory(k.K0sVars.KubeletVolumePluginDir, constant.KubeletVolumePluginDirMode)
-	if err != nil {
-		return errors.Wrapf(err, "failed to create %s", k.K0sVars.KubeletVolumePluginDir)
-	}
-
 	return nil
 }
 
@@ -82,7 +77,6 @@ func (k *Kubelet) Run() error {
 
 	args := []string{
 		fmt.Sprintf("--root-dir=%s", k.dataDir),
-		fmt.Sprintf("--volume-plugin-dir=%s", k.K0sVars.KubeletVolumePluginDir),
 
 		fmt.Sprintf("--config=%s", kubeletConfigPath),
 		fmt.Sprintf("--bootstrap-kubeconfig=%s", k.K0sVars.KubeletBootstrapConfigPath),