From 69456f21c080ba75dd074ca5eeef27ed5865834e Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Wed, 28 Apr 2021 14:25:21 +0200 Subject: [PATCH 1/4] fix/workaround: add workaround for cgroupv2 until fixed in k3s - Cgroupv2 fix as per https://github.com/k3s-io/k3s/pull/3237 & https://github.com/k3s-io/k3s/pull/3242 --- pkg/actions/nodehooks.go | 4 +- pkg/client/cluster.go | 20 ++++++++ pkg/runtimes/docker/translate.go | 10 ++++ pkg/runtimes/docker/translate_test.go | 6 +++ pkg/runtimes/docker/util.go | 5 +- pkg/runtimes/runtime.go | 5 +- pkg/types/fixes/assets/cgroupv2-entrypoint.sh | 22 +++++++++ pkg/types/fixes/fixes.go | 47 +++++++++++++++++++ 8 files changed, 113 insertions(+), 6 deletions(-) create mode 100755 pkg/types/fixes/assets/cgroupv2-entrypoint.sh create mode 100644 pkg/types/fixes/fixes.go diff --git a/pkg/actions/nodehooks.go b/pkg/actions/nodehooks.go index 88491154b..efd22ed20 100644 --- a/pkg/actions/nodehooks.go +++ b/pkg/actions/nodehooks.go @@ -23,6 +23,7 @@ package actions import ( "context" + "os" "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" @@ -32,8 +33,9 @@ type WriteFileAction struct { Runtime runtimes.Runtime Content []byte Dest string + Mode os.FileMode } func (act WriteFileAction) Run(ctx context.Context, node *k3d.Node) error { - return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, node) + return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, act.Mode, node) } diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index c5fa0b8c4..7d7a33604 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -23,6 +23,7 @@ package client import ( "context" + _ "embed" "errors" "fmt" "sort" @@ -41,6 +42,7 @@ import ( runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" "github.com/rancher/k3d/v4/pkg/types" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/types/fixes" "github.com/rancher/k3d/v4/pkg/types/k3s" "github.com/rancher/k3d/v4/pkg/util" "github.com/rancher/k3d/v4/version" @@ -205,6 +207,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf Runtime: runtime, Content: regCm, Dest: k3d.DefaultLocalRegistryHostingConfigmapTempPath, + Mode: 0644, }, }) @@ -233,6 +236,23 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf Runtime: runtime, Content: regConfBytes, Dest: k3d.DefaultRegistriesFilePath, + Mode: 0644, + }, + }) + } + + // FIXME: FixCgroupV2 - to be removed when fixed upstream + if fixes.FixCgroupV2Enabled() { + + log.Debugln("experimental cgroupv2 fix enabled") + + clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{ + Stage: k3d.LifecycleStagePreStart, + Action: actions.WriteFileAction{ + Runtime: runtime, + Content: fixes.CgroupV2Entrypoint, + Dest: "/bin/entrypoint.sh", + Mode: 0744, }, }) } diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index 82f846bbc..cf56bf51f 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -33,6 +33,7 @@ import ( "github.com/docker/go-connections/nat" runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/types/fixes" log "github.com/sirupsen/logrus" dockercliopts "github.com/docker/cli/opts" @@ -54,6 +55,15 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) { containerConfig.Image = node.Image /* Command & Arguments */ + // FIXME: FixCgroupV2 - to be removed when fixed upstream + if fixes.FixCgroupV2Enabled() { + if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole { + containerConfig.Entrypoint = []string{ + "/bin/entrypoint.sh", + } + } + } + containerConfig.Cmd = []string{} containerConfig.Cmd = append(containerConfig.Cmd, node.Cmd...) // contains k3s command and role-specific required flags/args diff --git a/pkg/runtimes/docker/translate_test.go b/pkg/runtimes/docker/translate_test.go index a64c53b95..3476ae2f8 100644 --- a/pkg/runtimes/docker/translate_test.go +++ b/pkg/runtimes/docker/translate_test.go @@ -31,6 +31,7 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/go-connections/nat" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/types/fixes" ) func TestTranslateNodeToContainer(t *testing.T) { @@ -93,6 +94,11 @@ func TestTranslateNodeToContainer(t *testing.T) { }, } + // TODO: // FIXME: FixCgroupV2 - to be removed when fixed upstream + if fixes.FixCgroupV2Enabled() { + expectedRepresentation.ContainerConfig.Entrypoint = []string{"/bin/entrypoint.sh"} + } + actualRepresentation, err := TranslateNodeToContainer(inputNode) if err != nil { t.Error(err) diff --git a/pkg/runtimes/docker/util.go b/pkg/runtimes/docker/util.go index 05ff5f09e..246075560 100644 --- a/pkg/runtimes/docker/util.go +++ b/pkg/runtimes/docker/util.go @@ -96,7 +96,7 @@ func (d Docker) CopyToNode(ctx context.Context, src string, dest string, node *k } // WriteToNode writes a byte array to the selected node -func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, node *k3d.Node) error { +func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, mode os.FileMode, node *k3d.Node) error { nodeContainer, err := getNodeContainer(ctx, node) if err != nil { @@ -116,7 +116,7 @@ func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, no defer tarWriter.Close() tarHeader := &tar.Header{ Name: dest, - Mode: 0644, + Mode: int64(mode), Size: int64(len(content)), } @@ -140,7 +140,6 @@ func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, no return nil } - // GetDockerClient returns a docker client func GetDockerClient() (*client.Client, error) { var err error diff --git a/pkg/runtimes/runtime.go b/pkg/runtimes/runtime.go index 5e5f2cd66..62ebcefc0 100644 --- a/pkg/runtimes/runtime.go +++ b/pkg/runtimes/runtime.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "net" + "os" "time" "github.com/rancher/k3d/v4/pkg/runtimes/docker" @@ -68,8 +69,8 @@ type Runtime interface { ExecInNodeGetLogs(context.Context, *k3d.Node, []string) (*bufio.Reader, error) GetNodeLogs(context.Context, *k3d.Node, time.Time) (io.ReadCloser, error) GetImages(context.Context) ([]string, error) - CopyToNode(context.Context, string, string, *k3d.Node) error // @param context, source, destination, node - WriteToNode(context.Context, []byte, string, *k3d.Node) error // @param context, content, destination, node + CopyToNode(context.Context, string, string, *k3d.Node) error // @param context, source, destination, node + WriteToNode(context.Context, []byte, string, os.FileMode, *k3d.Node) error // @param context, content, destination, filemode, node GetHostIP(context.Context, string) (net.IP, error) ConnectNodeToNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name DisconnectNodeFromNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name diff --git a/pkg/types/fixes/assets/cgroupv2-entrypoint.sh b/pkg/types/fixes/assets/cgroupv2-entrypoint.sh new file mode 100755 index 000000000..786c32b92 --- /dev/null +++ b/pkg/types/fixes/assets/cgroupv2-entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail + +######################################################################################################################################### +# DISCLAIMER # +# Copied from https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/hack/dind#L28-L37 # +# Permission granted by Akihiro Suda (https://github.com/rancher/k3d/issues/493#issuecomment-827405962) # +# Moby License Apache 2.0: https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/LICENSE # +######################################################################################################################################### +if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + # move the processes from the root group to the /init group, + # otherwise writing subtree_control fails with EBUSY. + mkdir -p /sys/fs/cgroup/init + busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : + # enable controllers + sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control" +fi + +exec /bin/k3s "$@" \ No newline at end of file diff --git a/pkg/types/fixes/fixes.go b/pkg/types/fixes/fixes.go new file mode 100644 index 000000000..5270166be --- /dev/null +++ b/pkg/types/fixes/fixes.go @@ -0,0 +1,47 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package fixes + +import ( + _ "embed" + "os" +) + +/* NOTE + * This file includes types used for workarounds and hotfixes which are subject to change + * and may disappear anytime, e.g. when the fix was included in an upstream project + */ + +/* + * Cgroupv2 fix as per https://github.com/k3s-io/k3s/pull/3237 & https://github.com/k3s-io/k3s/pull/3242 + * FIXME: FixCgroupV2 - to be removed when fixed upstream + */ + +// EnvFixCgroupV2 is the environment variable that k3d will check for to enable/disable the cgroupv2 workaround +const EnvFixCgroupV2 = "K3D_FIX_CGROUPV2" + +//go:embed assets/cgroupv2-entrypoint.sh +var CgroupV2Entrypoint []byte + +func FixCgroupV2Enabled() bool { + return os.Getenv(EnvFixCgroupV2) == "1" || os.Getenv(EnvFixCgroupV2) == "true" +} From 0e2a3a77b058ae8b56b49cfec78d4cbe0c066fa2 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Wed, 28 Apr 2021 14:57:16 +0200 Subject: [PATCH 2/4] add debug log for proper runtime info --- cmd/root.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 8db9e06e8..9922d8744 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -32,8 +32,6 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" - rt "runtime" - "github.com/rancher/k3d/v4/cmd/cluster" cfg "github.com/rancher/k3d/v4/cmd/config" "github.com/rancher/k3d/v4/cmd/image" @@ -206,7 +204,9 @@ func initRuntime() { log.Fatalln(err) } runtimes.SelectedRuntime = runtime - log.Debugf("Selected runtime is '%T' on GOOS '%s/%s'", runtimes.SelectedRuntime, rt.GOOS, rt.GOARCH) + if rtinfo, err := runtime.Info(); err == nil { + log.Debugf("Runtime Info:\n%+v", rtinfo) + } } func printVersion() { From d7ac650a052ea41294f137fb5a8ac9ea66575f9d Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Wed, 28 Apr 2021 15:38:17 +0200 Subject: [PATCH 3/4] use parsebool to determine whether K3D_FIX_CGROUPV2 is enabled or not --- pkg/types/fixes/fixes.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/types/fixes/fixes.go b/pkg/types/fixes/fixes.go index 5270166be..077553eeb 100644 --- a/pkg/types/fixes/fixes.go +++ b/pkg/types/fixes/fixes.go @@ -24,6 +24,7 @@ package fixes import ( _ "embed" "os" + "strconv" ) /* NOTE @@ -43,5 +44,9 @@ const EnvFixCgroupV2 = "K3D_FIX_CGROUPV2" var CgroupV2Entrypoint []byte func FixCgroupV2Enabled() bool { - return os.Getenv(EnvFixCgroupV2) == "1" || os.Getenv(EnvFixCgroupV2) == "true" + enabled, err := strconv.ParseBool(os.Getenv(EnvFixCgroupV2)) + if err != nil { + return false + } + return enabled } From 5e0e265ce2f36d8640c95aa70290105fda9b8210 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 29 Apr 2021 09:16:58 +0200 Subject: [PATCH 4/4] update changelog to include K3D_FIX_CGROUPV2 --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a06635e5c..86bdf6e18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v4.4.3 + +### Highlights + +- cgroupv2 support: to properly work on cgroupv2 systems, k3s has to move all the processes from the root cgroup to a new /init cgroup and enable subtree_control + - this is going to be included in the k3s agent code directly () + - for now we're overriding the container entrypoint with a script that does this (#579, compare ) + - thanks a lot for all the input and support @AkihiroSuda + - **Usage**: set the environment variable `K3D_FIX_CGROUPV2` to a `true` value before/when creating a cluster with k3d + - e.g. `export K3D_FIX_CGROUPV2=1` + ## v4.4.2 ### Fixes