From 2dbd317407dcf451f293f7cf948ad441f91f6e4a Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 22 May 2024 20:08:16 +0000 Subject: [PATCH 01/31] Add WithSkipMissing to not fail import on missing blobs Signed-off-by: Brad Davidson (cherry picked from commit 5f6b813cc8694fd6822c8fb05b3703d634949a13) Signed-off-by: Brad Davidson --- pkg/agent/containerd/containerd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/containerd/containerd.go b/pkg/agent/containerd/containerd.go index 4af199e52cbd..098a0fbb500e 100644 --- a/pkg/agent/containerd/containerd.go +++ b/pkg/agent/containerd/containerd.go @@ -208,7 +208,7 @@ func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Clien defer imageReader.Close() logrus.Infof("Importing images from %s", filePath) - images, err = client.Import(ctx, imageReader, containerd.WithAllPlatforms(true)) + images, err = client.Import(ctx, imageReader, containerd.WithAllPlatforms(true), containerd.WithSkipMissing()) if err != nil { return errors.Wrap(err, "failed to import images from "+filePath) } From 146153dec70eecb7396dd4b39d2b9f3347fc21d2 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 18 Apr 2024 23:28:31 +0000 Subject: [PATCH 02/31] Use fixed stream server bind address for cri-dockerd Will now use 127.0.0.1:10010, same as containerd's CRI Signed-off-by: Brad Davidson (cherry picked from commit 7374010c0cae97b448594db946ade22ced981a75) Signed-off-by: Brad Davidson --- pkg/agent/cridockerd/cridockerd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/agent/cridockerd/cridockerd.go b/pkg/agent/cridockerd/cridockerd.go index b17a5646cc96..9b910f3f242c 100644 --- a/pkg/agent/cridockerd/cridockerd.go +++ b/pkg/agent/cridockerd/cridockerd.go @@ -53,6 +53,7 @@ func getDockerCRIArgs(cfg *config.Node) []string { argsMap := map[string]string{ "container-runtime-endpoint": cfg.CRIDockerd.Address, "cri-dockerd-root-directory": cfg.CRIDockerd.Root, + "streaming-bind-addr": "127.0.0.1:10010", } if dualNode, _ := utilsnet.IsDualStackIPs(cfg.AgentConfig.NodeIPs); dualNode { From cf9da9427f5f40a07e86013c70855f3c0ee3020c Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 19 Apr 2024 00:16:37 +0000 Subject: [PATCH 03/31] Switch stargz over to cri registry config_path Signed-off-by: Brad Davidson (cherry picked from commit 30999f9a07879ffc8c2a0e001c03d9e06a16df1d) Signed-off-by: Brad Davidson --- pkg/agent/containerd/config_test.go | 19 +++++++++++++------ pkg/agent/templates/templates_linux.go | 23 ++++------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/pkg/agent/containerd/config_test.go b/pkg/agent/containerd/config_test.go index 07a0700bbddd..98a948a22435 100644 --- a/pkg/agent/containerd/config_test.go +++ b/pkg/agent/containerd/config_test.go @@ -1471,6 +1471,17 @@ func Test_UnitGetHostConfigs(t *testing.T) { t.Fatalf("failed to parse %s: %v\n", registriesFile, err) } + nodeConfig := &config.Node{ + Containerd: config.Containerd{ + Registry: tempDir + "/hosts.d", + }, + AgentConfig: config.Agent{ + ImageServiceSocket: "containerd-stargz-grpc.sock", + Registry: registry.Registry, + Snapshotter: "stargz", + }, + } + // set up embedded registry, if enabled for the test if tt.args.mirrorAddr != "" { conf := spegel.DefaultRegistry @@ -1478,7 +1489,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { conf.ClientKeyFile = "client-key" conf.ClientCertFile = "client-cert" conf.InternalAddress, conf.RegistryPort, _ = net.SplitHostPort(tt.args.mirrorAddr) - conf.InjectMirror(&config.Node{AgentConfig: config.Agent{Registry: registry.Registry}}) + conf.InjectMirror(nodeConfig) } // Generate config template struct for all hosts @@ -1494,11 +1505,7 @@ func Test_UnitGetHostConfigs(t *testing.T) { // Confirm that the main containerd config.toml renders properly containerdConfig := templates.ContainerdConfig{ - NodeConfig: &config.Node{ - Containerd: config.Containerd{ - Registry: tempDir + "/hosts.d", - }, - }, + NodeConfig: nodeConfig, PrivateRegistryConfig: registry.Registry, Program: "k3s", } diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 0df107abaae7..c064f6fcb4cd 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -44,19 +44,11 @@ cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket [plugins."io.containerd.snapshotter.v1.stargz".cri_keychain] enable_keychain = true {{end}} + +[plugins."io.containerd.snapshotter.v1.stargz".registry] + config_path = "{{ .NodeConfig.Containerd.Registry }}" + {{ if .PrivateRegistryConfig }} -{{ if .PrivateRegistryConfig.Mirrors }} -[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors]{{end}} -{{range $k, $v := .PrivateRegistryConfig.Mirrors }} -[plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}"] - endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] -{{if $v.Rewrites}} - [plugins."io.containerd.snapshotter.v1.stargz".registry.mirrors."{{$k}}".rewrite] -{{range $pattern, $replace := $v.Rewrites}} - "{{$pattern}}" = "{{$replace}}" -{{end}} -{{end}} -{{end}} {{range $k, $v := .PrivateRegistryConfig.Configs }} {{ if $v.Auth }} [plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth] @@ -65,13 +57,6 @@ enable_keychain = true {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} {{end}} -{{ if $v.TLS }} -[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".tls] - {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} - {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} - {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} - {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} -{{end}} {{end}} {{end}} {{end}} From 4476c79d4c7f43c0c2a0a657bbb6472adc281196 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 17 May 2024 18:14:59 +0000 Subject: [PATCH 04/31] Bump containerd to v1.7.17 Signed-off-by: Brad Davidson (cherry picked from commit aaa578785cb978a42555bab7ab237ab6483ecebe) Signed-off-by: Brad Davidson --- go.mod | 18 +++++++++--------- go.sum | 60 ++++++++++++++++++++++++++++++++++++---------------------- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index de6199388a24..a7dfd17eca4c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace ( github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.11.0 github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.12-k3s1 // k3s/release-1.28 github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.1.2 - github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.15-k3s1 + github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.17-k3s1 github.com/docker/distribution => github.com/docker/distribution v2.8.3+incompatible github.com/docker/docker => github.com/docker/docker v25.0.4+incompatible github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.9.0 @@ -30,10 +30,10 @@ replace ( go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1 go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful => go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 golang.org/x/crypto => golang.org/x/crypto v0.17.0 golang.org/x/net => golang.org/x/net v0.17.0 - golang.org/x/sys => golang.org/x/sys v0.13.0 + golang.org/x/sys => golang.org/x/sys v0.18.0 google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230525234035-dd9d682886f9 google.golang.org/grpc => google.golang.org/grpc v1.58.3 gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.6.0 @@ -214,11 +214,11 @@ require ( github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/go-cni v1.1.9 // indirect github.com/containerd/go-runc v1.0.0 // indirect - github.com/containerd/imgcrypt v1.1.7 // indirect + github.com/containerd/imgcrypt v1.1.8 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/nri v0.6.0 // indirect + github.com/containerd/nri v0.6.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect - github.com/containerd/ttrpc v1.2.3 // indirect + github.com/containerd/ttrpc v1.2.4 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/containernetworking/cni v1.1.2 // indirect @@ -424,7 +424,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cobra v1.8.0 // indirect - github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect + github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -511,6 +511,6 @@ require ( sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - tags.cncf.io/container-device-interface v0.6.2 // indirect - tags.cncf.io/container-device-interface/specs-go v0.6.0 // indirect + tags.cncf.io/container-device-interface v0.7.2 // indirect + tags.cncf.io/container-device-interface/specs-go v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 6a5e398d7bf0..183f3110d230 100644 --- a/go.sum +++ b/go.sum @@ -269,7 +269,6 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -292,6 +291,7 @@ github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -404,35 +404,38 @@ github.com/containerd/go-cni v1.1.9 h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9 github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM= github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.1.7 h1:WSf9o9EQ0KGHiUx2ESFZ+PKf4nxK9BcvV/nJDX8RkB4= -github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k= +github.com/containerd/imgcrypt v1.1.8 h1:ZS7TuywcRNLoHpU0g+v4/PsKynl6TYlw5xDVWWoIyFA= +github.com/containerd/imgcrypt v1.1.8/go.mod h1:x6QvFIkMyO2qGIY2zXc88ivEzcbgvLdWjoZyGqDap5U= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/nri v0.6.0 h1:hdztxwL0gCS1CrCa9bvD1SoJiFN4jBuRQhplCvCPMj8= -github.com/containerd/nri v0.6.0/go.mod h1:F7OZfO4QTPqw5r87aq+syZJwiVvRYLIlHZiZDBV1W3A= +github.com/containerd/nri v0.6.1 h1:xSQ6elnQ4Ynidm9u49ARK9wRKHs80HCUI+bkXOxV4mA= +github.com/containerd/nri v0.6.1/go.mod h1:7+sX3wNx+LR7RzhjnJiUkFDhn18P5Bg/0VnJ/uXpRJM= github.com/containerd/stargz-snapshotter v0.15.1 h1:fpsP4kf/Z4n2EYnU0WT8ZCE3eiKDwikDhL6VwxIlgeA= github.com/containerd/stargz-snapshotter v0.15.1/go.mod h1:74D+J1m1RMXytLmWxegXWhtOSRHPWZKpKc2NdK3S+us= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/ttrpc v1.1.2/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= -github.com/containerd/ttrpc v1.2.3-0.20231030150553-baadfd8e7956/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= -github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/ttrpc v1.2.4 h1:eQCQK4h9dxDmpOb9QOOMh2NHTfzroH1IkmHiKZi05Oo= +github.com/containerd/ttrpc v1.2.4/go.mod h1:ojvb8SJBSch0XkqNO0L0YX/5NxR3UnVk2LzFKBK0upc= github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/containerd/zfs v1.1.0 h1:n7OZ7jZumLIqNJqXrEc/paBM840mORnmGdJDmAmJZHM= github.com/containerd/zfs v1.1.0/go.mod h1:oZF9wBnrnQjpWLaPKEinrx3TQ9a+W/RJO7Zb41d8YLE= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4= github.com/containernetworking/plugins v1.4.1 h1:+sJRRv8PKhLkXIl6tH1D7RMi+CbbHutDGU+ErLBORWA= github.com/containernetworking/plugins v1.4.1/go.mod h1:n6FFGKcaY4o2o5msgu/UImtoC+fpQXM3076VHfHbj60= -github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= +github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/LFMtH0FIRt34= github.com/containers/ocicrypt v1.1.10 h1:r7UR6o8+lyhkEywetubUUgcKFjOWOaWz8cEBrCPX0ic= github.com/containers/ocicrypt v1.1.10/go.mod h1:YfzSSr06PTHQwSTUKqDSjish9BeW1E4HUmreluQcMd8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -603,6 +606,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -669,6 +673,7 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -945,8 +950,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/k3s-io/containerd v1.7.15-k3s1 h1:X+GVNp3FiBy8rZzTMXShQJBmycPVi8vcwzsRBLdvqhM= -github.com/k3s-io/containerd v1.7.15-k3s1/go.mod h1:SOFk39t+bfDZC8jPYg11uxrzG3Fh30ZOociJwXfvk8Y= +github.com/k3s-io/containerd v1.7.17-k3s1 h1:jXPVFdg+vEwsx7amOvjPIx180ltbKBBZM5tfBaQtlzA= +github.com/k3s-io/containerd v1.7.17-k3s1/go.mod h1:T36IsoYQp97IT+64ws3GTq27V+M3518W11PDvOlBKPQ= github.com/k3s-io/cri-dockerd v0.3.12-k3s1 h1:jGTy2U1Nn8d9o23NwLV1NAigTBvePTA7XaZMifZ01Q4= github.com/k3s-io/cri-dockerd v0.3.12-k3s1/go.mod h1:S98trivsinxuNGQANgrZ9ComFqQkVv7vUvsXSNBRCFs= github.com/k3s-io/cri-tools v1.29.0-k3s1 h1:16IXZ5lbPCmZM8FkgSMAPkhI4O2wVGExe3qEZbisFT0= @@ -1043,6 +1048,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -1215,6 +1221,7 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc= @@ -1305,6 +1312,7 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= @@ -1329,6 +1337,7 @@ github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3Hig github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= @@ -1352,8 +1361,8 @@ github.com/open-policy-agent/opa v0.59.0/go.mod h1:rdJSkEc4oQ+0074/3Fsgno5bkPsYx github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -1476,7 +1485,9 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= @@ -1554,8 +1565,9 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= +github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1617,6 +1629,7 @@ github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4J github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= @@ -1683,8 +1696,8 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= @@ -1692,7 +1705,6 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1: go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= @@ -1721,7 +1733,6 @@ go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6 go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= @@ -1902,12 +1913,14 @@ golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= @@ -2024,6 +2037,7 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= @@ -2285,7 +2299,7 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= -tags.cncf.io/container-device-interface v0.6.2 h1:dThE6dtp/93ZDGhqaED2Pu374SOeUkBfuvkLuiTdwzg= -tags.cncf.io/container-device-interface v0.6.2/go.mod h1:Shusyhjs1A5Na/kqPVLL0KqnHQHuunol9LFeUNkuGVE= -tags.cncf.io/container-device-interface/specs-go v0.6.0 h1:V+tJJN6dqu8Vym6p+Ru+K5mJ49WL6Aoc5SJFSY0RLsQ= -tags.cncf.io/container-device-interface/specs-go v0.6.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80= +tags.cncf.io/container-device-interface v0.7.2 h1:MLqGnWfOr1wB7m08ieI4YJ3IoLKKozEnnNYBtacDPQU= +tags.cncf.io/container-device-interface v0.7.2/go.mod h1:Xb1PvXv2BhfNb3tla4r9JL129ck1Lxv9KuU6eVOfKto= +tags.cncf.io/container-device-interface/specs-go v0.7.0 h1:w/maMGVeLP6TIQJVYT5pbqTi8SCw/iHZ+n4ignuGHqg= +tags.cncf.io/container-device-interface/specs-go v0.7.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80= From 624a8021b2938fc4f0c53bc714e9610cf4a92557 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 17 May 2024 20:16:38 +0000 Subject: [PATCH 05/31] bump etcd to v3.5.13 Signed-off-by: Brad Davidson (cherry picked from commit bf8b15e7ae48064f661f43b36c822d4df138915e) Signed-off-by: Brad Davidson --- go.mod | 16 ++++++++-------- go.sum | 45 ++++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index a7dfd17eca4c..4eaf276dd51c 100644 --- a/go.mod +++ b/go.mod @@ -21,14 +21,14 @@ replace ( github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.20-k3s1 github.com/ugorji/go => github.com/ugorji/go v1.2.11 - go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1 - go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1 - go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1 - go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1 - go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.9-k3s1 - go.etcd.io/etcd/pkg/v3 => github.com/k3s-io/etcd/pkg/v3 v3.5.9-k3s1 - go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1 - go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 + go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1 + go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1 + go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.13-k3s1 + go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.13-k3s1 + go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.13-k3s1 + go.etcd.io/etcd/pkg/v3 => github.com/k3s-io/etcd/pkg/v3 v3.5.13-k3s1 + go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1 + go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1 go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful => go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 golang.org/x/crypto => golang.org/x/crypto v0.17.0 diff --git a/go.sum b/go.sum index 183f3110d230..2c1e2a18ca3c 100644 --- a/go.sum +++ b/go.sum @@ -339,7 +339,6 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/canonical/go-dqlite v1.5.1 h1:1YjtIrFsC1A3XlgsX38ARAiKhvkZS63PqsEd8z3T4yU= github.com/canonical/go-dqlite v1.5.1/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -956,22 +955,22 @@ github.com/k3s-io/cri-dockerd v0.3.12-k3s1 h1:jGTy2U1Nn8d9o23NwLV1NAigTBvePTA7Xa github.com/k3s-io/cri-dockerd v0.3.12-k3s1/go.mod h1:S98trivsinxuNGQANgrZ9ComFqQkVv7vUvsXSNBRCFs= github.com/k3s-io/cri-tools v1.29.0-k3s1 h1:16IXZ5lbPCmZM8FkgSMAPkhI4O2wVGExe3qEZbisFT0= github.com/k3s-io/cri-tools v1.29.0-k3s1/go.mod h1:fZeWlv+qq4gZ005I13j4JcvgFb6ZobVTtON3PqM5JVc= -github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1 h1:y4ont0HdnS7gtWNTXM8gahpKjAHtctgON/sjVRthlZY= -github.com/k3s-io/etcd/api/v3 v3.5.9-k3s1/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1 h1:LJFtNHaBJg2BqFE3lRxWZkUsKTYLbh0s0NCXPMjW3cg= -github.com/k3s-io/etcd/client/pkg/v3 v3.5.9-k3s1/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1 h1:/IyNFC677PfYafrm4sWPShbmw1bkpvEio6YaxxFA9cU= -github.com/k3s-io/etcd/client/v2 v2.305.9-k3s1/go.mod h1:0NBdNx9wbxtEQLwAQtrDHwx58m02vXpDcgSYI2seohQ= -github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1 h1:Knr/8l7Sx92zUyevYO0gIO5P6EEc6ztvRO5EzSnMy+A= -github.com/k3s-io/etcd/client/v3 v3.5.9-k3s1/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= -github.com/k3s-io/etcd/etcdutl/v3 v3.5.9-k3s1 h1:IkCP2oKkQwyu+ad4FuToJu9SOuEVQZwCpjXj6SJqwvs= -github.com/k3s-io/etcd/etcdutl/v3 v3.5.9-k3s1/go.mod h1:rQ6z0HAAxVgYwBTWJbs3ei8gMYiNQzF51lQ2kI+6LZU= -github.com/k3s-io/etcd/pkg/v3 v3.5.9-k3s1 h1:au8ekw/8/wNokQ5dHB7MEdStKMCNBNm4tDsPWEMqW4Y= -github.com/k3s-io/etcd/pkg/v3 v3.5.9-k3s1/go.mod h1:BZl0SAShQFk0IpLWR78T/+pyt8AruMHhTNNX73hkNVY= -github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1 h1:nlix2+EM1UDofoHgp/X2VHzMvJW7oYbZbEinblZusNc= -github.com/k3s-io/etcd/raft/v3 v3.5.9-k3s1/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg= -github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZe8rfaBPfbXo= -github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= +github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1 h1:aq6fxlEKdwCooLE3HOR6227U51DEvOw3DEbriJxD2QM= +github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= +github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1 h1:t2I25UtBvohVAhlyXpYjd/Lznm+ybxNhvs3cnEGsF4Y= +github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= +github.com/k3s-io/etcd/client/v2 v2.305.13-k3s1 h1:lvIdlAI6xRIHSUJC43sJx9lmxehq2quGb+8z5TJldGg= +github.com/k3s-io/etcd/client/v2 v2.305.13-k3s1/go.mod h1:iQnL7fepbiomdXMb3om1rHq96htNNGv2sJkEcZGDRRg= +github.com/k3s-io/etcd/client/v3 v3.5.13-k3s1 h1:/D6KAEGVzwivnjxZ5CzVIykVloLoKB/TBeKw2tKKVQ0= +github.com/k3s-io/etcd/client/v3 v3.5.13-k3s1/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= +github.com/k3s-io/etcd/etcdutl/v3 v3.5.13-k3s1 h1:fIt+PVHCeINM5fl9OfMI+o9BJKf951pRiVcCytFW97c= +github.com/k3s-io/etcd/etcdutl/v3 v3.5.13-k3s1/go.mod h1:2vhvTIQobP+Cb04qzlcbKGvX6J5oq/N1kquk1yCDIQY= +github.com/k3s-io/etcd/pkg/v3 v3.5.13-k3s1 h1:uLU/SnBuhtSkdBk830x0pseHSsQQvh99C3deG6nc9d0= +github.com/k3s-io/etcd/pkg/v3 v3.5.13-k3s1/go.mod h1:N+4PLrp7agI/Viy+dUYpX7iRtSPvKq+w8Y14d1vX+m0= +github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1 h1:yexUwAPPdmYfIMWOj6sSyJ2nEe8QOrFzNuvYGRAsm5E= +github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1/go.mod h1:uUFibGLn2Ksm2URMxN1fICGhk8Wu96EfDQyuLhAcAmw= +github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1 h1:Pqcxkg7V60c26ZpHoekP9QoUdLuduxFn827A/5CIwm4= +github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1/go.mod h1:K/8nbsGupHqmr5MkgaZpLlH1QdX1pcNQLAkODy44XcQ= github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64= github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= github.com/k3s-io/kine v0.11.7 h1:+I4TrxozQv4cdmD8RULI35r4o5G+A7gOD3F75lfjDP0= @@ -1704,19 +1703,19 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= go.opentelemetry.io/contrib/propagators/b3 v1.19.0/go.mod h1:OzCmE2IVS+asTI+odXQstRGVfXQ4bXv9nMBRK0nNyqQ= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= @@ -1724,22 +1723,22 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/sdk v1.20.0/go.mod h1:rmkSx1cZCm/tn16iWDn1GQbLtsW/LvsdEEFzCSRM6V0= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= From 90d1f03eee0ef22f5ea0155825435fb6a5dfe73e Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 17 May 2024 00:19:09 +0000 Subject: [PATCH 06/31] Bump spegel version Signed-off-by: Brad Davidson (cherry picked from commit 5cf4d757495d850ee0a3c6fd306024d789be74b1) Signed-off-by: Brad Davidson --- Dockerfile.test | 2 +- go.mod | 27 +++++------------------ go.sum | 52 ++++++-------------------------------------- pkg/spegel/spegel.go | 6 +++-- 4 files changed, 18 insertions(+), 69 deletions(-) diff --git a/Dockerfile.test b/Dockerfile.test index 1996f39b8d1a..23bfc7112a82 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -44,7 +44,7 @@ RUN vagrant box add generic/ubuntu2204 --provider libvirt --force RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \ chmod +x ./kubectl; \ mv ./kubectl /usr/local/bin/kubectl -RUN GO_VERSION=go1.21.5; \ +RUN GO_VERSION=go1.21.9; \ curl -O -L "https://golang.org/dl/${GO_VERSION}.linux-amd64.tar.gz"; \ rm -rf /usr/local/go; \ tar -C /usr/local -xzf ${GO_VERSION}.linux-amd64.tar.gz; diff --git a/go.mod b/go.mod index 4eaf276dd51c..670a43a6d56c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/k3s-io/k3s -go 1.21 +go 1.21.9 replace ( github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.11.0 @@ -19,7 +19,7 @@ replace ( github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0 github.com/prometheus/common => github.com/prometheus/common v0.45.0 github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d - github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.20-k3s1 + github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314 github.com/ugorji/go => github.com/ugorji/go v1.2.11 go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.13-k3s1 go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.13-k3s1 @@ -81,7 +81,7 @@ require ( github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000 github.com/containerd/aufs v1.0.0 github.com/containerd/cgroups/v3 v3.0.2 - github.com/containerd/containerd v1.7.14 + github.com/containerd/containerd v1.7.16 github.com/containerd/fuse-overlayfs-snapshotter v1.0.8 github.com/containerd/stargz-snapshotter v0.15.1 github.com/containerd/zfs v1.1.0 @@ -120,7 +120,7 @@ require ( github.com/opencontainers/selinux v1.11.0 github.com/otiai10/copy v1.7.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/prometheus/common v0.49.0 github.com/rancher/dynamiclistener v0.3.6 github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29 @@ -144,7 +144,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/crypto v0.22.0 golang.org/x/net v0.24.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.7.0 golang.org/x/sys v0.19.0 google.golang.org/grpc v1.63.2 gopkg.in/yaml.v2 v2.4.0 @@ -198,13 +198,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c // indirect - github.com/bytedance/sonic v1.9.1 // indirect github.com/canonical/go-dqlite v1.5.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/container-storage-interface/spec v1.8.0 // indirect github.com/containerd/btrfs/v2 v2.0.0 // indirect @@ -254,20 +252,13 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fvbommel/sortorder v1.1.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/swag v0.22.9 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect @@ -326,7 +317,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/leodido/go-urn v1.2.4 // indirect github.com/libopenstorage/openstorage v1.0.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect @@ -402,7 +392,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -419,7 +409,6 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/seccomp/libseccomp-golang v0.10.0 // indirect github.com/shengdoushi/base58 v1.0.0 // indirect - github.com/slok/go-http-metrics v0.10.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect @@ -431,14 +420,11 @@ require ( github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect github.com/urfave/cli/v2 v2.26.0 // indirect github.com/vbatts/tar-split v0.11.5 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/vmware/govmomi v0.30.6 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect - github.com/xenitab/pkg/gin v0.0.9 // indirect github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect @@ -463,7 +449,6 @@ require ( go.uber.org/fx v1.20.1 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.3.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect diff --git a/go.sum b/go.sum index 2c1e2a18ca3c..54c42c25995b 100644 --- a/go.sum +++ b/go.sum @@ -334,9 +334,6 @@ github.com/bronze1man/goStrongswanVici v0.0.0-20221114103242-3f6dc524986c/go.mod github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/canonical/go-dqlite v1.5.1 h1:1YjtIrFsC1A3XlgsX38ARAiKhvkZS63PqsEd8z3T4yU= github.com/canonical/go-dqlite v1.5.1/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -353,9 +350,6 @@ github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNS github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/checkpoint-restore/go-criu/v5 v5.3.0 h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -582,14 +576,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= @@ -626,9 +614,8 @@ github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5 h1:aj5xnNwNY2GCk38Vga4FMm4GSX1bDzu8Z5JcQQdmOqg= github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= -github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= @@ -641,14 +628,6 @@ github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZC github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -661,8 +640,6 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -1037,8 +1014,8 @@ github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.5-k github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.5-k3s1/go.mod h1:3nvUgy9DAoVbLCBJcIBDCldv+vAc7hcHl6xJFRSpvb4= github.com/k3s-io/runc v1.1.12-k3s1 h1:p2x48K2BbRdF8crLEB4xoJ1pdjSprlvNNGpYBBULHL4= github.com/k3s-io/runc v1.1.12-k3s1/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= -github.com/k3s-io/spegel v0.0.20-k3s1 h1:alwhmC5jbaXrVEImbAdvmND8DtCi97/cRABRSkiEiUw= -github.com/k3s-io/spegel v0.0.20-k3s1/go.mod h1:4neUkvTVGk6+Z+oiX40k15F21EsA/RnbcJHjXHlACCs= +github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314 h1:TrZb/yM0OtBuifPXlKaOfcxpJqzakA8+KsoO4c69ZLM= +github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314/go.mod h1:bMHfSjj1+Zf5VITCZe/wLjuni6rYAj/DjPU/kIVnhfA= github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= @@ -1079,8 +1056,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= @@ -1395,8 +1370,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= -github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= @@ -1525,8 +1500,6 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= -github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1603,13 +1576,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/tonglil/buflogr v1.0.1 h1:WXFZLKxLfqcVSmckwiMCF8jJwjIgmStJmg63YKRF1p0= -github.com/tonglil/buflogr v1.0.1/go.mod h1:yYWwvSpn/3uAaqjf6mJg/XMiAciaR0QcRJH2gJGDxNE= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1650,8 +1617,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xenitab/pkg/gin v0.0.9 h1:BGdxnKoXAJBkthQTwQdaRdN7jTiNO+/C8hIexBrasfU= -github.com/xenitab/pkg/gin v0.0.9/go.mod h1:8rzqJ8X5KJOo31PBOD4/Wtlt2ac8hCjN1mpOf1YAFs4= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= @@ -1779,9 +1744,6 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= @@ -1910,8 +1872,8 @@ golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/pkg/spegel/spegel.go b/pkg/spegel/spegel.go index bdfd8291b299..e9dbd1192461 100644 --- a/pkg/spegel/spegel.go +++ b/pkg/spegel/spegel.go @@ -147,7 +147,8 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { ipfslog.SetAllLoggers(level) // Get containerd client - ociClient, err := oci.NewContainerd(nodeConfig.Containerd.Address, registryNamespace, nodeConfig.Containerd.Registry, urls) + ociOpts := []oci.Option{oci.WithContentPath(filepath.Join(nodeConfig.Containerd.Root, "io.containerd.content.v1.content"))} + ociClient, err := oci.NewContainerd(nodeConfig.Containerd.Address, registryNamespace, nodeConfig.Containerd.Registry, urls, ociOpts...) if err != nil { return errors.Wrap(err, "failed to create OCI client") } @@ -222,9 +223,10 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { registry.WithResolveRetries(resolveRetries), registry.WithResolveTimeout(resolveTimeout), registry.WithTransport(client.Transport), + registry.WithLogger(logr.FromContextOrDiscard(ctx)), } reg := registry.NewRegistry(ociClient, router, registryOpts...) - regSvr := reg.Server(":"+c.RegistryPort, logr.FromContextOrDiscard(ctx)) + regSvr := reg.Server(":" + c.RegistryPort) // Close router on shutdown go func() { From 311335c0c3dcf666b1c9f4b8db8a69f7af538b31 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 17 Apr 2024 17:19:32 +0000 Subject: [PATCH 07/31] Fix issue with local traffic policy for single-stack services on dual-stack nodes. Just enable IP forwarding for all address families regardless of service address families. Signed-off-by: Brad Davidson (cherry picked from commit 095ecdb0346c038b0c16c39f6f66ad4f67ad10b9) Signed-off-by: Brad Davidson --- pkg/cloudprovider/servicelb.go | 40 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/pkg/cloudprovider/servicelb.go b/pkg/cloudprovider/servicelb.go index fa2e3d4ccdb7..20d699b42786 100644 --- a/pkg/cloudprovider/servicelb.go +++ b/pkg/cloudprovider/servicelb.go @@ -23,6 +23,7 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/util/retry" @@ -320,10 +321,8 @@ func (k *k3s) patchStatus(svc *core.Service, previousStatus, newStatus *core.Loa // If at least one node has External IPs available, only external IPs are returned. // If no nodes have External IPs set, the Internal IPs of all nodes running pods are returned. func (k *k3s) podIPs(pods []*core.Pod, svc *core.Service, readyNodes map[string]bool) ([]string, error) { - // Go doesn't have sets so we stuff things into a map of bools and then get lists of keys - // to determine the unique set of IPs in use by pods. - extIPs := map[string]bool{} - intIPs := map[string]bool{} + extIPs := sets.Set[string]{} + intIPs := sets.Set[string]{} for _, pod := range pods { if pod.Spec.NodeName == "" || pod.Status.PodIP == "" { @@ -345,25 +344,18 @@ func (k *k3s) podIPs(pods []*core.Pod, svc *core.Service, readyNodes map[string] for _, addr := range node.Status.Addresses { if addr.Type == core.NodeExternalIP { - extIPs[addr.Address] = true + extIPs.Insert(addr.Address) } else if addr.Type == core.NodeInternalIP { - intIPs[addr.Address] = true + intIPs.Insert(addr.Address) } } } - keys := func(addrs map[string]bool) (ips []string) { - for k := range addrs { - ips = append(ips, k) - } - return ips - } - var ips []string - if len(extIPs) > 0 { - ips = keys(extIPs) + if extIPs.Len() > 0 { + ips = extIPs.UnsortedList() } else { - ips = keys(intIPs) + ips = intIPs.UnsortedList() } ips, err := filterByIPFamily(ips, svc) @@ -443,18 +435,11 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { } sourceRanges := strings.Join(sourceRangesSet.StringSlice(), ",") - var sysctls []core.Sysctl for _, ipFamily := range svc.Spec.IPFamilies { - switch ipFamily { - case core.IPv4Protocol: - sysctls = append(sysctls, core.Sysctl{Name: "net.ipv4.ip_forward", Value: "1"}) - case core.IPv6Protocol: - sysctls = append(sysctls, core.Sysctl{Name: "net.ipv6.conf.all.forwarding", Value: "1"}) + if ipFamily == core.IPv6Protocol && sourceRanges == "0.0.0.0/0" { // The upstream default load-balancer source range only includes IPv4, even if the service is IPv6-only or dual-stack. // If using the default range, and IPv6 is enabled, also allow IPv6. - if sourceRanges == "0.0.0.0/0" { - sourceRanges += ",::/0" - } + sourceRanges += ",::/0" } } @@ -490,7 +475,10 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { ServiceAccountName: "svclb", AutomountServiceAccountToken: utilsptr.To(false), SecurityContext: &core.PodSecurityContext{ - Sysctls: sysctls, + Sysctls: []core.Sysctl{ + {Name: "net.ipv4.ip_forward", Value: "1"}, + {Name: "net.ipv6.conf.all.forwarding", Value: "1"}, + }, }, Tolerations: []core.Toleration{ { From d5b1cc76e32933144a36fc938dee50a03535a9ee Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 17 Apr 2024 18:15:36 +0000 Subject: [PATCH 08/31] Update local-path-provisioner helper script Signed-off-by: Brad Davidson (cherry picked from commit b453630478a71f31dea5bd72ec0326c980ee4c79) Signed-off-by: Brad Davidson --- manifests/local-storage.yaml | 36 +++++------------------------- pkg/deploy/zz_generated_bindata.go | 2 +- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/manifests/local-storage.yaml b/manifests/local-storage.yaml index 1714af1fb1e0..35f85af42bbb 100644 --- a/manifests/local-storage.yaml +++ b/manifests/local-storage.yaml @@ -115,39 +115,13 @@ data: } setup: |- #!/bin/sh - while getopts "m:s:p:" opt - do - case $opt in - p) - absolutePath=$OPTARG - ;; - s) - sizeInBytes=$OPTARG - ;; - m) - volMode=$OPTARG - ;; - esac - done - mkdir -m 0777 -p ${absolutePath} - chmod 700 ${absolutePath}/.. + set -eu + mkdir -m 0777 -p "${VOL_DIR}" + chmod 700 "${VOL_DIR}/.." teardown: |- #!/bin/sh - while getopts "m:s:p:" opt - do - case $opt in - p) - absolutePath=$OPTARG - ;; - s) - sizeInBytes=$OPTARG - ;; - m) - volMode=$OPTARG - ;; - esac - done - rm -rf ${absolutePath} + set -eu + rm -rf "${VOL_DIR}" helperPod.yaml: |- apiVersion: v1 kind: Pod diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 0a4398cac144..b85b2b10819a 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -132,7 +132,7 @@ func corednsYaml() (*asset, error) { return a, nil } -var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\xdf\x6f\xdb\xb6\x13\x7f\xd7\x5f\x71\x5f\x7d\x9b\x87\x0d\xa5\x9d\x6c\x40\x33\xb0\xd8\x83\x9b\x38\x69\x80\xc4\x36\x6c\xb7\x43\x51\x14\x06\x2d\x9d\x6d\x36\x14\x49\x90\x94\x5b\x35\xcb\xff\x3e\x90\x94\x1d\x29\x71\x12\x07\xdb\xde\xa6\x17\x81\xe4\xfd\xe2\xe7\x3e\x77\x47\xa6\xf9\x47\x34\x96\x2b\x49\x61\x7d\x94\x5c\x73\x99\x53\x98\xa0\x59\xf3\x0c\x7b\x59\xa6\x4a\xe9\x92\x02\x1d\xcb\x99\x63\x34\x01\x90\xac\x40\x0a\x42\x65\x4c\x10\xcd\xdc\x8a\x68\xa3\xd6\xdc\xeb\xa3\x21\x36\xea\x11\x56\x2b\x46\x71\xab\x59\x86\x14\xae\xcb\x39\x12\x5b\x59\x87\x45\x42\x08\x49\x9a\x9e\xcd\x9c\x65\x1d\x56\xba\x95\x32\xfc\x07\x73\x5c\xc9\xce\xf5\x6f\xb6\xc3\x55\x77\x1b\xd3\x89\x28\xad\x43\x33\x56\x02\xf7\x0f\xc8\x78\x69\x53\x0a\xb4\x34\x21\xc0\x34\x3f\x37\xaa\xd4\x96\xc2\xe7\x34\xfd\x92\x00\x18\xb4\xaa\x34\x19\x86\x1d\xa9\x72\xb4\xe9\x6b\x48\xb5\x0f\xcb\x3a\x94\x6e\xad\x44\x59\x60\x26\x18\x2f\xc2\x49\xa6\xe4\x82\x2f\x0b\xa6\xa3\x9c\xca\x6d\x57\xa8\x65\x30\xb5\x46\x33\x0f\x66\x96\xe8\xfc\xa1\xe0\x36\xfc\xbf\x31\x97\xad\xd2\x2f\xcf\xbb\x47\x99\x6b\xc5\xa5\xdb\x19\xc2\xd6\x5f\xdb\xd7\xcf\x7b\x19\x5e\xa3\xb7\xda\x52\xcc\x0c\x32\x87\xc1\xe8\xee\xf8\xac\x53\x86\x2d\xb1\x4e\xc3\x43\xa3\xf5\x79\x26\x98\xb5\x68\xf7\x43\xe0\x6f\x25\xfd\x1d\x97\x39\x97\xcb\xfd\x73\x3f\xe7\x32\x4f\x3c\x01\xc6\xb8\xf0\xc2\x9b\xeb\x3d\xe1\x38\x01\x78\x48\xb6\x7d\x28\x66\xcb\xf9\x57\xcc\x5c\x60\xd9\xce\x12\xfa\xb7\x0a\x87\x69\x6d\xef\xe0\x3a\x45\x2d\x54\x55\xe0\x0b\x6a\xf6\x71\x57\x56\x63\x46\x43\xda\xa3\xec\x7b\xee\x73\x5e\x5d\xf2\x82\x3b\x0a\x87\x09\x80\x75\x86\x39\x5c\x56\x5e\x0a\xc0\x55\x1a\x29\x8c\x95\x10\x5c\x2e\x3f\xe8\x9c\x39\x0c\xfb\xa6\xb9\x13\x45\x01\x0a\xf6\xfd\x83\x64\x6b\xc6\x05\x9b\x0b\xa4\x70\xe4\xcd\xa1\xc0\xcc\x29\x13\x65\x0a\xcf\x9a\x4b\x36\x47\x61\x37\x4a\x4c\xeb\x27\xae\xe1\xb0\xd0\x62\xeb\xa2\x79\x7f\xff\x89\x96\xa5\xe7\x6c\x01\x6c\x6e\xef\x3f\x6d\xb8\x32\xdc\x55\x27\x9e\xec\x83\x00\x66\x1a\x41\x22\xbe\x67\x90\xcc\x70\xc7\x33\x26\xd2\x5a\xde\xb6\x72\x3f\x78\x59\xe2\x03\x94\x4a\xa0\x09\xc4\x6c\x44\x0c\x40\xe0\x1a\x2b\x0a\xe9\x49\xed\xaf\x97\xe7\x4a\xda\xa1\x14\x55\xda\x90\x02\x50\xda\x6b\x2b\x43\x21\xed\x7f\xe7\xd6\xd9\x74\x87\x91\x10\xb9\x27\x6f\xc7\x27\xdd\x48\x74\x18\x6a\x2f\x53\xd2\x19\x25\x88\x16\x4c\xe2\x0b\xec\x02\xe0\x62\x81\x99\xa3\x90\x0e\xd4\x24\x5b\x61\x5e\x0a\x7c\x89\xe3\x82\xf9\x92\xfb\xa7\x3c\xfa\x6b\x30\x2e\xd1\x6c\x11\x24\xcf\xd5\x41\xfc\x78\xc1\x96\x3e\xc1\x07\x37\x93\x4f\x93\x69\xff\x6a\x76\xda\x3f\xeb\x7d\xb8\x9c\xce\xc6\xfd\xf3\x8b\xc9\x74\xfc\xe9\xf6\xc0\x30\x99\xad\xd0\x74\x77\x5b\xa2\xeb\xc3\xce\x61\xe7\x97\x37\x69\xdb\xe4\xa8\x14\x62\xa4\x04\xcf\x2a\x0a\x17\x8b\x81\x72\x23\x83\x16\xb7\x29\xf7\x11\x17\x05\x93\xf9\x5d\xc2\xc9\x73\xa1\x12\xb0\x8e\x19\xd7\x58\x13\x12\x27\x54\x63\xab\x8b\x2e\xeb\xc6\xdd\xfa\xd7\xf9\x6a\x95\xdc\x4a\xc4\xf9\x72\xe5\xd9\x67\x9b\xbe\x23\x58\x51\x83\x44\xa1\x06\xf6\x85\x97\x1f\x31\xb7\xa2\x2d\x07\x5b\x09\x94\xeb\x87\xc6\x46\xc3\xd3\xd9\xa0\x77\xd5\x9f\x8c\x7a\x27\xfd\x86\xb1\x35\x13\x25\x9e\x19\x55\xd0\x56\x76\x17\x1c\x45\x5e\x37\xef\x07\xfb\xd1\xf7\xa6\xca\x3b\xdb\x1e\x96\x34\x6f\xf5\x82\x0b\xc5\xfd\x2b\xa6\xdb\xde\x1e\x50\xa6\xc6\xf7\x7e\x1f\x6e\x8f\xcb\xbb\x8e\x3c\x89\xfb\xa1\x73\x3c\xd9\x93\xfd\x80\x92\x52\xb9\x66\xd5\xe7\xb8\x60\xa5\x70\x1f\x43\xac\xd3\xd0\x5e\xd3\xa0\x11\xa9\xd5\x1c\xc1\xf7\x6a\x89\x5b\x52\x2b\x93\x70\x4c\x21\x75\xa6\xc4\x34\x69\xf2\x14\x6a\x1e\x7b\x85\x46\x20\x11\x9a\x7a\xdc\x5e\xa9\x1c\x29\xfc\xc1\xb8\x3b\x53\xe6\x8c\x1b\xeb\x4e\x94\xb4\x65\x81\x26\x31\xf1\x5d\xb4\xe1\xf4\x29\x0a\x74\x18\x80\xa9\x67\xe8\x06\xd1\xe4\xde\x1b\xf3\xc9\xd1\xb4\xe5\xef\x23\x53\x69\xa3\xd8\xa0\x32\x85\x3f\x49\x00\xe4\xa6\x4e\x5d\x68\x31\x9e\x20\x57\x4c\xa7\xf4\x73\xbd\x7b\xb3\x4d\x6c\x38\x4f\x69\xba\xa9\xec\x51\x6f\xfa\x7e\x76\x36\x1c\xcf\x06\xc3\xc1\xec\xf2\x62\x32\xed\x9f\xce\x06\xc3\xd3\xfe\x24\x7d\x7d\xa7\xe3\xa3\xb3\x29\xfd\x9c\x1e\xdc\x6c\xf4\x2e\x87\x27\xbd\xcb\xd9\x64\x3a\x1c\xf7\xce\xfb\xc1\xca\xed\x41\x78\x09\xf9\xef\xb6\xfe\xc7\xf5\x6d\x98\x6f\xce\xbf\x3e\xea\x60\xff\xff\xbf\xee\x9c\xcb\xae\x5d\x85\xd5\xb7\x15\x17\x08\x4b\x74\x4a\x3b\x0b\x69\x41\x2d\xd5\x34\x05\xa5\x63\x75\xe7\xea\xae\x4d\x30\x8b\xf0\x4a\x69\x07\x5c\xb6\xa8\xaa\x7f\x6a\x2d\xd9\xdc\x2a\x51\xba\x80\xc3\xef\xaf\x86\xa3\x69\x6f\x7c\xde\x12\x78\xfb\xb6\xb5\xb4\x6d\x75\xcb\x7f\xe0\x85\x7c\x57\x39\xb4\xfb\x68\x17\x6d\xed\xb5\x12\x9e\x39\xcf\x69\xa2\x65\x59\x7d\x3f\x19\x8b\xb1\xb8\xce\xb9\x01\x52\xc0\xe1\xf1\xf1\x31\x10\x0d\xaf\x6e\x9a\x17\x89\xa0\x66\xab\x42\xe5\x70\x7c\x78\x78\xff\xb4\xdb\xe9\x84\x87\x00\x33\xb9\xfa\x26\xff\x83\xfa\x49\xa8\x4d\x01\xc4\x2c\x76\x00\xbc\x42\xa1\xd1\x8c\x54\xde\xa9\x58\x21\xb6\x28\xde\xab\x62\xbf\x15\x0b\x7d\xa4\xf2\x9d\x4f\xae\x58\xdb\xd1\x1a\xd1\xb5\x50\xf3\x5d\xf5\xf8\x8c\xbe\xa7\x04\x2f\x9b\xcb\x05\x37\x46\x19\xcc\x89\xe0\x73\xc3\x4c\x45\xe6\xa5\xad\xe6\xea\x3b\x3d\xea\xfc\xfa\xa6\x73\xb4\xef\x60\xfe\x2b\x00\x00\xff\xff\x23\x2c\xa0\x6c\x1b\x0f\x00\x00") +var _localStorageYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\xdf\x6f\xdb\xb6\x13\x7f\xd7\x5f\x71\x5f\x7d\x97\x97\xa1\x94\x93\x0d\x68\x06\xbe\x79\xb1\xd3\x06\x70\x6c\xc3\x76\x3b\x14\x45\x61\xd0\xd4\xd9\x66\x43\x91\x04\x49\xb9\xf5\xb2\xfc\xef\x03\x49\xd9\x91\x93\x34\x71\xb0\x4d\x2f\x82\x8e\x77\x9f\x3b\xde\xe7\x7e\x88\x19\xf1\x11\xad\x13\x5a\x51\xd8\x9c\x65\x37\x42\x95\x14\xa6\x68\x37\x82\x63\x97\x73\x5d\x2b\x9f\x55\xe8\x59\xc9\x3c\xa3\x19\x80\x62\x15\x52\x90\x9a\x33\x49\x0c\xf3\x6b\x62\xac\xde\x88\x60\x8f\x96\xb8\x64\x47\x58\x63\x98\xd4\x9d\x61\x1c\x29\xdc\xd4\x0b\x24\x6e\xeb\x3c\x56\x19\x21\x24\x6b\x7b\xb6\x0b\xc6\x0b\x56\xfb\xb5\xb6\xe2\x4f\xe6\x85\x56\xc5\xcd\x6f\xae\x10\xba\xb3\x8f\xe9\x42\xd6\xce\xa3\x9d\x68\x89\xc7\x07\x64\x83\xb6\xad\x25\x3a\x9a\x11\x60\x46\xbc\xb3\xba\x36\x8e\xc2\xe7\x3c\xff\x92\x01\x58\x74\xba\xb6\x1c\xa3\x44\xe9\x12\x5d\xfe\x06\x72\x13\xc2\x72\x1e\x95\xdf\x68\x59\x57\xc8\x25\x13\x55\x3c\xe1\x5a\x2d\xc5\xaa\x62\x26\xe9\xe9\xd2\x75\xa4\x5e\x45\xa8\x0d\xda\x45\x84\x59\xa1\x0f\x87\x52\xb8\xf8\xfe\xc6\x3c\x5f\xe7\x5f\x5e\x76\x8f\xaa\x34\x5a\x28\xff\x64\x08\x7b\x7f\x87\xbe\x7e\x3e\x0a\x78\x83\x01\xf5\xc0\x90\x5b\x64\x1e\x23\xe8\xd3\xf1\x39\xaf\x2d\x5b\x61\x43\xc3\x63\xd0\xe6\x9c\x4b\xe6\x1c\xba\xe3\x32\xf0\x8f\x48\xff\x5d\xa8\x52\xa8\xd5\xf1\xdc\x2f\x84\x2a\xb3\x50\x00\x13\x5c\x06\xe5\xdd\xf5\x9e\x71\x9c\x01\x3c\x2e\xb6\x63\x4a\xcc\xd5\x8b\xaf\xc8\x7d\xac\xb2\x27\x5b\xe8\xbf\x6a\x1c\x66\x8c\xbb\x4f\x57\x0f\x8d\xd4\xdb\x0a\x5f\xd1\xb3\x3f\x76\xe5\x0c\x72\x1a\x69\x4f\xba\xef\x45\xe0\x7c\x3b\x10\x95\xf0\x14\x4e\x33\x00\xe7\x2d\xf3\xb8\xda\x06\x2d\x00\xbf\x35\x48\x61\xa2\xa5\x14\x6a\xf5\xc1\x94\xcc\x63\x94\xdb\xb6\x24\xa9\x02\x54\xec\xfb\x07\xc5\x36\x4c\x48\xb6\x90\x48\xe1\x2c\xc0\xa1\x44\xee\xb5\x4d\x3a\x55\xa8\x9a\x01\x5b\xa0\x74\x3b\x23\x66\xcc\x33\xd7\xf0\x58\x19\xb9\x77\xd1\xbe\x7f\x78\xe4\x01\xd2\x4b\x58\x00\xbb\xdb\x87\xc7\x58\xa1\xad\xf0\xdb\x8b\x50\xec\xc3\x98\xcc\x3c\x25\x89\x84\x99\x41\xb8\x15\x5e\x70\x26\xf3\x46\xdf\x1d\x70\x3f\x7c\x1d\xf1\x31\x95\x5a\xa2\x8d\x85\xd9\x8a\x18\x80\xc0\x0d\x6e\x29\xe4\x17\x8d\xbf\x6e\x59\x6a\xe5\x46\x4a\x6e\xf3\x96\x16\x80\x36\xc1\x5a\x5b\x0a\x79\xff\xbb\x70\xde\xe5\x4f\x80\xc4\xc8\x43\xf1\x16\x81\x74\xab\xd0\x63\xec\x3d\xae\x95\xb7\x5a\x12\x23\x99\xc2\x57\xe0\x02\xe0\x72\x89\xdc\x53\xc8\x87\x7a\xca\xd7\x58\xd6\x12\x5f\xe3\xb8\x62\xa1\xe5\xfe\x2d\x8f\xe1\x1a\x4c\x28\xb4\xfb\x0c\x92\x97\xfa\x20\x3d\xa2\x62\xab\x40\xf0\xc9\xed\xf4\xd3\x74\xd6\xbf\x9e\xf7\xfa\x97\xdd\x0f\x83\xd9\x7c\xd2\x7f\x77\x35\x9d\x4d\x3e\xdd\x9d\x58\xa6\xf8\x1a\x6d\xe7\x69\x24\xba\x39\x2d\x4e\x8b\x5f\xde\xe6\x87\x90\xe3\x5a\xca\xb1\x96\x82\x6f\x29\x5c\x2d\x87\xda\x8f\x2d\x3a\xdc\x53\x1e\x22\xae\x2a\xa6\xca\x7b\xc2\xc9\x4b\xa1\x12\x70\x9e\x59\xdf\xfa\x26\x24\x6d\xa8\x96\xa8\x83\x9e\x77\x92\xb4\x79\x15\x5f\x9d\x56\x7b\x8d\xb4\x5f\xae\x43\xf5\xb9\xb6\xef\x94\xac\x64\x41\x92\x52\x2b\xf7\x55\xd0\x1f\x33\xbf\xa6\x07\x0e\xf6\x1a\xa8\x36\x8f\xc1\xc6\xa3\xde\x7c\xd8\xbd\xee\x4f\xc7\xdd\x8b\x7e\x0b\x6c\xc3\x64\x8d\x97\x56\x57\xf4\x80\xdd\xa5\x40\x59\x36\xc3\xfb\x91\x3c\xf9\xde\x75\x79\xb1\x9f\x61\x59\xfb\x56\xaf\xb8\x50\x92\x5f\x33\x73\xe8\xed\x51\xc9\x34\xf9\x7d\x38\x87\x0f\xd7\xe5\xfd\x44\x9e\x26\x79\x9c\x1c\xcf\xce\xe4\xb0\xa0\x94\xd2\xbe\xdd\xf5\x25\x2e\x59\x2d\xfd\xc7\x18\xeb\x2c\x8e\xd7\x3c\x5a\xa4\xd2\x6a\xaf\xe0\x07\xbd\x24\x1c\x69\x8c\x49\x3c\xa6\x90\x7b\x5b\x63\x9e\xb5\xeb\x14\x9a\x3a\x0e\x06\xad\x40\x52\x6a\x9a\x75\x7b\xad\x4b\xa4\xf0\x07\x13\xfe\x52\xdb\x4b\x61\x9d\xbf\xd0\xca\xd5\x15\xda\xcc\xa6\xff\xa2\x5d\x4d\xf7\x50\xa2\xc7\x98\x98\x66\x87\xee\x32\x9a\x3d\xf8\xc7\x7c\x76\x35\xed\xeb\xf7\x07\x5b\x69\x67\xd8\x2a\x65\x0a\x7f\x91\x98\x90\xdb\x86\xba\x38\x62\x42\x81\x5c\x33\x93\xd3\xcf\x8d\xf4\x76\x4f\x6c\x3c\xcf\x69\xbe\xeb\xec\x71\x77\xf6\x7e\x7e\x39\x9a\xcc\x87\xa3\xe1\x7c\x70\x35\x9d\xf5\x7b\xf3\xe1\xa8\xd7\x9f\xe6\x6f\xee\x6d\x42\x74\x2e\xa7\x9f\xf3\x93\xdb\x9d\xdd\x60\x74\xd1\x1d\xcc\xa7\xb3\xd1\xa4\xfb\xae\x1f\x51\xee\x4e\xe2\x9f\x50\x78\xee\x9a\x77\xfa\xbe\x8b\xfb\xcd\x87\xbf\x8f\x26\xd8\xff\xff\xaf\xb3\x10\xaa\xe3\xd6\x89\x4b\xf4\x40\xb0\x4e\xab\xeb\xa6\x14\x16\x48\x05\xa7\xe7\xe7\xe7\x40\x0c\xe4\x3f\xdd\x7e\x1c\x0d\xe6\xbd\xab\xc9\x5d\x62\x9e\xaf\x2b\x5d\xc2\xf9\xe9\x69\xfb\xa8\x53\x14\x79\x5c\x83\xcc\x96\xfa\x9b\x3a\xc2\x91\xad\x80\xd8\xe5\x43\xf8\x35\x4a\x83\x76\xac\xcb\x62\xcb\x2a\xb9\x87\x79\x40\x62\x10\x25\x9e\xc7\xba\x7c\x72\xe3\x26\x6a\x13\x1a\x31\x8d\x52\x7b\xad\xfe\x78\x44\x3f\x30\x82\xd7\x8d\xe5\x4a\x58\xab\x2d\x96\x44\x8a\x85\x65\x76\x4b\x16\xb5\xdb\x2e\xf4\x77\x7a\x56\xfc\xfa\xb6\x38\x3b\x76\x2e\xff\x1d\x00\x00\xff\xff\x33\x50\x2d\x30\x1a\x0d\x00\x00") func localStorageYamlBytes() ([]byte, error) { return bindataRead( From fd5d1a9c022fcd2a2a3a045033388cf3cda3ca36 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 29 Apr 2024 19:39:35 +0000 Subject: [PATCH 09/31] Add support for svclb pod PriorityClassName Signed-off-by: Brad Davidson (cherry picked from commit 37f97b33c9e4d12ff65ad290f98abd7ca98648d6) Signed-off-by: Brad Davidson --- pkg/cloudprovider/cloudprovider.go | 20 +++++++++++--------- pkg/cloudprovider/servicelb.go | 19 +++++++++++++++++-- pkg/daemons/control/deps/deps.go | 11 ++++++----- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 20927921cad2..8d03ebfa3633 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -28,11 +28,12 @@ import ( // Config describes externally-configurable cloud provider configuration. // This is normally unmarshalled from a JSON config file. type Config struct { - LBEnabled bool `json:"lbEnabled"` - LBImage string `json:"lbImage"` - LBNamespace string `json:"lbNamespace"` - NodeEnabled bool `json:"nodeEnabled"` - Rootless bool `json:"rootless"` + LBDefaultPriorityClassName string `json:"lbDefaultPriorityClassName"` + LBEnabled bool `json:"lbEnabled"` + LBImage string `json:"lbImage"` + LBNamespace string `json:"lbNamespace"` + NodeEnabled bool `json:"nodeEnabled"` + Rootless bool `json:"rootless"` } type k3s struct { @@ -56,10 +57,11 @@ func init() { var err error k := k3s{ Config: Config{ - LBEnabled: true, - LBImage: DefaultLBImage, - LBNamespace: DefaultLBNS, - NodeEnabled: true, + LBDefaultPriorityClassName: DefaultLBPriorityClassName, + LBEnabled: true, + LBImage: DefaultLBImage, + LBNamespace: DefaultLBNS, + NodeEnabled: true, }, } diff --git a/pkg/cloudprovider/servicelb.go b/pkg/cloudprovider/servicelb.go index 20d699b42786..79d34159a2a3 100644 --- a/pkg/cloudprovider/servicelb.go +++ b/pkg/cloudprovider/servicelb.go @@ -41,12 +41,14 @@ var ( daemonsetNodeLabel = "svccontroller." + version.Program + ".cattle.io/enablelb" daemonsetNodePoolLabel = "svccontroller." + version.Program + ".cattle.io/lbpool" nodeSelectorLabel = "svccontroller." + version.Program + ".cattle.io/nodeselector" + priorityAnnotation = "svccontroller." + version.Program + ".cattle.io/priorityclassname" controllerName = ccmapp.DefaultInitFuncConstructors["service"].InitContext.ClientName ) const ( - Ready = condition.Cond("Ready") - DefaultLBNS = meta.NamespaceSystem + Ready = condition.Cond("Ready") + DefaultLBNS = meta.NamespaceSystem + DefaultLBPriorityClassName = "system-node-critical" ) var ( @@ -428,6 +430,7 @@ func (k *k3s) deleteDaemonSet(ctx context.Context, svc *core.Service) error { func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { name := generateName(svc) oneInt := intstr.FromInt(1) + priorityClassName := k.getPriorityClassName(svc) localTraffic := servicehelper.RequestsOnlyLocalTraffic(svc) sourceRangesSet, err := servicehelper.GetLoadBalancerSourceRanges(svc) if err != nil { @@ -472,6 +475,7 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) { }, }, Spec: core.PodSpec{ + PriorityClassName: priorityClassName, ServiceAccountName: "svclb", AutomountServiceAccountToken: utilsptr.To(false), SecurityContext: &core.PodSecurityContext{ @@ -682,6 +686,17 @@ func (k *k3s) removeFinalizer(ctx context.Context, svc *core.Service) (*core.Ser return svc, nil } +// getPriorityClassName returns the value of the priority class name annotation on the service, +// or the system default priority class name. +func (k *k3s) getPriorityClassName(svc *core.Service) string { + if svc != nil { + if v, ok := svc.Annotations[priorityAnnotation]; ok { + return v + } + } + return k.LBDefaultPriorityClassName +} + // generateName generates a distinct name for the DaemonSet based on the service name and UID func generateName(svc *core.Service) string { return fmt.Sprintf("svclb-%s-%s", svc.Name, svc.UID[:8]) diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index fd370130c9ae..1e8eb6de0890 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -829,11 +829,12 @@ func genEgressSelectorConfig(controlConfig *config.Control) error { func genCloudConfig(controlConfig *config.Control) error { cloudConfig := cloudprovider.Config{ - LBEnabled: !controlConfig.DisableServiceLB, - LBNamespace: controlConfig.ServiceLBNamespace, - LBImage: cloudprovider.DefaultLBImage, - Rootless: controlConfig.Rootless, - NodeEnabled: !controlConfig.DisableCCM, + LBDefaultPriorityClassName: cloudprovider.DefaultLBPriorityClassName, + LBEnabled: !controlConfig.DisableServiceLB, + LBNamespace: controlConfig.ServiceLBNamespace, + LBImage: cloudprovider.DefaultLBImage, + Rootless: controlConfig.Rootless, + NodeEnabled: !controlConfig.DisableCCM, } if controlConfig.SystemDefaultRegistry != "" { cloudConfig.LBImage = controlConfig.SystemDefaultRegistry + "/" + cloudConfig.LBImage From 49b4e3f0562ba2596d93acc907ed4ad34bcc8128 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 9 May 2024 18:36:33 +0000 Subject: [PATCH 10/31] bump minio-go to v7.0.70 Signed-off-by: Brad Davidson (cherry picked from commit afdcc83afebdb8144ad4fa0a81a73c36b300bd1c) Signed-off-by: Brad Davidson --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 670a43a6d56c..d12d2a7b2d7b 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/lib/pq v1.10.2 github.com/libp2p/go-libp2p v0.33.2 github.com/mattn/go-sqlite3 v1.14.19 - github.com/minio/minio-go/v7 v7.0.33 + github.com/minio/minio-go/v7 v7.0.70 github.com/mwitkow/go-http-dialer v0.0.0-20161116154839-378f744fb2b8 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/onsi/ginkgo/v2 v2.16.0 diff --git a/go.sum b/go.sum index 54c42c25995b..5c94004c092b 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.33 h1:jLEHTp9jg2zWBa5w9W1i8WXq6o+oGRcjsdk9HbFgdlc= -github.com/minio/minio-go/v7 v7.0.33/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= +github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= +github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= From 404ad9854cc5aedc0338a223b25ab1f756fa4371 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 9 May 2024 18:51:53 +0000 Subject: [PATCH 11/31] Bump kine to v0.11.9 to fix pagination Signed-off-by: Brad Davidson (cherry picked from commit 2669d67a9b3a913565bf172ffa54301670f64f3f) Signed-off-by: Brad Davidson --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d12d2a7b2d7b..18549fb3e3e4 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/json-iterator/go v1.1.12 github.com/k3s-io/helm-controller v0.15.9 - github.com/k3s-io/kine v0.11.7 + github.com/k3s-io/kine v0.11.9 github.com/klauspost/compress v1.17.7 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 github.com/lib/pq v1.10.2 @@ -259,6 +259,7 @@ require ( github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/swag v0.22.9 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect diff --git a/go.sum b/go.sum index 5c94004c092b..3de6835c279c 100644 --- a/go.sum +++ b/go.sum @@ -640,6 +640,8 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -950,8 +952,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1 h1:Pqcxkg7V60c26ZpHoekP9QoUdLuduxF github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1/go.mod h1:K/8nbsGupHqmr5MkgaZpLlH1QdX1pcNQLAkODy44XcQ= github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64= github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= -github.com/k3s-io/kine v0.11.7 h1:+I4TrxozQv4cdmD8RULI35r4o5G+A7gOD3F75lfjDP0= -github.com/k3s-io/kine v0.11.7/go.mod h1:4C/zNVwl3FU1EubA2ju1Hq36JIjp8gAZaM+Hfnuvqt4= +github.com/k3s-io/kine v0.11.9 h1:7HfWSwtOowb7GuV6nECnNlFKShgRgVBLdWXj0/4t0sE= +github.com/k3s-io/kine v0.11.9/go.mod h1:N8rc1GDmEvvYRuTxhKTZfSc4fm/vyI6GbDxwBjccAjs= github.com/k3s-io/klog/v2 v2.100.1-k3s1 h1:xb/Ta8dpQuIZueQEw2YTZUYrKoILdBmPiITVkNmYPa0= github.com/k3s-io/klog/v2 v2.100.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= github.com/k3s-io/kube-router/v2 v2.1.2 h1:/eLfIsELLsqqRW1skIJ2qe7bWL6IZZ9Hg3IniIgObXo= From df744fe90f95445cc67a317c3ec8ebe512ef3af4 Mon Sep 17 00:00:00 2001 From: linxin Date: Mon, 15 Apr 2024 11:38:42 +0800 Subject: [PATCH 12/31] Validate resolv.conf for presence of nameserver entries Co-authored-by: Brad Davidson Signed-off-by: linxin (cherry picked from commit f24ba9d3a9d7965fee4e9a9c44541356c14dc6a1) Signed-off-by: Brad Davidson --- pkg/agent/config/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index b7a03ca7010c..f81c36beae34 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -316,19 +316,22 @@ func isValidResolvConf(resolvConfFile string) bool { nameserver := regexp.MustCompile(`^nameserver\s+([^\s]*)`) scanner := bufio.NewScanner(file) + foundNameserver := false for scanner.Scan() { ipMatch := nameserver.FindStringSubmatch(scanner.Text()) if len(ipMatch) == 2 { ip := net.ParseIP(ipMatch[1]) if ip == nil || !ip.IsGlobalUnicast() { return false + } else { + foundNameserver = true } } } if err := scanner.Err(); err != nil { return false } - return true + return foundNameserver } func locateOrGenerateResolvConf(envInfo *cmds.Agent) string { From 4d34f358961d765028fb4b78955961d58f6a4102 Mon Sep 17 00:00:00 2001 From: zouxianyu <2979121738@qq.com> Date: Wed, 15 May 2024 14:56:30 +0800 Subject: [PATCH 13/31] add missing kernel config check Signed-off-by: zouxianyu <2979121738@qq.com> (cherry picked from commit c1cb5d63b962f47a60eaef25cbd41a4ab900fafc) Signed-off-by: Brad Davidson --- contrib/util/check-config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/util/check-config.sh b/contrib/util/check-config.sh index 0c5388cc0da5..2283e5de5868 100755 --- a/contrib/util/check-config.sh +++ b/contrib/util/check-config.sh @@ -388,7 +388,7 @@ flags=" CGROUPS CGROUP_PIDS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG KEYS VETH BRIDGE BRIDGE_NETFILTER - IP_NF_FILTER IP_NF_TARGET_MASQUERADE + IP_NF_FILTER IP_NF_TARGET_MASQUERADE IP_NF_TARGET_REJECT NETFILTER_XT_MATCH_ADDRTYPE NETFILTER_XT_MATCH_CONNTRACK NETFILTER_XT_MATCH_IPVS NETFILTER_XT_MATCH_COMMENT NETFILTER_XT_MATCH_MULTIPORT IP_NF_NAT NF_NAT POSIX_MQUEUE From a250ebaddd1e74b1c5f0fd7a2d937512474ab504 Mon Sep 17 00:00:00 2001 From: Robert Rose Date: Tue, 30 Apr 2024 17:41:07 +0200 Subject: [PATCH 14/31] Follow directory symlinks in auto deploying manifests (#9288) Signed-off-by: Robert Rose (cherry picked from commit 6886c0977f9d9305d983669d524b967b1dd224be) Signed-off-by: Brad Davidson --- pkg/deploy/controller.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/deploy/controller.go b/pkg/deploy/controller.go index d26a88906aee..e4626134cb78 100644 --- a/pkg/deploy/controller.go +++ b/pkg/deploy/controller.go @@ -119,6 +119,26 @@ func (w *watcher) listFilesIn(base string, force bool) error { if err != nil { return err } + // Descend into symlinked directories, however, only top-level links are followed + if info.Mode()&os.ModeSymlink != 0 { + linkInfo, err := os.Stat(path) + if err != nil { + return err + } + if linkInfo.IsDir() { + evalPath, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + filepath.Walk(evalPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + files[path] = info + return nil + }) + } + } files[path] = info return nil }); err != nil { From 08cf09ead423fcb4205ffb5a0e3adc91d74044de Mon Sep 17 00:00:00 2001 From: huangzy Date: Tue, 30 Apr 2024 17:21:46 +0800 Subject: [PATCH 15/31] allow helm controller set owner reference Signed-off-by: huangzy (cherry picked from commit 6fcaad553da898e5b5dc2d66e64d23b37d602691) Signed-off-by: Brad Davidson --- pkg/server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 925830a9575a..0ccaa49f9163 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -220,7 +220,7 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error { return err } - apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup() + apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup().WithSetOwnerReference(false, false) helm := sc.Helm.WithAgent(restConfig.UserAgent) batch := sc.Batch.WithAgent(restConfig.UserAgent) auth := sc.Auth.WithAgent(restConfig.UserAgent) From 74e56b5d3a8986f0737316a57488061de1763e0d Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 24 May 2024 17:49:06 +0000 Subject: [PATCH 16/31] Bump klipper-helm image for tls secret support Signed-off-by: Brad Davidson (cherry picked from commit 6683fcdb65609f0df041b10707fdafc16a3f90e3) Signed-off-by: Brad Davidson --- go.mod | 2 +- go.sum | 4 ++-- scripts/airgap/image-list.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 18549fb3e3e4..82ebeb55a91d 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/joho/godotenv v1.5.1 github.com/json-iterator/go v1.1.12 - github.com/k3s-io/helm-controller v0.15.9 + github.com/k3s-io/helm-controller v0.15.10 github.com/k3s-io/kine v0.11.9 github.com/klauspost/compress v1.17.7 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index 3de6835c279c..67d2ff373f5f 100644 --- a/go.sum +++ b/go.sum @@ -950,8 +950,8 @@ github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1 h1:yexUwAPPdmYfIMWOj6sSyJ2nEe8QOrFzN github.com/k3s-io/etcd/raft/v3 v3.5.13-k3s1/go.mod h1:uUFibGLn2Ksm2URMxN1fICGhk8Wu96EfDQyuLhAcAmw= github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1 h1:Pqcxkg7V60c26ZpHoekP9QoUdLuduxFn827A/5CIwm4= github.com/k3s-io/etcd/server/v3 v3.5.13-k3s1/go.mod h1:K/8nbsGupHqmr5MkgaZpLlH1QdX1pcNQLAkODy44XcQ= -github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64= -github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= +github.com/k3s-io/helm-controller v0.15.10 h1:TIfbbCbv8mJ1AquPzSxH3vMqIcqfgZ9Pr/Pq/jka/zc= +github.com/k3s-io/helm-controller v0.15.10/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= github.com/k3s-io/kine v0.11.9 h1:7HfWSwtOowb7GuV6nECnNlFKShgRgVBLdWXj0/4t0sE= github.com/k3s-io/kine v0.11.9/go.mod h1:N8rc1GDmEvvYRuTxhKTZfSc4fm/vyI6GbDxwBjccAjs= github.com/k3s-io/klog/v2 v2.100.1-k3s1 h1:xb/Ta8dpQuIZueQEw2YTZUYrKoILdBmPiITVkNmYPa0= diff --git a/scripts/airgap/image-list.txt b/scripts/airgap/image-list.txt index dc5e8c68522a..932f47ab2fa3 100644 --- a/scripts/airgap/image-list.txt +++ b/scripts/airgap/image-list.txt @@ -1,4 +1,4 @@ -docker.io/rancher/klipper-helm:v0.8.3-build20240228 +docker.io/rancher/klipper-helm:v0.8.4-build20240523 docker.io/rancher/klipper-lb:v0.4.7 docker.io/rancher/local-path-provisioner:v0.0.26 docker.io/rancher/mirrored-coredns-coredns:1.10.1 From df5531eb656fc3287ae28931e488119c0ab7e5a1 Mon Sep 17 00:00:00 2001 From: Anuj Garg Date: Mon, 22 Apr 2024 11:43:35 +0530 Subject: [PATCH 17/31] Updating the script binary_size_check to complete the command name by adding .exe extension to the k3s binary name to make it available to run stat command Signed-off-by: Anuj Garg (cherry picked from commit eb192197eb69e9be223c39ec83f59dee5ceadd3e) Signed-off-by: Brad Davidson --- scripts/binary_size_check.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/binary_size_check.sh b/scripts/binary_size_check.sh index 9ba138e3f4c8..f019ec53e1ed 100755 --- a/scripts/binary_size_check.sh +++ b/scripts/binary_size_check.sh @@ -2,6 +2,8 @@ set -e +. ./scripts/version.sh + GO=${GO-go} ARCH=${ARCH:-$("${GO}" env GOARCH)} @@ -22,7 +24,7 @@ elif [ ${ARCH} = s390x ]; then BIN_SUFFIX="-s390x" fi -CMD_NAME="dist/artifacts/k3s${BIN_SUFFIX}" +CMD_NAME="dist/artifacts/k3s${BIN_SUFFIX}${BINARY_POSTFIX}" SIZE=$(stat -c '%s' ${CMD_NAME}) if [ -n "${DEBUG}" ]; then From 38ede61b0d6970f4bbea95ee1df927ea9a4a1aa3 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 29 Apr 2024 23:29:49 +0000 Subject: [PATCH 18/31] Fix issue with k3s-etcd informers not starting Start shared informer caches when k3s-etcd controller wins leader election. Previously, these were only started when the main k3s apiserver controller won an election. If the leaders ended up going to different nodes, some informers wouldn't be started Signed-off-by: Brad Davidson (cherry picked from commit 3d14092f76f27f4978968597174707a5cb2a80e7) Signed-off-by: Brad Davidson --- pkg/cluster/managed.go | 4 +++- pkg/etcd/etcd.go | 7 +++++++ pkg/server/server.go | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/managed.go b/pkg/cluster/managed.go index d6c668998a8a..b0e6f71861dc 100644 --- a/pkg/cluster/managed.go +++ b/pkg/cluster/managed.go @@ -91,7 +91,9 @@ func (c *Cluster) start(ctx context.Context) error { return c.managedDB.Start(ctx, c.clientAccessInfo) } -// registerDBHandlers registers routes for database info with the http request handler +// registerDBHandlers registers managed-datastore-specific callbacks, and installs additional HTTP route handlers. +// Note that for etcd, controllers only run on nodes with a local apiserver, in order to provide stable external +// management of etcd cluster membership without being disrupted when a member is removed from the cluster. func (c *Cluster) registerDBHandlers(handler http.Handler) (http.Handler, error) { if c.managedDB == nil { return handler, nil diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index abf142a908f0..e923e105196e 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -34,6 +34,7 @@ import ( "github.com/pkg/errors" certutil "github.com/rancher/dynamiclistener/cert" controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" + "github.com/rancher/wrangler/pkg/start" "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/etcdserverpb" @@ -619,6 +620,12 @@ func (e *ETCD) Register(handler http.Handler) (http.Handler, error) { registerEndpointsHandlers(ctx, e) registerMemberHandlers(ctx, e) registerSnapshotHandlers(ctx, e) + + // Re-run informer factory startup after core and leader-elected controllers have started. + // Additional caches may need to start for the newly added OnChange/OnRemove callbacks. + if err := start.All(ctx, 5, e.config.Runtime.K3s, e.config.Runtime.Core); err != nil { + panic(errors.Wrap(err, "failed to start wrangler controllers")) + } } } diff --git a/pkg/server/server.go b/pkg/server/server.go index 0ccaa49f9163..a13d2caf051b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -167,8 +167,8 @@ func apiserverControllers(ctx context.Context, sc *Context, config *Config) { } } - // Re-run context startup after core and leader-elected controllers have started. Additional - // informer caches may need to start for the newly added OnChange callbacks. + // Re-run informer factory startup after core and leader-elected controllers have started. + // Additional caches may need to start for the newly added OnChange/OnRemove callbacks. if err := sc.Start(ctx); err != nil { panic(errors.Wrap(err, "failed to start wranger controllers")) } From c309eb004ce0942d7a6185e399f2feca1707296b Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Fri, 3 May 2024 03:22:01 +0300 Subject: [PATCH 19/31] Add proctitle package with linux and windows constraints Signed-off-by: galal-hussein (cherry picked from commit 48ff3bcddb8f5a95bbd81d9629702d7d8b42eb85) Signed-off-by: Brad Davidson --- main.go | 1 + pkg/cli/agent/agent.go | 4 ++-- pkg/cli/cert/cert.go | 4 ++-- pkg/cli/cmds/log_linux.go | 4 ++-- pkg/cli/etcdsnapshot/etcd_snapshot.go | 4 ++-- pkg/cli/secretsencrypt/secrets_encrypt.go | 4 ++-- pkg/cli/server/server.go | 4 ++-- pkg/cli/token/token.go | 4 ++-- pkg/proctitle/proctile.go | 14 ++++++++++++++ pkg/proctitle/proctile_windows.go | 6 ++++++ 10 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 pkg/proctitle/proctile.go create mode 100644 pkg/proctitle/proctile_windows.go diff --git a/main.go b/main.go index 8857094b87a2..7859d13a4dfb 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,7 @@ func main() { secretsencrypt.RotateKeys, ), cmds.NewCertCommands( + cert.Check, cert.Rotate, cert.RotateCA, ), diff --git a/pkg/cli/agent/agent.go b/pkg/cli/agent/agent.go index 694fa9aa6e3e..415737840c29 100644 --- a/pkg/cli/agent/agent.go +++ b/pkg/cli/agent/agent.go @@ -9,12 +9,12 @@ import ( "path/filepath" "runtime" - "github.com/erikdubbelboer/gspt" "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/agent" "github.com/k3s-io/k3s/pkg/authenticator" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/datadir" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/spegel" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" @@ -31,7 +31,7 @@ func Run(ctx *cli.Context) error { // hide process arguments from ps output, since they may contain // database credentials or other secrets. - gspt.SetProcTitle(os.Args[0] + " agent") + proctitle.SetProcTitle(os.Args[0] + " agent") // Evacuate cgroup v2 before doing anything else that may fork. if err := cmds.EvacuateCgroup2(); err != nil { diff --git a/pkg/cli/cert/cert.go b/pkg/cli/cert/cert.go index 72848fdd2aa5..d5b0b96b8ae5 100644 --- a/pkg/cli/cert/cert.go +++ b/pkg/cli/cert/cert.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/erikdubbelboer/gspt" "github.com/k3s-io/k3s/pkg/agent/util" "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/cli/cmds" @@ -16,6 +15,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/control/deps" "github.com/k3s-io/k3s/pkg/datadir" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/util/services" "github.com/k3s-io/k3s/pkg/version" @@ -27,7 +27,7 @@ import ( ) func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, error) { - gspt.SetProcTitle(os.Args[0]) + proctitle.SetProcTitle(os.Args[0]) dataDir, err := datadir.Resolve(cfg.DataDir) if err != nil { diff --git a/pkg/cli/cmds/log_linux.go b/pkg/cli/cmds/log_linux.go index 5b836d9740d3..fa227ca4161c 100644 --- a/pkg/cli/cmds/log_linux.go +++ b/pkg/cli/cmds/log_linux.go @@ -11,7 +11,7 @@ import ( "syscall" systemd "github.com/coreos/go-systemd/v22/daemon" - "github.com/erikdubbelboer/gspt" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/version" "github.com/natefinch/lumberjack" "github.com/pkg/errors" @@ -42,7 +42,7 @@ func forkIfLoggingOrReaping() error { } if enableLogRedirect || enableReaping { - gspt.SetProcTitle(os.Args[0] + " init") + proctitle.SetProcTitle(os.Args[0] + " init") pwd, err := os.Getwd() if err != nil { diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index e4e880243c26..b6e774affec8 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -12,12 +12,12 @@ import ( "text/tabwriter" "time" - "github.com/erikdubbelboer/gspt" k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/cluster/managed" "github.com/k3s-io/k3s/pkg/etcd" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/server" util2 "github.com/k3s-io/k3s/pkg/util" "github.com/pkg/errors" @@ -34,7 +34,7 @@ var timeout = 2 * time.Minute func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *clientaccess.Info, error) { // hide process arguments from ps output, since they may contain // database credentials or other secrets. - gspt.SetProcTitle(os.Args[0] + " etcd-snapshot") + proctitle.SetProcTitle(os.Args[0] + " etcd-snapshot") sr := &etcd.SnapshotRequest{} // Operation and name are set by the command handler. diff --git a/pkg/cli/secretsencrypt/secrets_encrypt.go b/pkg/cli/secretsencrypt/secrets_encrypt.go index b0bd8525013b..b0c6256e2877 100644 --- a/pkg/cli/secretsencrypt/secrets_encrypt.go +++ b/pkg/cli/secretsencrypt/secrets_encrypt.go @@ -10,9 +10,9 @@ import ( "text/tabwriter" "time" - "github.com/erikdubbelboer/gspt" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/secretsencrypt" "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/version" @@ -24,7 +24,7 @@ import ( func commandPrep(cfg *cmds.Server) (*clientaccess.Info, error) { // hide process arguments from ps output, since they may contain // database credentials or other secrets. - gspt.SetProcTitle(os.Args[0] + " secrets-encrypt") + proctitle.SetProcTitle(os.Args[0] + " secrets-encrypt") dataDir, err := server.ResolveDataDir(cfg.DataDir) if err != nil { diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index a17c38e21e18..21417d178dd8 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -10,7 +10,6 @@ import ( "time" systemd "github.com/coreos/go-systemd/v22/daemon" - "github.com/erikdubbelboer/gspt" "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/agent" "github.com/k3s-io/k3s/pkg/agent/loadbalancer" @@ -19,6 +18,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/datadir" "github.com/k3s-io/k3s/pkg/etcd" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/rootless" "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/spegel" @@ -55,7 +55,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont // hide process arguments from ps output, since they may contain // database credentials or other secrets. - gspt.SetProcTitle(os.Args[0] + " server") + proctitle.SetProcTitle(os.Args[0] + " server") // If the agent is enabled, evacuate cgroup v2 before doing anything else that may fork. // If the agent is disabled, we don't need to bother doing this as it is only the kubelet diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index 7af7cee82cac..e16038fea5b6 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -11,10 +11,10 @@ import ( "text/tabwriter" "time" - "github.com/erikdubbelboer/gspt" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/kubeadm" + "github.com/k3s-io/k3s/pkg/proctitle" "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" @@ -171,7 +171,7 @@ func Rotate(app *cli.Context) error { func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) { // hide process arguments from ps output, since they likely contain tokens. - gspt.SetProcTitle(os.Args[0] + " token") + proctitle.SetProcTitle(os.Args[0] + " token") dataDir, err := server.ResolveDataDir("") if err != nil { diff --git a/pkg/proctitle/proctile.go b/pkg/proctitle/proctile.go new file mode 100644 index 000000000000..16bed1cab39d --- /dev/null +++ b/pkg/proctitle/proctile.go @@ -0,0 +1,14 @@ +//go:build linux +// +build linux + +package proctitle + +import ( + "os" + + "github.com/erikdubbelboer/gspt" +) + +func SetProcTitle(cmd string) { + gspt.SetProcTitle(os.Args[0] + " agent") +} diff --git a/pkg/proctitle/proctile_windows.go b/pkg/proctitle/proctile_windows.go new file mode 100644 index 000000000000..9ade88241c7c --- /dev/null +++ b/pkg/proctitle/proctile_windows.go @@ -0,0 +1,6 @@ +//go:build windows +// +build windows + +package proctitle + +func SetProcTitle(cmd string) {} From 940bc667b5eb16c8ab894d82ea700c2182fb19a6 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 25 Apr 2024 01:02:05 +0000 Subject: [PATCH 20/31] Refactor supervisor listener startup and add metrics * Refactor agent supervisor listener startup and authn/authz to use upstream auth delegators to perform for SubjectAccessReview for access to metrics. * Convert spegel and pprof handlers over to new structure. * Promote bind-address to agent flag to allow setting supervisor bind address for both agent and server. * Promote enable-pprof to agent flag to allow profiling agents. Access to the pprof endpoint now requires client cert auth, similar to the spegel registry api endpoint. * Add prometheus metrics handler. Signed-off-by: Brad Davidson (cherry picked from commit ff679fb3abd5b06a82b62004de8703e0f4d6d664) Signed-off-by: Brad Davidson --- pkg/agent/config/config.go | 19 ++++-- pkg/agent/https/https.go | 106 +++++++++++++++++++++++++++++++ pkg/agent/netpol/netpol.go | 16 ++--- pkg/agent/run.go | 14 +++++ pkg/certmonitor/certmonitor.go | 12 +--- pkg/cli/agent/agent.go | 50 ++++++--------- pkg/cli/cmds/agent.go | 14 +++++ pkg/cli/cmds/server.go | 22 +++---- pkg/cli/server/server.go | 52 +++++++++------ pkg/cluster/https.go | 23 +------ pkg/daemons/config/types.go | 12 +++- pkg/metrics/metrics.go | 45 +++++++++++++ pkg/profile/profile.go | 38 +++++++++++ pkg/spegel/spegel.go | 40 +++--------- pkg/util/net.go | 111 +++++++++++++++++++++++++++++++++ pkg/util/net_unix.go | 18 ++++++ pkg/util/net_windows.go | 11 ++++ 17 files changed, 462 insertions(+), 141 deletions(-) create mode 100644 pkg/agent/https/https.go create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/profile/profile.go create mode 100644 pkg/util/net_unix.go create mode 100644 pkg/util/net_windows.go diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index f81c36beae34..1de1bb5ec222 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -524,12 +524,14 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N SELinux: envInfo.EnableSELinux, ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint, ImageServiceEndpoint: envInfo.ImageServiceEndpoint, + EnablePProf: envInfo.EnablePProf, EmbeddedRegistry: controlConfig.EmbeddedRegistry, FlannelBackend: controlConfig.FlannelBackend, FlannelIPv6Masq: controlConfig.FlannelIPv6Masq, FlannelExternalIP: controlConfig.FlannelExternalIP, EgressSelectorMode: controlConfig.EgressSelectorMode, ServerHTTPSPort: controlConfig.HTTPSPort, + SupervisorMetrics: controlConfig.SupervisorMetrics, Token: info.String(), } nodeConfig.FlannelIface = flannelIface @@ -592,13 +594,18 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl") nodeConfig.Certificate = servingCert - nodeConfig.AgentConfig.NodeIPs = nodeIPs - listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0]) - if err != nil { - return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip") + if envInfo.BindAddress != "" { + nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress + } else { + listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0]) + if err != nil { + return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip") + } + nodeConfig.AgentConfig.ListenAddress = listenAddress } + nodeConfig.AgentConfig.NodeIP = nodeIPs[0].String() - nodeConfig.AgentConfig.ListenAddress = listenAddress + nodeConfig.AgentConfig.NodeIPs = nodeIPs nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs // if configured, set NodeExternalIP to the first IPv4 address, for legacy clients @@ -689,6 +696,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.ImageCredProvConfig = envInfo.ImageCredProvConfig nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM nodeConfig.AgentConfig.DisableNPC = controlConfig.DisableNPC + nodeConfig.AgentConfig.MinTLSVersion = controlConfig.MinTLSVersion + nodeConfig.AgentConfig.CipherSuites = controlConfig.CipherSuites nodeConfig.AgentConfig.Rootless = envInfo.Rootless nodeConfig.AgentConfig.PodManifests = filepath.Join(envInfo.DataDir, "agent", DefaultPodManifestPath) nodeConfig.AgentConfig.ProtectKernelDefaults = envInfo.ProtectKernelDefaults diff --git a/pkg/agent/https/https.go b/pkg/agent/https/https.go new file mode 100644 index 000000000000..2b5927107f9b --- /dev/null +++ b/pkg/agent/https/https.go @@ -0,0 +1,106 @@ +package https + +import ( + "context" + "net/http" + "strconv" + "sync" + + "github.com/gorilla/mux" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" + "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authorization/authorizer" + genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" + apirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/options" +) + +// RouterFunc provides a hook for components to register additional routes to a request router +type RouterFunc func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) + +var once sync.Once +var router *mux.Router +var err error + +// Start returns a router with authn/authz filters applied. +// The first time it is called, the router is created and a new HTTPS listener is started if the handler is nil. +// Subsequent calls will return the same router. +func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.ControlRuntime) (*mux.Router, error) { + once.Do(func() { + router = mux.NewRouter().SkipClean(true) + config := server.Config{} + + if runtime == nil { + // If we do not have an existing handler, set up a new listener + tcp, lerr := util.ListenWithLoopback(ctx, nodeConfig.AgentConfig.ListenAddress, strconv.Itoa(nodeConfig.ServerHTTPSPort)) + if lerr != nil { + err = lerr + return + } + + serving := options.NewSecureServingOptions() + serving.Listener = tcp + serving.CipherSuites = nodeConfig.AgentConfig.CipherSuites + serving.MinTLSVersion = nodeConfig.AgentConfig.MinTLSVersion + serving.ServerCert = options.GeneratableKeyCert{ + CertKey: options.CertKey{ + CertFile: nodeConfig.AgentConfig.ServingKubeletCert, + KeyFile: nodeConfig.AgentConfig.ServingKubeletKey, + }, + } + if aerr := serving.ApplyTo(&config.SecureServing); aerr != nil { + err = aerr + return + } + } else { + // If we have an existing handler, wrap it + router.NotFoundHandler = runtime.Handler + runtime.Handler = router + } + + authn := options.NewDelegatingAuthenticationOptions() + authn.DisableAnonymous = true + authn.SkipInClusterLookup = true + authn.ClientCert = options.ClientCertAuthenticationOptions{ + ClientCA: nodeConfig.AgentConfig.ClientCA, + } + authn.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet + if applyErr := authn.ApplyTo(&config.Authentication, config.SecureServing, nil); applyErr != nil { + err = applyErr + return + } + + authz := options.NewDelegatingAuthorizationOptions() + authz.AlwaysAllowPaths = []string{"/v2", "/debug/pprof", "/v1-" + version.Program + "/p2p"} + authz.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet + if applyErr := authz.ApplyTo(&config.Authorization); applyErr != nil { + err = applyErr + return + } + + router.Use(filterChain(config.Authentication.Authenticator, config.Authorization.Authorizer)) + + if config.SecureServing != nil { + _, _, err = config.SecureServing.Serve(router, 0, ctx.Done()) + } + }) + + return router, err +} + +// filterChain runs the kubernetes authn/authz filter chain using the mux middleware API +func filterChain(authn authenticator.Request, authz authorizer.Authorizer) mux.MiddlewareFunc { + return func(handler http.Handler) http.Handler { + requestInfoResolver := &apirequest.RequestInfoFactory{} + failedHandler := genericapifilters.Unauthorized(scheme.Codecs) + handler = genericapifilters.WithAuthorization(handler, authz, scheme.Codecs) + handler = genericapifilters.WithAuthentication(handler, authn, failedHandler, nil, nil) + handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) + handler = genericapifilters.WithCacheControl(handler) + return handler + } +} diff --git a/pkg/agent/netpol/netpol.go b/pkg/agent/netpol/netpol.go index a8bb760bcfc1..60d2c1f07f45 100644 --- a/pkg/agent/netpol/netpol.go +++ b/pkg/agent/netpol/netpol.go @@ -19,25 +19,25 @@ import ( "github.com/cloudnativelabs/kube-router/v2/pkg/controllers/netpol" "github.com/cloudnativelabs/kube-router/v2/pkg/healthcheck" - "github.com/cloudnativelabs/kube-router/v2/pkg/metrics" + krmetrics "github.com/cloudnativelabs/kube-router/v2/pkg/metrics" "github.com/cloudnativelabs/kube-router/v2/pkg/options" "github.com/cloudnativelabs/kube-router/v2/pkg/utils" "github.com/cloudnativelabs/kube-router/v2/pkg/version" "github.com/coreos/go-iptables/iptables" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/metrics" "github.com/pkg/errors" "github.com/sirupsen/logrus" v1core "k8s.io/api/core/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - "k8s.io/component-base/metrics/legacyregistry" ) func init() { // ensure that kube-router exposes metrics through the same registry used by Kubernetes components - metrics.DefaultRegisterer = legacyregistry.Registerer() - metrics.DefaultGatherer = legacyregistry.DefaultGatherer + krmetrics.DefaultRegisterer = metrics.DefaultRegisterer + krmetrics.DefaultGatherer = metrics.DefaultGatherer } // Run creates and starts a new instance of the kube-router network policy controller @@ -156,7 +156,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { } // Start kube-router metrics controller to avoid complaints about metrics heartbeat missing - mc, err := metrics.NewMetricsController(krConfig) + mc, err := krmetrics.NewMetricsController(krConfig) if err != nil { return nil } @@ -188,13 +188,13 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { } // metricsRunCheck is a stub version of mc.Run() that doesn't start up a dedicated http server. -func metricsRunCheck(mc *metrics.Controller, healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) { +func metricsRunCheck(mc *krmetrics.Controller, healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) { t := time.NewTicker(3 * time.Second) defer wg.Done() // register metrics for this controller - metrics.BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1) - metrics.DefaultRegisterer.MustRegister(metrics.BuildInfo) + krmetrics.BuildInfo.WithLabelValues(runtime.Version(), version.Version).Set(1) + krmetrics.DefaultRegisterer.MustRegister(krmetrics.BuildInfo) for { healthcheck.SendHeartBeat(healthChan, "MC") diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 607cfc7bc4f2..f3342767ad29 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -27,7 +27,9 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/agent" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/executor" + "github.com/k3s-io/k3s/pkg/metrics" "github.com/k3s-io/k3s/pkg/nodeconfig" + "github.com/k3s-io/k3s/pkg/profile" "github.com/k3s-io/k3s/pkg/rootless" "github.com/k3s-io/k3s/pkg/spegel" "github.com/k3s-io/k3s/pkg/util" @@ -113,6 +115,18 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { } } + if nodeConfig.SupervisorMetrics { + if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil { + return errors.Wrap(err, "failed to serve metrics") + } + } + + if nodeConfig.EnablePProf { + if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil { + return errors.Wrap(err, "failed to serve pprof") + } + } + if err := setupCriCtlConfig(cfg, nodeConfig); err != nil { return err } diff --git a/pkg/certmonitor/certmonitor.go b/pkg/certmonitor/certmonitor.go index d2818007cba4..85b718f07895 100644 --- a/pkg/certmonitor/certmonitor.go +++ b/pkg/certmonitor/certmonitor.go @@ -11,6 +11,7 @@ import ( daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/control/deps" + "github.com/k3s-io/k3s/pkg/metrics" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/util/services" "github.com/k3s-io/k3s/pkg/version" @@ -22,18 +23,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/component-base/metrics/legacyregistry" ) var ( - // DefaultRegisterer and DefaultGatherer are the implementations of the - // prometheus Registerer and Gatherer interfaces that all metrics operations - // will use. They are variables so that packages that embed this library can - // replace them at runtime, instead of having to pass around specific - // registries. - DefaultRegisterer = legacyregistry.Registerer() - DefaultGatherer = legacyregistry.DefaultGatherer - // Check certificates twice an hour. Kubernetes events have a TTL of 1 hour by default, // so similar events should be aggregated and refreshed by the event recorder as long // as they are created within the TTL period. @@ -50,7 +42,7 @@ var ( // Setup starts the certificate expiration monitor func Setup(ctx context.Context, nodeConfig *daemonconfig.Node, dataDir string) error { logrus.Debugf("Starting %s with monitoring period %s", controllerName, certCheckInterval) - DefaultRegisterer.MustRegister(certificateExpirationSeconds) + metrics.DefaultRegisterer.MustRegister(certificateExpirationSeconds) client, err := util.GetClientSet(nodeConfig.AgentConfig.KubeConfigKubelet) if err != nil { diff --git a/pkg/cli/agent/agent.go b/pkg/cli/agent/agent.go index 415737840c29..7c206eb1dba5 100644 --- a/pkg/cli/agent/agent.go +++ b/pkg/cli/agent/agent.go @@ -1,20 +1,22 @@ package agent import ( + "context" "crypto/tls" - "errors" "fmt" - "net/http" "os" "path/filepath" "runtime" "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/agent" - "github.com/k3s-io/k3s/pkg/authenticator" + "github.com/k3s-io/k3s/pkg/agent/https" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/datadir" + k3smetrics "github.com/k3s-io/k3s/pkg/metrics" "github.com/k3s-io/k3s/pkg/proctitle" + "github.com/k3s-io/k3s/pkg/profile" "github.com/k3s-io/k3s/pkg/spegel" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" @@ -22,7 +24,6 @@ import ( "github.com/rancher/wrangler/pkg/signals" "github.com/sirupsen/logrus" "github.com/urfave/cli" - apiauth "k8s.io/apiserver/pkg/authentication/authenticator" ) func Run(ctx *cli.Context) error { @@ -108,33 +109,22 @@ func Run(ctx *cli.Context) error { // Until the agent is run and retrieves config from the server, we won't know // if the embedded registry is enabled. If it is not enabled, these are not // used as the registry is never started. - conf := spegel.DefaultRegistry - conf.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir) - conf.HandlerFunc = func(conf *spegel.Config, router *mux.Router) error { - // Create and bind a new authenticator using the configured client CA - authArgs := []string{"--client-ca-file=" + conf.ClientCAFile} - auth, err := authenticator.FromArgs(authArgs) - if err != nil { - return err - } - conf.AuthFunc = func() apiauth.Request { - return auth - } + registry := spegel.DefaultRegistry + registry.Bootstrapper = spegel.NewAgentBootstrapper(cfg.ServerURL, cfg.Token, cfg.DataDir) + registry.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) { + return https.Start(ctx, nodeConfig, nil) + } - // Create a new server and listen on the configured port - server := &http.Server{ - Handler: router, - Addr: ":" + conf.RegistryPort, - TLSConfig: &tls.Config{ - ClientAuth: tls.RequestClientCert, - }, - } - go func() { - if err := server.ListenAndServeTLS(conf.ServerCertFile, conf.ServerKeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) { - logrus.Fatalf("registry server failed: %v", err) - } - }() - return nil + // same deal for metrics - these are not used if the extra metrics listener is not enabled. + metrics := k3smetrics.DefaultMetrics + metrics.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) { + return https.Start(ctx, nodeConfig, nil) + } + + // and for pprof as well + pprof := profile.DefaultProfiler + pprof.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) { + return https.Start(ctx, nodeConfig, nil) } return agent.Run(contextCtx, cfg) diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 53b620f4d08b..16e0a196c106 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -20,6 +20,7 @@ type Agent struct { LBServerPort int ResolvConf string DataDir string + BindAddress string NodeIP cli.StringSlice NodeExternalIP cli.StringSlice NodeName string @@ -36,6 +37,7 @@ type Agent struct { VPNAuth string VPNAuthFile string Debug bool + EnablePProf bool Rootless bool RootlessAlreadyUnshared bool WithNodeID bool @@ -226,6 +228,16 @@ var ( Usage: "(agent/containerd) Disables containerd's fallback default registry endpoint when a mirror is configured for that registry", Destination: &AgentConfig.ContainerdNoDefault, } + EnablePProfFlag = &cli.BoolFlag{ + Name: "enable-pprof", + Usage: "(experimental) Enable pprof endpoint on supervisor port", + Destination: &AgentConfig.EnablePProf, + } + BindAddressFlag = &cli.StringFlag{ + Name: "bind-address", + Usage: "(listener) " + version.Program + " bind address (default: 0.0.0.0)", + Destination: &AgentConfig.BindAddress, + } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -278,6 +290,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { DisableDefaultRegistryEndpointFlag, AirgapExtraRegistryFlag, NodeIPFlag, + BindAddressFlag, NodeExternalIPFlag, ResolvConfFlag, FlannelIfaceFlag, @@ -286,6 +299,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { ExtraKubeletArgs, ExtraKubeProxyArgs, // Experimental flags + EnablePProfFlag, &cli.BoolFlag{ Name: "rootless", Usage: "(experimental) Run rootless", diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index e59a88688926..cfb9349fbb10 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -48,8 +48,6 @@ type Server struct { HelmJobImage string TLSSan cli.StringSlice TLSSanSecurity bool - BindAddress string - EnablePProf bool ExtraAPIArgs cli.StringSlice ExtraEtcdArgs cli.StringSlice ExtraSchedulerArgs cli.StringSlice @@ -87,6 +85,7 @@ type Server struct { EncryptSkip bool SystemDefaultRegistry string StartupHooks []StartupHook + SupervisorMetrics bool EtcdSnapshotName string EtcdDisableSnapshots bool EtcdExposeMetrics bool @@ -178,11 +177,7 @@ var ServerFlags = []cli.Flag{ VModule, LogFile, AlsoLogToStderr, - &cli.StringFlag{ - Name: "bind-address", - Usage: "(listener) " + version.Program + " bind address (default: 0.0.0.0)", - Destination: &ServerConfig.BindAddress, - }, + BindAddressFlag, &cli.IntFlag{ Name: "https-listen-port", Usage: "(listener) HTTPS listen port", @@ -493,9 +488,14 @@ var ServerFlags = []cli.Flag{ }, &cli.BoolFlag{ Name: "embedded-registry", - Usage: "(experimental/components) Enable embedded distributed container registry; requires use of embedded containerd", + Usage: "(experimental/components) Enable embedded distributed container registry; requires use of embedded containerd; when enabled agents will also listen on the supervisor port", Destination: &ServerConfig.EmbeddedRegistry, }, + &cli.BoolFlag{ + Name: "supervisor-metrics", + Usage: "(experimental/components) Enable serving " + version.Program + " internal metrics on the supervisor port; when enabled agents will also listen on the supervisor port", + Destination: &ServerConfig.SupervisorMetrics, + }, NodeNameFlag, WithNodeIDFlag, NodeLabels, @@ -534,11 +534,7 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.EncryptSecrets, }, // Experimental flags - &cli.BoolFlag{ - Name: "enable-pprof", - Usage: "(experimental) Enable pprof endpoint on supervisor port", - Destination: &ServerConfig.EnablePProf, - }, + EnablePProfFlag, &cli.BoolFlag{ Name: "rootless", Usage: "(experimental) Run rootless", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 21417d178dd8..88a389b039b0 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -12,13 +12,16 @@ import ( systemd "github.com/coreos/go-systemd/v22/daemon" "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/agent" + "github.com/k3s-io/k3s/pkg/agent/https" "github.com/k3s-io/k3s/pkg/agent/loadbalancer" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/datadir" "github.com/k3s-io/k3s/pkg/etcd" + k3smetrics "github.com/k3s-io/k3s/pkg/metrics" "github.com/k3s-io/k3s/pkg/proctitle" + "github.com/k3s-io/k3s/pkg/profile" "github.com/k3s-io/k3s/pkg/rootless" "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/spegel" @@ -30,7 +33,6 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli" utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apiserver/pkg/authentication/authenticator" kubeapiserverflag "k8s.io/component-base/cli/flag" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" utilsnet "k8s.io/utils/net" @@ -136,12 +138,11 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.ServiceLBNamespace = cfg.ServiceLBNamespace serverConfig.ControlConfig.SANs = util.SplitStringSlice(cfg.TLSSan) serverConfig.ControlConfig.SANSecurity = cfg.TLSSanSecurity - serverConfig.ControlConfig.BindAddress = cfg.BindAddress + serverConfig.ControlConfig.BindAddress = cmds.AgentConfig.BindAddress serverConfig.ControlConfig.SupervisorPort = cfg.SupervisorPort serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort serverConfig.ControlConfig.APIServerPort = cfg.APIServerPort serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress - serverConfig.ControlConfig.EnablePProf = cfg.EnablePProf serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs @@ -174,6 +175,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets serverConfig.ControlConfig.EtcdExposeMetrics = cfg.EtcdExposeMetrics serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots + serverConfig.ControlConfig.SupervisorMetrics = cfg.SupervisorMetrics serverConfig.ControlConfig.VLevel = cmds.LogConfig.VLevel serverConfig.ControlConfig.VModule = cmds.LogConfig.VModule @@ -406,6 +408,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } tlsMinVersionArg := getArgValueFromList("tls-min-version", serverConfig.ControlConfig.ExtraAPIArgs) + serverConfig.ControlConfig.MinTLSVersion = tlsMinVersionArg serverConfig.ControlConfig.TLSMinVersion, err = kubeapiserverflag.TLSVersion(tlsMinVersionArg) if err != nil { return errors.Wrap(err, "invalid tls-min-version") @@ -435,6 +438,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } serverConfig.ControlConfig.ExtraAPIArgs = append(serverConfig.ControlConfig.ExtraAPIArgs, "tls-cipher-suites="+strings.Join(tlsCipherSuites, ",")) } + serverConfig.ControlConfig.CipherSuites = tlsCipherSuites serverConfig.ControlConfig.TLSCipherSuites, err = kubeapiserverflag.TLSCipherSuites(tlsCipherSuites) if err != nil { return errors.Wrap(err, "invalid tls-cipher-suites") @@ -556,28 +560,36 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont go getAPIAddressFromEtcd(ctx, serverConfig, agentConfig) } + // Until the agent is run and retrieves config from the server, we won't know + // if the embedded registry is enabled. If it is not enabled, these are not + // used as the registry is never started. + registry := spegel.DefaultRegistry + registry.Bootstrapper = spegel.NewChainingBootstrapper( + spegel.NewServerBootstrapper(&serverConfig.ControlConfig), + spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir), + spegel.NewSelfBootstrapper(), + ) + registry.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) { + return https.Start(ctx, nodeConfig, serverConfig.ControlConfig.Runtime) + } + + // same deal for metrics - these are not used if the extra metrics listener is not enabled. + metrics := k3smetrics.DefaultMetrics + metrics.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) { + return https.Start(ctx, nodeConfig, serverConfig.ControlConfig.Runtime) + } + + // and for pprof as well + pprof := profile.DefaultProfiler + pprof.Router = func(ctx context.Context, nodeConfig *config.Node) (*mux.Router, error) { + return https.Start(ctx, nodeConfig, serverConfig.ControlConfig.Runtime) + } + if cfg.DisableAgent { agentConfig.ContainerRuntimeEndpoint = "/dev/null" return agent.RunStandalone(ctx, agentConfig) } - if cfg.EmbeddedRegistry { - conf := spegel.DefaultRegistry - conf.Bootstrapper = spegel.NewChainingBootstrapper( - spegel.NewServerBootstrapper(&serverConfig.ControlConfig), - spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir), - spegel.NewSelfBootstrapper(), - ) - conf.HandlerFunc = func(_ *spegel.Config, router *mux.Router) error { - router.NotFoundHandler = serverConfig.ControlConfig.Runtime.Handler - serverConfig.ControlConfig.Runtime.Handler = router - return nil - } - conf.AuthFunc = func() authenticator.Request { - return serverConfig.ControlConfig.Runtime.Authenticator - } - } - return agent.Run(ctx, agentConfig) } diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go index 6ac3e6cb9e24..82d4e2cccd23 100644 --- a/pkg/cluster/https.go +++ b/pkg/cluster/https.go @@ -4,17 +4,16 @@ import ( "context" "crypto/tls" "errors" - "fmt" "io" "log" "net" "net/http" - "net/http/pprof" "os" "path/filepath" + "strconv" - "github.com/gorilla/mux" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/rancher/dynamiclistener" "github.com/rancher/dynamiclistener/factory" @@ -24,7 +23,6 @@ import ( "github.com/rancher/wrangler/pkg/generated/controllers/core" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilsnet "k8s.io/utils/net" ) // newListener returns a new TCP listener and HTTP request handler using dynamiclistener. @@ -43,11 +41,7 @@ func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, os.Remove(filepath.Join(c.config.DataDir, "tls/dynamic-cert.json")) } } - ip := c.config.BindAddress - if utilsnet.IsIPv6String(ip) { - ip = fmt.Sprintf("[%s]", ip) - } - tcp, err := dynamiclistener.NewTCPListener(ip, c.config.SupervisorPort) + tcp, err := util.ListenWithLoopback(ctx, c.config.BindAddress, strconv.Itoa(c.config.SupervisorPort)) if err != nil { return nil, nil, err } @@ -114,17 +108,6 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { return err } - if c.config.EnablePProf { - mux := mux.NewRouter().SkipClean(true) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - mux.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) - mux.NotFoundHandler = handler - handler = mux - } - // Create a HTTP server with the registered request handlers, using logrus for logging server := http.Server{ Handler: handler, diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index f217c4c2a67b..e347e34539c7 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -41,6 +41,8 @@ type Node struct { ImageServiceEndpoint string NoFlannel bool SELinux bool + EnablePProf bool + SupervisorMetrics bool EmbeddedRegistry bool FlannelBackend string FlannelConfFile string @@ -128,6 +130,8 @@ type Agent struct { AirgapExtraRegistry []string DisableCCM bool DisableNPC bool + MinTLSVersion string + CipherSuites []string Rootless bool ProtectKernelDefaults bool DisableServiceLB bool @@ -159,6 +163,7 @@ type CriticalControlArgs struct { EgressSelectorMode string `cli:"egress-selector-mode"` ServiceIPRange *net.IPNet `cli:"service-cidr"` ServiceIPRanges []*net.IPNet `cli:"service-cidr"` + SupervisorMetrics bool `cli:"supervisor-metrics"` } type Control struct { @@ -191,7 +196,6 @@ type Control struct { DisableServiceLB bool Rootless bool ServiceLBNamespace string - EnablePProf bool ExtraAPIArgs []string ExtraControllerArgs []string ExtraCloudControllerArgs []string @@ -208,8 +212,10 @@ type Control struct { ClusterResetRestorePath string EncryptForce bool EncryptSkip bool - TLSMinVersion uint16 - TLSCipherSuites []uint16 + MinTLSVersion string + CipherSuites []string + TLSMinVersion uint16 `json:"-"` + TLSCipherSuites []uint16 `json:"-"` EtcdSnapshotName string `json:"-"` EtcdDisableSnapshots bool `json:"-"` EtcdExposeMetrics bool `json:"-"` diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000000..4ebd1d529251 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,45 @@ +package metrics + +import ( + "context" + "errors" + + "github.com/gorilla/mux" + "github.com/k3s-io/k3s/pkg/agent/https" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/component-base/metrics/legacyregistry" +) + +// DefaultRegisterer is the implementation of the +// prometheus Registerer interface that all metrics operations +// will use. +var DefaultRegisterer = legacyregistry.Registerer() + +// DefaultGatherer is the implementation of the +// prometheus Gatherere interface that all metrics operations +// will use. +var DefaultGatherer = legacyregistry.DefaultGatherer + +// DefaultMetrics is the default instance of a Metrics server +var DefaultMetrics = &Config{ + Router: func(context.Context, *config.Node) (*mux.Router, error) { + return nil, errors.New("not implemented") + }, +} + +// Config holds fields for the metrics listener +type Config struct { + // Router will be called to add the metrics API handler to an existing router. + Router https.RouterFunc +} + +// Start starts binds the metrics API to an existing HTTP router. +func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { + mRouter, err := c.Router(ctx, nodeConfig) + if err != nil { + return err + } + mRouter.Handle("/metrics", promhttp.HandlerFor(DefaultGatherer, promhttp.HandlerOpts{})) + return nil +} diff --git a/pkg/profile/profile.go b/pkg/profile/profile.go new file mode 100644 index 000000000000..39c3929a580e --- /dev/null +++ b/pkg/profile/profile.go @@ -0,0 +1,38 @@ +package profile + +import ( + "context" + "errors" + "net/http/pprof" + + "github.com/gorilla/mux" + "github.com/k3s-io/k3s/pkg/agent/https" + "github.com/k3s-io/k3s/pkg/daemons/config" +) + +// DefaultProfiler the default instance of a performance profiling server +var DefaultProfiler = &Config{ + Router: func(context.Context, *config.Node) (*mux.Router, error) { + return nil, errors.New("not implemented") + }, +} + +// Config holds fields for the pprof listener +type Config struct { + // Router will be called to add the pprof API handler to an existing router. + Router https.RouterFunc +} + +// Start starts binds the pprof API to an existing HTTP router. +func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { + mRouter, err := c.Router(ctx, nodeConfig) + if err != nil { + return err + } + mRouter.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mRouter.HandleFunc("/debug/pprof/profile", pprof.Profile) + mRouter.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mRouter.HandleFunc("/debug/pprof/trace", pprof.Trace) + mRouter.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) + return nil +} diff --git a/pkg/spegel/spegel.go b/pkg/spegel/spegel.go index e9dbd1192461..785a31c915a4 100644 --- a/pkg/spegel/spegel.go +++ b/pkg/spegel/spegel.go @@ -13,13 +13,12 @@ import ( "time" "github.com/containerd/containerd/remotes/docker" + "github.com/k3s-io/k3s/pkg/agent/https" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/version" "github.com/rancher/dynamiclistener/cert" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authentication/request/union" "k8s.io/utils/ptr" "github.com/go-logr/logr" @@ -43,11 +42,8 @@ import ( // DefaultRegistry is the default instance of a Spegel distributed registry var DefaultRegistry = &Config{ Bootstrapper: NewSelfBootstrapper(), - HandlerFunc: func(_ *Config, _ *mux.Router) error { - return errors.New("not implemented") - }, - AuthFunc: func() authenticator.Request { - return union.New(nil) + Router: func(context.Context, *config.Node) (*mux.Router, error) { + return nil, errors.New("not implemented") }, } @@ -60,9 +56,6 @@ var ( resolveLatestTag = false ) -type authFunc func() authenticator.Request -type handlerFunc func(config *Config, router *mux.Router) error - // Config holds fields for a distributed registry type Config struct { ClientCAFile string @@ -89,10 +82,7 @@ type Config struct { Bootstrapper routing.Bootstrapper // HandlerFunc will be called to add the registry API handler to an existing router. - HandlerFunc handlerFunc - - // Authenticator will be called to retrieve an authenticator used to validate the request to the registry API. - AuthFunc authFunc + Router https.RouterFunc } // These values are not currently configurable @@ -237,13 +227,12 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error { // Track images available in containerd and publish via p2p router go state.Track(ctx, ociClient, router, resolveLatestTag) - mRouter := mux.NewRouter().SkipClean(true) - mRouter.Use(c.authMiddleware()) - mRouter.PathPrefix("/v2").Handler(regSvr.Handler) - mRouter.PathPrefix("/v1-" + version.Program + "/p2p").Handler(c.peerInfo()) - if err := c.HandlerFunc(c, mRouter); err != nil { + mRouter, err := c.Router(ctx, nodeConfig) + if err != nil { return err } + mRouter.PathPrefix("/v2").Handler(regSvr.Handler) + mRouter.PathPrefix("/v1-" + version.Program + "/p2p").Handler(c.peerInfo()) // Wait up to 5 seconds for the p2p network to find peers. This will return // immediately if the node is bootstrapping from itself. @@ -269,16 +258,3 @@ func (c *Config) peerInfo() http.HandlerFunc { fmt.Fprintf(resp, "%s/p2p/%s", info.Addrs[0].String(), info.ID.String()) }) } - -// authMiddleware calls the configured authenticator to gate access to the registry API -func (c *Config) authMiddleware() mux.MiddlewareFunc { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if _, ok, err := c.AuthFunc().AuthenticateRequest(req); !ok || err != nil { - http.Error(resp, "Unauthorized", http.StatusUnauthorized) - return - } - next.ServeHTTP(resp, req) - }) - } -} diff --git a/pkg/util/net.go b/pkg/util/net.go index 7bc9f2ec4d79..6f372709135c 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -1,12 +1,15 @@ package util import ( + "context" "errors" "fmt" "net" "os" "strings" + "time" + "github.com/rancher/wrangler/pkg/merr" "github.com/sirupsen/logrus" "github.com/urfave/cli" apinet "k8s.io/apimachinery/pkg/util/net" @@ -319,3 +322,111 @@ func getIPFromInterface(ifaceName string) (string, error) { return "", fmt.Errorf("can't find ip for interface %s", ifaceName) } + +type multiListener struct { + listeners []net.Listener + closing chan struct{} + conns chan acceptRes +} + +type acceptRes struct { + conn net.Conn + err error +} + +// explicit interface check +var _ net.Listener = &multiListener{} + +var loopbacks = []string{"127.0.0.1", "::1"} + +// ListenWithLoopback listens on the given address, as well as on IPv4 and IPv6 loopback addresses. +// If the address is a wildcard, the listener is return unwrapped. +func ListenWithLoopback(ctx context.Context, addr string, port string) (net.Listener, error) { + lc := &net.ListenConfig{ + KeepAlive: 3 * time.Minute, + Control: permitReuse, + } + l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(addr, port)) + if err != nil { + return nil, err + } + + // If we're listening on a wildcard address, we don't need to wrap with the other loopback addresses + switch addr { + case "", "::", "0.0.0.0": + return l, nil + } + + ml := &multiListener{ + listeners: []net.Listener{l}, + closing: make(chan struct{}), + conns: make(chan acceptRes), + } + + for _, laddr := range loopbacks { + if laddr == addr { + continue + } + if l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(laddr, port)); err == nil { + ml.listeners = append(ml.listeners, l) + } else { + logrus.Debugf("Failed to listen on %s: %v", net.JoinHostPort(laddr, port), err) + } + } + + for i := range ml.listeners { + go ml.accept(ml.listeners[i]) + } + + return ml, nil +} + +// Addr returns the address of the non-loopback address that this multiListener is listening on +func (ml *multiListener) Addr() net.Addr { + return ml.listeners[0].Addr() +} + +// Close closes all the listeners +func (ml *multiListener) Close() error { + close(ml.closing) + var errs merr.Errors + for i := range ml.listeners { + err := ml.listeners[i].Close() + if err != nil { + errs = append(errs, err) + } + } + return merr.NewErrors(errs) +} + +// Accept returns a Conn/err pair from one of the waiting listeners +func (ml *multiListener) Accept() (net.Conn, error) { + select { + case res, ok := <-ml.conns: + if ok { + return res.conn, res.err + } + return nil, fmt.Errorf("connection channel closed") + case <-ml.closing: + return nil, fmt.Errorf("listener closed") + } +} + +// accept runs a loop, accepting connections and trying to send on the result channel +func (ml *multiListener) accept(listener net.Listener) { + for { + conn, err := listener.Accept() + r := acceptRes{ + conn: conn, + err: err, + } + select { + case ml.conns <- r: + case <-ml.closing: + if r.err == nil { + r.conn.Close() + } + return + } + } +} diff --git a/pkg/util/net_unix.go b/pkg/util/net_unix.go new file mode 100644 index 000000000000..521e577cd15c --- /dev/null +++ b/pkg/util/net_unix.go @@ -0,0 +1,18 @@ +//go:build !windows +// +build !windows + +package util + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +// permitReuse enables port and address sharing on the socket +func permitReuse(network, addr string, conn syscall.RawConn) error { + return conn.Control(func(fd uintptr) { + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1) + syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1) + }) +} diff --git a/pkg/util/net_windows.go b/pkg/util/net_windows.go new file mode 100644 index 000000000000..bb895c095a77 --- /dev/null +++ b/pkg/util/net_windows.go @@ -0,0 +1,11 @@ +//go:build windows +// +build windows + +package util + +import "syscall" + +// permitReuse is a no-op; port and address reuse is not supported on Windows +func permitReuse(network, addr string, conn syscall.RawConn) error { + return nil +} From 09a217d4c09311bbc01c4663cfa513a4db30c765 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 25 Apr 2024 23:49:47 +0000 Subject: [PATCH 21/31] Convert remaining http handlers over to use util.SendError Signed-off-by: Brad Davidson (cherry picked from commit f8e0648304a99cd5ea6a0653250feeb0514c5a26) Signed-off-by: Brad Davidson --- pkg/cluster/router.go | 12 +++++----- pkg/daemons/control/tunnel.go | 3 +-- pkg/etcd/etcd.go | 4 ++-- pkg/server/cert.go | 4 ++-- pkg/server/router.go | 41 +++++++++-------------------------- pkg/server/secrets-encrypt.go | 13 ++--------- pkg/server/token.go | 7 +++--- pkg/spegel/bootstrap.go | 3 ++- pkg/util/apierrors.go | 1 + 9 files changed, 30 insertions(+), 58 deletions(-) diff --git a/pkg/cluster/router.go b/pkg/cluster/router.go index 4fe2694a7264..39dc5e216482 100644 --- a/pkg/cluster/router.go +++ b/pkg/cluster/router.go @@ -1,7 +1,10 @@ package cluster import ( + "fmt" "net/http" + + "github.com/k3s-io/k3s/pkg/util" ) // getHandler returns a basic request handler that processes requests through @@ -19,11 +22,10 @@ func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) { // if no additional handlers are available. func (c *Cluster) router() http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if c.config.Runtime.Handler == nil { - http.Error(rw, "starting", http.StatusServiceUnavailable) - return + if c.config.Runtime.Handler != nil { + c.config.Runtime.Handler.ServeHTTP(rw, req) + } else { + util.SendError(fmt.Errorf("starting"), rw, req, http.StatusServiceUnavailable) } - - c.config.Runtime.Handler.ServeHTTP(rw, req) }) } diff --git a/pkg/daemons/control/tunnel.go b/pkg/daemons/control/tunnel.go index 3c4b2d54ce0d..86c685318b3f 100644 --- a/pkg/daemons/control/tunnel.go +++ b/pkg/daemons/control/tunnel.go @@ -29,8 +29,7 @@ var defaultDialer = net.Dialer{} func loggingErrorWriter(rw http.ResponseWriter, req *http.Request, code int, err error) { logrus.Debugf("Tunnel server error: %d %v", code, err) - rw.WriteHeader(code) - rw.Write([]byte(err.Error())) + util.SendError(err, rw, req, code) } func setupTunnel(ctx context.Context, cfg *config.Control) (http.Handler, error) { diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index e923e105196e..8d087f227a11 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -761,7 +761,7 @@ func getEndpoints(control *config.Control) []string { // for use by etcd. func toTLSConfig(runtime *config.ControlRuntime) (*tls.Config, error) { if runtime.ClientETCDCert == "" || runtime.ClientETCDKey == "" || runtime.ETCDServerCA == "" { - return nil, errors.New("runtime is not ready yet") + return nil, util.ErrCoreNotReady } clientCert, err := tls.LoadX509KeyPair(runtime.ClientETCDCert, runtime.ClientETCDKey) @@ -1177,7 +1177,7 @@ func (e *ETCD) manageLearners(ctx context.Context) { func (e *ETCD) getETCDNodes() ([]*v1.Node, error) { if e.config.Runtime.Core == nil { - return nil, errors.New("runtime core not ready") + return nil, util.ErrCoreNotReady } nodes := e.config.Runtime.Core.Core().V1().Node() diff --git a/pkg/server/cert.go b/pkg/server/cert.go index 8b20dc99c143..79524ca73100 100644 --- a/pkg/server/cert.go +++ b/pkg/server/cert.go @@ -30,8 +30,8 @@ import ( func caCertReplaceHandler(server *config.Control) http.HandlerFunc { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil || req.Method != http.MethodPut { - resp.WriteHeader(http.StatusNotFound) + if req.Method != http.MethodPut { + util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed) return } force, _ := strconv.ParseBool(req.FormValue("force")) diff --git a/pkg/server/router.go b/pkg/server/router.go index b5b1ad52cdda..a12215a4748a 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -200,11 +200,6 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil { - resp.WriteHeader(http.StatusNotFound) - return - } - nodeName, errCode, err := auth(req) if err != nil { util.SendError(err, resp, req, errCode) @@ -256,11 +251,6 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil { - resp.WriteHeader(http.StatusNotFound) - return - } - nodeName, errCode, err := auth(req) if err != nil { util.SendError(err, resp, req, errCode) @@ -296,10 +286,6 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot func fileHandler(fileName ...string) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil { - resp.WriteHeader(http.StatusNotFound) - return - } resp.Header().Set("Content-Type", "text/plain") if len(fileName) == 1 { @@ -310,8 +296,7 @@ func fileHandler(fileName ...string) http.Handler { for _, f := range fileName { bytes, err := os.ReadFile(f) if err != nil { - logrus.Errorf("Failed to read %s: %v", f, err) - resp.WriteHeader(http.StatusInternalServerError) + util.SendError(errors.Wrapf(err, "failed to read %s", f), resp, req, http.StatusInternalServerError) return } resp.Write(bytes) @@ -336,18 +321,13 @@ func apiserversHandler(server *config.Control) http.Handler { resp.Header().Set("content-type", "application/json") if err := json.NewEncoder(resp).Encode(endpoints); err != nil { - logrus.Errorf("Failed to encode apiserver endpoints: %v", err) - resp.WriteHeader(http.StatusInternalServerError) + util.SendError(errors.Wrap(err, "failed to encode apiserver endpoints"), resp, req, http.StatusInternalServerError) } }) } func configHandler(server *config.Control, cfg *cmds.Server) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil { - resp.WriteHeader(http.StatusNotFound) - return - } // Startup hooks may read and modify cmds.Server in a goroutine, but as these are copied into // config.Control before the startup hooks are called, any modifications need to be sync'd back // into the struct before it is sent to agents. @@ -355,23 +335,21 @@ func configHandler(server *config.Control, cfg *cmds.Server) http.Handler { server.DisableKubeProxy = cfg.DisableKubeProxy resp.Header().Set("content-type", "application/json") if err := json.NewEncoder(resp).Encode(server); err != nil { - logrus.Errorf("Failed to encode agent config: %v", err) - resp.WriteHeader(http.StatusInternalServerError) + util.SendError(errors.Wrap(err, "failed to encode agent config"), resp, req, http.StatusInternalServerError) } }) } func readyzHandler(server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - code := http.StatusOK - data := []byte("ok") if server.Runtime.Core == nil { - code = http.StatusInternalServerError - data = []byte("runtime core not ready") + util.SendError(util.ErrCoreNotReady, resp, req, http.StatusServiceUnavailable) + return } - resp.WriteHeader(code) + data := []byte("ok") + resp.WriteHeader(http.StatusOK) resp.Header().Set("Content-Type", "text/plain") - resp.Header().Set("Content-length", strconv.Itoa(len(data))) + resp.Header().Set("Content-Length", strconv.Itoa(len(data))) resp.Write(data) }) } @@ -379,6 +357,7 @@ func readyzHandler(server *config.Control) http.Handler { func ping() http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { data := []byte("pong") + resp.WriteHeader(http.StatusOK) resp.Header().Set("Content-Type", "text/plain") resp.Header().Set("Content-Length", strconv.Itoa(len(data))) resp.Write(data) @@ -432,7 +411,7 @@ func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper return verifyRemotePassword(ctx, config, &mu, deferredNodes, node) } else { // Otherwise, reject the request until the core is ready. - return "", http.StatusServiceUnavailable, errors.New("runtime core not ready") + return "", http.StatusServiceUnavailable, util.ErrCoreNotReady } } diff --git a/pkg/server/secrets-encrypt.go b/pkg/server/secrets-encrypt.go index 0a5e12d3a24f..dab6c7471019 100644 --- a/pkg/server/secrets-encrypt.go +++ b/pkg/server/secrets-encrypt.go @@ -56,10 +56,6 @@ func getEncryptionRequest(req *http.Request) (*EncryptionRequest, error) { func encryptionStatusHandler(server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil { - resp.WriteHeader(http.StatusNotFound) - return - } status, err := encryptionStatus(server) if err != nil { util.SendErrorWithID(err, "secret-encrypt", resp, req, http.StatusInternalServerError) @@ -160,18 +156,13 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool) func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil { - resp.WriteHeader(http.StatusNotFound) - return - } if req.Method != http.MethodPut { - resp.WriteHeader(http.StatusBadRequest) + util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed) return } encryptReq, err := getEncryptionRequest(req) if err != nil { - resp.WriteHeader(http.StatusBadRequest) - resp.Write([]byte(err.Error())) + util.SendError(err, resp, req, http.StatusBadRequest) return } if encryptReq.Stage != nil { diff --git a/pkg/server/token.go b/pkg/server/token.go index c5da332fa6e7..efd095013f43 100644 --- a/pkg/server/token.go +++ b/pkg/server/token.go @@ -32,16 +32,15 @@ func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) { func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if req.TLS == nil || req.Method != http.MethodPut { - resp.WriteHeader(http.StatusBadRequest) + if req.Method != http.MethodPut { + util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed) return } var err error sTokenReq, err := getServerTokenRequest(req) logrus.Debug("Received token request") if err != nil { - resp.WriteHeader(http.StatusBadRequest) - resp.Write([]byte(err.Error())) + util.SendError(err, resp, req, http.StatusBadRequest) return } if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil { diff --git a/pkg/spegel/bootstrap.go b/pkg/spegel/bootstrap.go index 6d3af649ec10..1acfcc29f429 100644 --- a/pkg/spegel/bootstrap.go +++ b/pkg/spegel/bootstrap.go @@ -10,6 +10,7 @@ import ( "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/libp2p/go-libp2p/core/peer" "github.com/pkg/errors" @@ -133,7 +134,7 @@ func (s *serverBootstrapper) Run(_ context.Context, id string) error { func (s *serverBootstrapper) Get() (addrInfo *peer.AddrInfo, err error) { if s.controlConfig.Runtime.Core == nil { - return nil, errors.New("runtime core not ready") + return nil, util.ErrCoreNotReady } nodeName := os.Getenv("NODE_NAME") if nodeName == "" { diff --git a/pkg/util/apierrors.go b/pkg/util/apierrors.go index 2edca1113986..ec61ecea5465 100644 --- a/pkg/util/apierrors.go +++ b/pkg/util/apierrors.go @@ -17,6 +17,7 @@ import ( var ErrAPINotReady = errors.New("apiserver not ready") var ErrAPIDisabled = errors.New("apiserver disabled") +var ErrCoreNotReady = errors.New("runtime core not ready") // SendErrorWithID sends and logs a random error ID so that logs can be correlated // between the REST API (which does not provide any detailed error output, to avoid From daf9914a52649e9e6753a1283dfedcc55cc501b2 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 13 May 2024 21:23:49 +0000 Subject: [PATCH 22/31] Update golangci-lint to stop using deprecated skip files/dirs Signed-off-by: Brad Davidson (cherry picked from commit 2eca3f1e2c0d5330e00b03ae197402604f1e0348) Signed-off-by: Brad Davidson --- .golangci.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.golangci.json b/.golangci.json index 88ab8ef6d95d..28a4b5daf003 100644 --- a/.golangci.json +++ b/.golangci.json @@ -10,7 +10,10 @@ ] }, "run": { - "skip-dirs": [ + "deadline": "5m" + }, + "issues": { + "exclude-dirs": [ "build", "contrib", "manifests", @@ -18,12 +21,9 @@ "scripts", "vendor" ], - "skip-files": [ + "exclude-files": [ "/zz_generated_" ], - "deadline": "5m" - }, - "issues": { "exclude-rules": [ { "linters": "typecheck", @@ -43,4 +43,4 @@ } ] } -} \ No newline at end of file +} From 02d702b3605c0d5eec9eeb715cb20b8ab2214301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 09:16:10 +0000 Subject: [PATCH 23/31] Bump alpine from 3.18 to 3.20 in /conformance Bumps alpine from 3.18 to 3.20. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] (cherry picked from commit de4cda57e655136930d3feb3e5cab38e95ce68ef) Signed-off-by: Brad Davidson --- conformance/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/Dockerfile b/conformance/Dockerfile index 4401984c01a5..e3bf6b65f0eb 100644 --- a/conformance/Dockerfile +++ b/conformance/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.20 ENV SONOBUOY_VERSION 0.57.1 RUN apk add curl tar gzip RUN curl -sfL https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz | tar xvzf - -C /usr/bin From c879732e3615234a1a2e8459610737ae8c6f89a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 09:46:18 +0000 Subject: [PATCH 24/31] Bump alpine from 3.18 to 3.20 in /package Bumps alpine from 3.18 to 3.20. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] (cherry picked from commit 86875c97bb1b9dd35e51e774bc9f226ccccec4e1) Signed-off-by: Brad Davidson --- package/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/Dockerfile b/package/Dockerfile index 60c03619ef76..7fd7a9875d48 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.18 as base +FROM alpine:3.20 as base RUN apk add -U ca-certificates tar zstd tzdata COPY build/out/data.tar.zst / RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \ From d293908b287b3a781790eb46db78cec647b52a62 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 29 May 2024 00:33:57 +0000 Subject: [PATCH 25/31] Use busybox tar to avoid issues with fchmodat2 on arm Signed-off-by: Brad Davidson (cherry picked from commit 84b578ec74dbdf62d27171bb4ef1e60ec057def9) Signed-off-by: Brad Davidson --- package/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index 7fd7a9875d48..193d6ce32c17 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,8 +1,8 @@ FROM alpine:3.20 as base -RUN apk add -U ca-certificates tar zstd tzdata +RUN apk add -U ca-certificates zstd tzdata COPY build/out/data.tar.zst / RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \ - tar -xa -C /image -f /data.tar.zst && \ + zstdcat -d /data.tar.zst | tar -xa -C /image && \ echo "root:x:0:0:root:/:/bin/sh" > /image/etc/passwd && \ echo "root:x:0:" > /image/etc/group && \ cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt From bc504ea5d0a266490d9a93f15b591eb08c0b971c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 16:15:38 +0000 Subject: [PATCH 26/31] Bump ubuntu from 22.04 to 24.04 in /tests/e2e/scripts Bumps ubuntu from 22.04 to 24.04. --- updated-dependencies: - dependency-name: ubuntu dependency-type: direct:production ... Signed-off-by: dependabot[bot] (cherry picked from commit 4cb4542c3a7002fc254c71bac92fd1d6d26f776f) Signed-off-by: Brad Davidson --- tests/e2e/scripts/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/scripts/Dockerfile b/tests/e2e/scripts/Dockerfile index f6beedbdae7e..acb0abe52882 100644 --- a/tests/e2e/scripts/Dockerfile +++ b/tests/e2e/scripts/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG EXTERNAL_ENCODED_VPN ARG VPN_ENCODED_LOGIN From 22f1b3133c35ac27bb69f6b093990ec5c6152757 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 26 May 2024 18:10:03 +0000 Subject: [PATCH 27/31] chore: Bump Trivy version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli (cherry picked from commit f2e7c01acfdc5f51bfd007c44bfe6605e8864975) Signed-off-by: Brad Davidson --- Dockerfile.dapper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 6666fa63596c..fa90d4ea0655 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -22,7 +22,7 @@ RUN apk -U --no-cache add \ RUN python3 -m pip install awscli # Install Trivy -ENV TRIVY_VERSION="0.50.1" +ENV TRIVY_VERSION="0.51.4" RUN case "$(go env GOARCH)" in \ arm64) TRIVY_ARCH="ARM64" ;; \ amd64) TRIVY_ARCH="64bit" ;; \ From 37a3ba61410616fd62a1887e1f0417ad396cfde6 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 6 May 2024 19:43:37 +0000 Subject: [PATCH 28/31] Fix netpol crash when node remains tained unintialized It is concievable that users might take more than 60 seconds to deploy their own cloud-provider. Instead of exiting, we should wait forever, but with more logging to indicate what's being waited on. Signed-off-by: Brad Davidson (cherry picked from commit ed23a2bb48d4c02321fe0e56890aef90e8299746) Signed-off-by: Brad Davidson --- pkg/agent/netpol/netpol.go | 15 +++++++-------- pkg/etcd/metadata_controller.go | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/agent/netpol/netpol.go b/pkg/agent/netpol/netpol.go index 60d2c1f07f45..f09d47d11e5b 100644 --- a/pkg/agent/netpol/netpol.go +++ b/pkg/agent/netpol/netpol.go @@ -67,27 +67,26 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { return err } - // As kube-router netpol requires addresses to be available in the node object - // Wait until the node has ready addresses to avoid race conditions (max 1 minute). + // kube-router netpol requires addresses to be available in the node object. + // Wait until the uninitialized taint has been removed, at which point the addresses should be set. // TODO: Replace with non-deprecated PollUntilContextTimeout when our and Kubernetes code migrate to it - if err := wait.PollImmediateWithContext(ctx, 2*time.Second, 60*time.Second, func(ctx context.Context) (bool, error) { + if err := wait.PollImmediateInfiniteWithContext(ctx, 2*time.Second, func(ctx context.Context) (bool, error) { // Get the node object node, err := client.CoreV1().Nodes().Get(ctx, nodeConfig.AgentConfig.NodeName, metav1.GetOptions{}) if err != nil { - logrus.Debugf("Network policy controller waiting to get Node %s: %v", nodeConfig.AgentConfig.NodeName, err) + logrus.Infof("Network policy controller waiting to get Node %s: %v", nodeConfig.AgentConfig.NodeName, err) return false, nil } - // Check for the uninitialized taint that should be removed by cloud-provider - // If there is no cloud-provider, the taint will not be there + // Check for the taint that should be removed by cloud-provider when the node has been initialized. for _, taint := range node.Spec.Taints { if taint.Key == cloudproviderapi.TaintExternalCloudProvider { - logrus.Debugf("Network policy controller waiting for removal of %s taint", cloudproviderapi.TaintExternalCloudProvider) + logrus.Infof("Network policy controller waiting for removal of %s taint", cloudproviderapi.TaintExternalCloudProvider) return false, nil } } return true, nil }); err != nil { - return errors.Wrapf(err, "network policy controller timed out waiting for %s taint to be removed from Node %s", cloudproviderapi.TaintExternalCloudProvider, nodeConfig.AgentConfig.NodeName) + return errors.Wrapf(err, "network policy controller failed to wait for %s taint to be removed from Node %s", cloudproviderapi.TaintExternalCloudProvider, nodeConfig.AgentConfig.NodeName) } krConfig := options.NewKubeRouterConfig() diff --git a/pkg/etcd/metadata_controller.go b/pkg/etcd/metadata_controller.go index 50b2f6c74d6f..42d19b8eccbe 100644 --- a/pkg/etcd/metadata_controller.go +++ b/pkg/etcd/metadata_controller.go @@ -13,7 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/util/retry" - nodeUtil "k8s.io/kubernetes/pkg/controller/util/node" + nodeutil "k8s.io/kubernetes/pkg/controller/util/node" ) func registerMetadataHandlers(ctx context.Context, etcd *ETCD) { @@ -109,7 +109,7 @@ func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) { node.Labels = map[string]string{} } - if find, _ := nodeUtil.GetNodeCondition(&node.Status, etcdStatusType); find >= 0 { + if find, _ := nodeutil.GetNodeCondition(&node.Status, etcdStatusType); find >= 0 { node.Status.Conditions = append(node.Status.Conditions[:find], node.Status.Conditions[find+1:]...) } From b9a0ded0b3a950183f36ee54e139c3687cc79c8c Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 29 May 2024 18:17:29 +0000 Subject: [PATCH 29/31] Fix issue caused by sole server marked as failed under load If health checks are failing for all servers, make a second pass through the server list with health-checks ignored before returning failure Signed-off-by: Brad Davidson (cherry picked from commit ca39614d4e7cb9963ee002dc798884167f2d2acf) Signed-off-by: Brad Davidson --- pkg/agent/loadbalancer/loadbalancer.go | 9 +++++++-- pkg/agent/loadbalancer/servers.go | 10 ++++++++-- pkg/etcd/etcdproxy.go | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pkg/agent/loadbalancer/loadbalancer.go b/pkg/agent/loadbalancer/loadbalancer.go index 2348fcb087bc..feddb4d872fc 100644 --- a/pkg/agent/loadbalancer/loadbalancer.go +++ b/pkg/agent/loadbalancer/loadbalancer.go @@ -158,6 +158,7 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, _ string) (net lb.mutex.RLock() defer lb.mutex.RUnlock() + var allChecksFailed bool startIndex := lb.nextServerIndex for { targetServer := lb.currentServerAddress @@ -165,7 +166,7 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, _ string) (net server := lb.servers[targetServer] if server == nil || targetServer == "" { logrus.Debugf("Nil server for load balancer %s: %s", lb.serviceName, targetServer) - } else if server.healthCheck() { + } else if allChecksFailed || server.healthCheck() { conn, err := server.dialContext(ctx, network, targetServer) if err == nil { return conn, nil @@ -189,7 +190,11 @@ func (lb *LoadBalancer) dialContext(ctx context.Context, network, _ string) (net startIndex = maxIndex } if lb.nextServerIndex == startIndex { - return nil, errors.New("all servers failed") + if allChecksFailed { + return nil, errors.New("all servers failed") + } + logrus.Debugf("Health checks for all servers in load balancer %s have failed: retrying with health checks ignored", lb.serviceName) + allChecksFailed = true } } } diff --git a/pkg/agent/loadbalancer/servers.go b/pkg/agent/loadbalancer/servers.go index 78ee88d74fbd..7dc80e493244 100644 --- a/pkg/agent/loadbalancer/servers.go +++ b/pkg/agent/loadbalancer/servers.go @@ -227,13 +227,19 @@ func (lb *LoadBalancer) SetHealthCheck(address string, healthCheck func() bool) // runHealthChecks periodically health-checks all servers. Any servers that fail the health-check will have their // connections closed, to force clients to switch over to a healthy server. func (lb *LoadBalancer) runHealthChecks(ctx context.Context) { + previousStatus := map[string]bool{} wait.Until(func() { lb.mutex.RLock() defer lb.mutex.RUnlock() - for _, server := range lb.servers { - if !server.healthCheck() { + for address, server := range lb.servers { + status := server.healthCheck() + if status == false && previousStatus[address] == true { + // Only close connections when the server transitions from healthy to unhealthy; + // we don't want to re-close all the connections every time as we might be ignoring + // health checks due to all servers being marked unhealthy. defer server.closeAll() } + previousStatus[address] = status } }, time.Second, ctx.Done()) logrus.Debugf("Stopped health checking for load balancer %s", lb.serviceName) diff --git a/pkg/etcd/etcdproxy.go b/pkg/etcd/etcdproxy.go index 40bee876b120..55918850b3ff 100644 --- a/pkg/etcd/etcdproxy.go +++ b/pkg/etcd/etcdproxy.go @@ -130,7 +130,7 @@ func (e etcdproxy) createHealthCheck(ctx context.Context, address string) func() statusCode = resp.StatusCode } if err != nil || statusCode != http.StatusOK { - logrus.Debugf("Health check %s failed: %v (StatusCode: %d)", url, err, statusCode) + logrus.Debugf("Health check %s failed: %v (StatusCode: %d)", address, err, statusCode) connected = false } else { connected = true From 41a26da3cb657fab7ef53ea3301b91cf4a8ab1d8 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 30 May 2024 19:00:47 +0000 Subject: [PATCH 30/31] Fix embedded mirror blocked by SAR RBAC and re-enable test Signed-off-by: Brad Davidson --- .github/workflows/e2e.yaml | 5 ++--- pkg/agent/https/https.go | 6 +++++- tests/e2e/embeddedmirror/Vagrantfile | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5479bfd3c668..65f7a61e2755 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -36,8 +36,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO fix embeddedmirror and add it to the matrix - etest: [startup, s3, btrfs, externalip, privateregistry, wasm] + etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm] max-parallel: 3 steps: - name: "Checkout" @@ -116,4 +115,4 @@ jobs: chmod +x ./dist/artifacts/k3s . ./tests/docker/test-helpers . ./tests/docker/test-run-${{ matrix.dtest }} - echo "Did test-run-${{ matrix.dtest }} pass $?" \ No newline at end of file + echo "Did test-run-${{ matrix.dtest }} pass $?" diff --git a/pkg/agent/https/https.go b/pkg/agent/https/https.go index 2b5927107f9b..da453742b8a7 100644 --- a/pkg/agent/https/https.go +++ b/pkg/agent/https/https.go @@ -75,7 +75,11 @@ func Start(ctx context.Context, nodeConfig *config.Node, runtime *config.Control } authz := options.NewDelegatingAuthorizationOptions() - authz.AlwaysAllowPaths = []string{"/v2", "/debug/pprof", "/v1-" + version.Program + "/p2p"} + authz.AlwaysAllowPaths = []string{ // skip authz for paths that should not use SubjectAccessReview; basically everything that will use this router other than metrics + "/v1-" + version.Program + "/p2p", // spegel libp2p peer discovery + "/v2/*", // spegel registry mirror + "/debug/pprof/*", // profiling + } authz.RemoteKubeConfigFile = nodeConfig.AgentConfig.KubeConfigKubelet if applyErr := authz.ApplyTo(&config.Authorization); applyErr != nil { err = applyErr diff --git a/tests/e2e/embeddedmirror/Vagrantfile b/tests/e2e/embeddedmirror/Vagrantfile index f510051361f3..67bc1709f128 100644 --- a/tests/e2e/embeddedmirror/Vagrantfile +++ b/tests/e2e/embeddedmirror/Vagrantfile @@ -38,6 +38,9 @@ def provision(vm, role, role_num, node_num) if role.include?("server") && role_num == 0 vm.provision "private-registry", type: "shell", inline: writePrivateRegistry + vm.provision "create-images-dir", type: "shell", inline: "mkdir -p -m 777 /tmp/images /var/lib/rancher/k3s/agent/images" + vm.provision "copy-images-file", type: "file", source: "../../../scripts/airgap/image-list.txt", destination: "/tmp/images/image-list.txt" + vm.provision "move-images-file", type: "shell", inline: "mv /tmp/images/image-list.txt /var/lib/rancher/k3s/agent/images/image-list.txt" vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s| k3s.args = "server " @@ -54,6 +57,9 @@ def provision(vm, role, role_num, node_num) elsif role.include?("server") && role_num != 0 vm.provision "shell", inline: writePrivateRegistry + vm.provision "create-images-dir", type: "shell", inline: "mkdir -p -m 777 /tmp/images /var/lib/rancher/k3s/agent/images" + vm.provision "copy-images-file", type: "file", source: "../../../scripts/airgap/image-list.txt", destination: "/tmp/images/image-list.txt" + vm.provision "move-images-file", type: "shell", inline: "mv /tmp/images/image-list.txt /var/lib/rancher/k3s/agent/images/image-list.txt" vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s| k3s.args = "server" From 3c8f89eeb479d215803b667e21478730e1b0f292 Mon Sep 17 00:00:00 2001 From: Katherine Door Date: Fri, 31 May 2024 08:45:34 +0200 Subject: [PATCH 31/31] Add write-kubeconfig-group flag to server (#9233) * Add write-kubeconfig-group flag to server * update kubectl unable to read config message for kubeconfig mode/group Signed-off-by: Katherine Pata (cherry picked from commit 7a0ea3c9539c9ac8efb560de32395c9611db7969) Signed-off-by: Brad Davidson --- pkg/cli/cmds/server.go | 7 +++++++ pkg/cli/server/server.go | 1 + pkg/daemons/config/types.go | 1 + pkg/kubectl/main.go | 3 ++- pkg/server/server.go | 7 +++++++ pkg/util/file.go | 23 +++++++++++++++++++++++ 6 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index cfb9349fbb10..e179f5237de3 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -45,6 +45,7 @@ type Server struct { DisableAgent bool KubeConfigOutput string KubeConfigMode string + KubeConfigGroup string HelmJobImage string TLSSan cli.StringSlice TLSSanSecurity bool @@ -250,6 +251,12 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.KubeConfigMode, EnvVar: version.ProgramUpper + "_KUBECONFIG_MODE", }, + &cli.StringFlag{ + Name: "write-kubeconfig-group", + Usage: "(client) Write kubeconfig with this group", + Destination: &ServerConfig.KubeConfigGroup, + EnvVar: version.ProgramUpper + "_KUBECONFIG_GROUP", + }, &cli.StringFlag{ Name: "helm-job-image", Usage: "(helm) Default image to use for helm jobs", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 88a389b039b0..7fd735bba495 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -133,6 +133,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.DataDir = cfg.DataDir serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode + serverConfig.ControlConfig.KubeConfigGroup = cfg.KubeConfigGroup serverConfig.ControlConfig.HelmJobImage = cfg.HelmJobImage serverConfig.ControlConfig.Rootless = cfg.Rootless serverConfig.ControlConfig.ServiceLBNamespace = cfg.ServiceLBNamespace diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index e347e34539c7..02c8d01441a8 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -182,6 +182,7 @@ type Control struct { ServiceNodePortRange *utilnet.PortRange KubeConfigOutput string KubeConfigMode string + KubeConfigGroup string HelmJobImage string DataDir string KineTLS bool diff --git a/pkg/kubectl/main.go b/pkg/kubectl/main.go index f3d77f24a11d..dfcab9292dca 100644 --- a/pkg/kubectl/main.go +++ b/pkg/kubectl/main.go @@ -54,7 +54,8 @@ func checkReadConfigPermissions(configFile string) error { if err != nil { if os.IsPermission(err) { return fmt.Errorf("Unable to read %s, please start server "+ - "with --write-kubeconfig-mode to modify kube config permissions", configFile) + "with --write-kubeconfig-mode or --write-kubeconfig-group "+ + "to modify kube config permissions", configFile) } } file.Close() diff --git a/pkg/server/server.go b/pkg/server/server.go index a13d2caf051b..81818e638407 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -465,6 +465,13 @@ func writeKubeConfig(certs string, config *Config) error { util.SetFileModeForPath(kubeConfig, os.FileMode(0600)) } + if config.ControlConfig.KubeConfigGroup != "" { + err := util.SetFileGroupForPath(kubeConfig, config.ControlConfig.KubeConfigGroup) + if err != nil { + logrus.Errorf("Failed to set %s to group %s: %v", kubeConfig, config.ControlConfig.KubeConfigGroup, err) + } + } + if kubeConfigSymlink != kubeConfig { if err := writeConfigSymlink(kubeConfig, kubeConfigSymlink); err != nil { logrus.Errorf("Failed to write kubeconfig symlink: %v", err) diff --git a/pkg/util/file.go b/pkg/util/file.go index d584ec8105c0..6d1a05ca84ad 100644 --- a/pkg/util/file.go +++ b/pkg/util/file.go @@ -2,7 +2,9 @@ package util import ( "os" + "os/user" "path/filepath" + "strconv" "strings" "time" @@ -14,6 +16,27 @@ func SetFileModeForPath(name string, mode os.FileMode) error { return os.Chmod(name, mode) } +func SetFileGroupForPath(name string, group string) error { + // Try to use as group id + gid, err := strconv.Atoi(group) + if err == nil { + return os.Chown(name, -1, gid) + } + + // Otherwise, it must be a group name + g, err := user.LookupGroup(group) + if err != nil { + return err + } + + gid, err = strconv.Atoi(g.Gid) + if err != nil { + return err + } + + return os.Chown(name, -1, gid) +} + func SetFileModeForFile(file *os.File, mode os.FileMode) error { return file.Chmod(mode) }