From ccc024f60adcc48591dd98befb7204fc56350014 Mon Sep 17 00:00:00 2001 From: Joey Wang Date: Wed, 1 Jun 2022 13:04:41 -0400 Subject: [PATCH] Add proxy config setup when generating CAPI objects in apibuilder (#2262) * Add http proxy support in apibuilder * Make NoProxyDefaults func * Move OS specific cmds to snow provider --- pkg/clusterapi/apibuilder.go | 8 - pkg/clusterapi/apibuilder_test.go | 33 +-- pkg/clusterapi/config/http-proxy.conf | 4 + pkg/clusterapi/proxy.go | 93 ++++++++ pkg/clusterapi/proxy_test.go | 82 +++++++ .../{registryMirror.go => registry_mirror.go} | 19 +- ...Mirror_test.go => registry_mirror_test.go} | 10 - pkg/clusterapi/systemctl.go | 45 ++++ pkg/clusterapi/systemctl_test.go | 133 +++++++++++ pkg/providers/cloudstack/cloudstack.go | 4 +- pkg/providers/common/common.go | 6 - pkg/providers/snow/apibuilder.go | 27 ++- pkg/providers/snow/apibuilder_test.go | 207 ++++++++++++++---- pkg/providers/vsphere/vsphere.go | 4 +- test/e2e/proxy_test.go | 13 ++ 15 files changed, 577 insertions(+), 111 deletions(-) create mode 100644 pkg/clusterapi/config/http-proxy.conf create mode 100644 pkg/clusterapi/proxy.go create mode 100644 pkg/clusterapi/proxy_test.go rename pkg/clusterapi/{registryMirror.go => registry_mirror.go} (73%) rename pkg/clusterapi/{registryMirror_test.go => registry_mirror_test.go} (90%) create mode 100644 pkg/clusterapi/systemctl.go create mode 100644 pkg/clusterapi/systemctl_test.go diff --git a/pkg/clusterapi/apibuilder.go b/pkg/clusterapi/apibuilder.go index c909d1fe13bf..19c7b72c47d6 100644 --- a/pkg/clusterapi/apibuilder.go +++ b/pkg/clusterapi/apibuilder.go @@ -162,10 +162,6 @@ func KubeadmControlPlane(clusterSpec *cluster.Spec, infrastructureObject APIObje }, } - if err := SetRegistryMirrorInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil { - return nil, err - } - SetIdentityAuthInKubeadmControlPlane(kcp, clusterSpec) return kcp, nil @@ -207,10 +203,6 @@ func KubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig v1al }, } - if err := SetRegistryMirrorInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil { - return nil, err - } - return kct, nil } diff --git a/pkg/clusterapi/apibuilder_test.go b/pkg/clusterapi/apibuilder_test.go index a353af116225..6adb3d64751f 100644 --- a/pkg/clusterapi/apibuilder_test.go +++ b/pkg/clusterapi/apibuilder_test.go @@ -65,6 +65,9 @@ func newApiBuilerTest(t *testing.T) apiBuilerTest { }, }, ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{ + Endpoint: &v1alpha1.Endpoint{ + Host: "1.2.3.4", + }, Count: 3, }, KubernetesVersion: "1.21", @@ -254,21 +257,6 @@ func TestKubeadmControlPlane(t *testing.T) { tt.Expect(got).To(Equal(want)) } -func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) { - for _, tt := range registryMirrorTests { - t.Run(tt.name, func(t *testing.T) { - g := newApiBuilerTest(t) - g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig - got, err := clusterapi.KubeadmControlPlane(g.clusterSpec, g.providerMachineTemplate) - g.Expect(err).To(Succeed()) - want := wantKubeadmControlPlane() - want.Spec.KubeadmConfigSpec.Files = tt.wantFiles - want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands() - g.Expect(got).To(Equal(want)) - }) - } -} - func wantKubeadmConfigTemplate() *bootstrapv1.KubeadmConfigTemplate { return &bootstrapv1.KubeadmConfigTemplate{ TypeMeta: metav1.TypeMeta{ @@ -314,21 +302,6 @@ func TestKubeadmConfigTemplate(t *testing.T) { tt.Expect(got).To(Equal(want)) } -func TestKubeadmConfigTemplateWithRegistryMirror(t *testing.T) { - for _, tt := range registryMirrorTests { - t.Run(tt.name, func(t *testing.T) { - g := newApiBuilerTest(t) - g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig - got, err := clusterapi.KubeadmConfigTemplate(g.clusterSpec, *g.workerNodeGroupConfig) - g.Expect(err).To(Succeed()) - want := wantKubeadmConfigTemplate() - want.Spec.Template.Spec.Files = tt.wantFiles - want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands() - g.Expect(got).To(Equal(want)) - }) - } -} - func TestMachineDeployment(t *testing.T) { tt := newApiBuilerTest(t) got := clusterapi.MachineDeployment(tt.clusterSpec, *tt.workerNodeGroupConfig, tt.kubeadmConfigTemplate, tt.providerMachineTemplate) diff --git a/pkg/clusterapi/config/http-proxy.conf b/pkg/clusterapi/config/http-proxy.conf new file mode 100644 index 000000000000..0e47b98a3a59 --- /dev/null +++ b/pkg/clusterapi/config/http-proxy.conf @@ -0,0 +1,4 @@ +[Service] +Environment="HTTP_PROXY={{.httpProxy}}" +Environment="HTTPS_PROXY={{.httpsProxy}}" +Environment="NO_PROXY={{ stringsJoin .noProxy "," }}" \ No newline at end of file diff --git a/pkg/clusterapi/proxy.go b/pkg/clusterapi/proxy.go new file mode 100644 index 000000000000..af6a1f0ce2da --- /dev/null +++ b/pkg/clusterapi/proxy.go @@ -0,0 +1,93 @@ +package clusterapi + +import ( + _ "embed" + "fmt" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/templater" +) + +//go:embed config/http-proxy.conf +var proxyConfig string + +func NoProxyDefaults() []string { + return []string{ + "localhost", + "127.0.0.1", + ".svc", + } +} + +func proxyConfigContent(cluster v1alpha1.ClusterSpec) (string, error) { + capacity := len(cluster.ClusterNetwork.Pods.CidrBlocks) + + len(cluster.ClusterNetwork.Services.CidrBlocks) + + len(cluster.ProxyConfiguration.NoProxy) + 4 + + noProxyList := make([]string, 0, capacity) + noProxyList = append(noProxyList, cluster.ClusterNetwork.Pods.CidrBlocks...) + noProxyList = append(noProxyList, cluster.ClusterNetwork.Services.CidrBlocks...) + noProxyList = append(noProxyList, cluster.ProxyConfiguration.NoProxy...) + + // Add no-proxy defaults + noProxyList = append(noProxyList, NoProxyDefaults()...) + noProxyList = append(noProxyList, cluster.ControlPlaneConfiguration.Endpoint.Host) + + val := values{ + "httpProxy": cluster.ProxyConfiguration.HttpProxy, + "httpsProxy": cluster.ProxyConfiguration.HttpsProxy, + "noProxy": noProxyList, + } + + config, err := templater.Execute(proxyConfig, val) + if err != nil { + return "", fmt.Errorf("building http-proxy.conf file: %v", err) + } + return string(config), nil +} + +func proxyConfigFile(cluster v1alpha1.ClusterSpec) (bootstrapv1.File, error) { + proxyConfig, err := proxyConfigContent(cluster) + if err != nil { + return bootstrapv1.File{}, err + } + + return bootstrapv1.File{ + Path: "/etc/systemd/system/containerd.service.d/http-proxy.conf", + Owner: "root:root", + Content: proxyConfig, + }, nil +} + +func SetProxyConfigInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) error { + if cluster.ProxyConfiguration == nil { + return nil + } + + proxyConfigFile, err := proxyConfigFile(cluster) + if err != nil { + return err + } + + kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, proxyConfigFile) + + return nil +} + +func SetProxyConfigInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) error { + if cluster.ProxyConfiguration == nil { + return nil + } + + proxyConfigFile, err := proxyConfigFile(cluster) + if err != nil { + return err + } + + kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, proxyConfigFile) + + return nil +} diff --git a/pkg/clusterapi/proxy_test.go b/pkg/clusterapi/proxy_test.go new file mode 100644 index 000000000000..12eec10b539d --- /dev/null +++ b/pkg/clusterapi/proxy_test.go @@ -0,0 +1,82 @@ +package clusterapi_test + +import ( + "testing" + + . "github.com/onsi/gomega" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/clusterapi" +) + +var proxyTests = []struct { + name string + proxy *v1alpha1.ProxyConfiguration + wantFiles []bootstrapv1.File +}{ + { + name: "proxy config nil", + proxy: nil, + wantFiles: []bootstrapv1.File{}, + }, + { + name: "with proxy, pods cidr, service cidr, cp endpoint", + proxy: &v1alpha1.ProxyConfiguration{ + HttpProxy: "1.2.3.4:8888", + HttpsProxy: "1.2.3.4:8888", + NoProxy: []string{ + "1.2.3.4/0", + "1.2.3.5/0", + }, + }, + wantFiles: []bootstrapv1.File{ + { + Path: "/etc/systemd/system/containerd.service.d/http-proxy.conf", + Owner: "root:root", + Content: `[Service] +Environment="HTTP_PROXY=1.2.3.4:8888" +Environment="HTTPS_PROXY=1.2.3.4:8888" +Environment="NO_PROXY=1.2.3.4/5,1.2.3.4/5,1.2.3.4/0,1.2.3.5/0,localhost,127.0.0.1,.svc,1.2.3.4"`, + }, + }, + }, +} + +func TestSetProxyConfigInKubeadmControlPlane(t *testing.T) { + for _, tt := range proxyTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + got := wantKubeadmControlPlane() + g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy + g.Expect(clusterapi.SetProxyConfigInKubeadmControlPlane(got, g.clusterSpec.Cluster.Spec)).To(Succeed()) + want := wantKubeadmControlPlane() + want.Spec.KubeadmConfigSpec.Files = tt.wantFiles + g.Expect(got).To(Equal(want)) + }) + } +} + +func TestSetProxyConfigInKubeadmConfigTemplate(t *testing.T) { + for _, tt := range proxyTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + got := wantKubeadmConfigTemplate() + g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy + g.Expect(clusterapi.SetProxyConfigInKubeadmConfigTemplate(got, g.clusterSpec.Cluster.Spec)).To(Succeed()) + want := wantKubeadmConfigTemplate() + want.Spec.Template.Spec.Files = tt.wantFiles + g.Expect(got).To(Equal(want)) + }) + } +} + +func TestNoProxyDefaults(t *testing.T) { + g := NewWithT(t) + want := []string{ + "localhost", + "127.0.0.1", + ".svc", + } + g.Expect(clusterapi.NoProxyDefaults()).To(Equal(want)) +} diff --git a/pkg/clusterapi/registryMirror.go b/pkg/clusterapi/registry_mirror.go similarity index 73% rename from pkg/clusterapi/registryMirror.go rename to pkg/clusterapi/registry_mirror.go index 832a7cfcfbd5..6311e1d77807 100644 --- a/pkg/clusterapi/registryMirror.go +++ b/pkg/clusterapi/registry_mirror.go @@ -26,16 +26,16 @@ func registryMirrorConfigContent(registryAddress, registryCert string, insecureS config, err := templater.Execute(containerdConfig, val) if err != nil { - return "", fmt.Errorf("failed building containerd config file: %v", err) + return "", fmt.Errorf("building containerd config file: %v", err) } return string(config), nil } -func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration) (files []bootstrapv1.File, preKubeadmCommands []string, err error) { +func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration) (files []bootstrapv1.File, err error) { registryAddress := net.JoinHostPort(registryMirrorConfig.Endpoint, registryMirrorConfig.Port) registryConfig, err := registryMirrorConfigContent(registryAddress, registryMirrorConfig.CACertContent, registryMirrorConfig.InsecureSkipVerify) if err != nil { - return nil, nil, err + return nil, err } files = []bootstrapv1.File{ { @@ -53,12 +53,7 @@ func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfigura }) } - preKubeadmCommands = []string{ - "cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml", - "sudo systemctl daemon-reload", - "sudo systemctl restart containerd", - } - return files, preKubeadmCommands, nil + return files, nil } func SetRegistryMirrorInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, mirrorConfig *v1alpha1.RegistryMirrorConfiguration) error { @@ -66,13 +61,12 @@ func SetRegistryMirrorInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPl return nil } - containerdFiles, containerdCommands, err := registryMirrorConfig(mirrorConfig) + containerdFiles, err := registryMirrorConfig(mirrorConfig) if err != nil { return fmt.Errorf("setting registry mirror configuration: %v", err) } kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, containerdFiles...) - kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, containerdCommands...) return nil } @@ -82,13 +76,12 @@ func SetRegistryMirrorInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemp return nil } - containerdFiles, containerdCommands, err := registryMirrorConfig(mirrorConfig) + containerdFiles, err := registryMirrorConfig(mirrorConfig) if err != nil { return fmt.Errorf("setting registry mirror configuration: %v", err) } kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, containerdFiles...) - kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, containerdCommands...) return nil } diff --git a/pkg/clusterapi/registryMirror_test.go b/pkg/clusterapi/registry_mirror_test.go similarity index 90% rename from pkg/clusterapi/registryMirror_test.go rename to pkg/clusterapi/registry_mirror_test.go index 27d15d6c72a2..b1f8dcf32d8e 100644 --- a/pkg/clusterapi/registryMirror_test.go +++ b/pkg/clusterapi/registry_mirror_test.go @@ -86,14 +86,6 @@ var registryMirrorTests = []struct { }, } -func wantRegistryMirrorCommands() []string { - return []string{ - "cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml", - "sudo systemctl daemon-reload", - "sudo systemctl restart containerd", - } -} - func TestSetRegistryMirrorInKubeadmControlPlane(t *testing.T) { for _, tt := range registryMirrorTests { t.Run(tt.name, func(t *testing.T) { @@ -102,7 +94,6 @@ func TestSetRegistryMirrorInKubeadmControlPlane(t *testing.T) { g.Expect(clusterapi.SetRegistryMirrorInKubeadmControlPlane(got, tt.registryMirrorConfig)).To(Succeed()) want := wantKubeadmControlPlane() want.Spec.KubeadmConfigSpec.Files = tt.wantFiles - want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands() g.Expect(got).To(Equal(want)) }) } @@ -116,7 +107,6 @@ func TestSetRegistryMirrorInKubeadmConfigTemplate(t *testing.T) { g.Expect(clusterapi.SetRegistryMirrorInKubeadmConfigTemplate(got, tt.registryMirrorConfig)).To(Succeed()) want := wantKubeadmConfigTemplate() want.Spec.Template.Spec.Files = tt.wantFiles - want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands() g.Expect(got).To(Equal(want)) }) } diff --git a/pkg/clusterapi/systemctl.go b/pkg/clusterapi/systemctl.go new file mode 100644 index 000000000000..b267c7af4472 --- /dev/null +++ b/pkg/clusterapi/systemctl.go @@ -0,0 +1,45 @@ +package clusterapi + +import ( + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" +) + +var buildContainerdConfigCommands = []string{ + "cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml", +} + +var restartContainerdCommands = []string{ + "sudo systemctl daemon-reload", + "sudo systemctl restart containerd", +} + +func CreateContainerdConfigFileInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) { + if cluster.RegistryMirrorConfiguration != nil { + kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, buildContainerdConfigCommands...) + } +} + +func CreateContainerdConfigFileInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) { + if cluster.RegistryMirrorConfiguration != nil { + kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, buildContainerdConfigCommands...) + } +} + +func RestartContainerdInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) { + if restartContainerdNeeded(cluster) { + kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, restartContainerdCommands...) + } +} + +func RestartContainerdInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) { + if restartContainerdNeeded(cluster) { + kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, restartContainerdCommands...) + } +} + +func restartContainerdNeeded(cluster v1alpha1.ClusterSpec) bool { + return cluster.RegistryMirrorConfiguration != nil || cluster.ProxyConfiguration != nil +} diff --git a/pkg/clusterapi/systemctl_test.go b/pkg/clusterapi/systemctl_test.go new file mode 100644 index 000000000000..9d50a2c1fa16 --- /dev/null +++ b/pkg/clusterapi/systemctl_test.go @@ -0,0 +1,133 @@ +package clusterapi_test + +import ( + "testing" + + . "github.com/onsi/gomega" + + "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/clusterapi" +) + +var restartContainerdCommands = []string{ + "sudo systemctl daemon-reload", + "sudo systemctl restart containerd", +} + +var restartContainerdTests = []struct { + name string + cluster v1alpha1.ClusterSpec + want []string +}{ + { + name: "registry mirror and proxy config both exist", + cluster: v1alpha1.ClusterSpec{ + RegistryMirrorConfiguration: nil, + ProxyConfiguration: nil, + }, + want: []string{}, + }, + { + name: "registry mirror nil", + cluster: v1alpha1.ClusterSpec{ + RegistryMirrorConfiguration: nil, + ProxyConfiguration: &v1alpha1.ProxyConfiguration{ + HttpProxy: "1.2.3.4:8888", + HttpsProxy: "1.2.3.4:8888", + NoProxy: []string{ + "1.2.3.4/0", + }, + }, + }, + want: restartContainerdCommands, + }, + { + name: "proxy config nil", + cluster: v1alpha1.ClusterSpec{ + RegistryMirrorConfiguration: &v1alpha1.RegistryMirrorConfiguration{ + Endpoint: "1.2.3.4", + }, + ProxyConfiguration: nil, + }, + want: restartContainerdCommands, + }, +} + +func TestRestartContainerdInKubeadmControlPlane(t *testing.T) { + for _, tt := range restartContainerdTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + got := wantKubeadmControlPlane() + clusterapi.RestartContainerdInKubeadmControlPlane(got, tt.cluster) + want := wantKubeadmControlPlane() + want.Spec.KubeadmConfigSpec.PreKubeadmCommands = tt.want + g.Expect(got).To(Equal(want)) + }) + } +} + +func TestRestartContainerdInKubeadmConfigTemplate(t *testing.T) { + for _, tt := range restartContainerdTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + got := wantKubeadmConfigTemplate() + clusterapi.RestartContainerdInKubeadmConfigTemplate(got, tt.cluster) + want := wantKubeadmConfigTemplate() + want.Spec.Template.Spec.PreKubeadmCommands = tt.want + g.Expect(got).To(Equal(want)) + }) + } +} + +var buildContainerdConfigCommands = []string{ + "cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml", +} + +var createContainerdConfigTests = []struct { + name string + cluster v1alpha1.ClusterSpec + want []string +}{ + { + name: "registry mirror exists", + cluster: v1alpha1.ClusterSpec{ + RegistryMirrorConfiguration: &v1alpha1.RegistryMirrorConfiguration{ + Endpoint: "1.2.3.4", + }, + }, + want: buildContainerdConfigCommands, + }, + { + name: "registry mirror nil", + cluster: v1alpha1.ClusterSpec{ + RegistryMirrorConfiguration: nil, + }, + want: []string{}, + }, +} + +func TestCreateContainerdConfigFileInKubeadmControlPlane(t *testing.T) { + for _, tt := range createContainerdConfigTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + got := wantKubeadmControlPlane() + clusterapi.CreateContainerdConfigFileInKubeadmControlPlane(got, tt.cluster) + want := wantKubeadmControlPlane() + want.Spec.KubeadmConfigSpec.PreKubeadmCommands = tt.want + g.Expect(got).To(Equal(want)) + }) + } +} + +func TestCreateContainerdConfigFileInKubeadmConfigTemplate(t *testing.T) { + for _, tt := range createContainerdConfigTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + got := wantKubeadmConfigTemplate() + clusterapi.CreateContainerdConfigFileInKubeadmConfigTemplate(got, tt.cluster) + want := wantKubeadmConfigTemplate() + want.Spec.Template.Spec.PreKubeadmCommands = tt.want + g.Expect(got).To(Equal(want)) + }) + } +} diff --git a/pkg/providers/cloudstack/cloudstack.go b/pkg/providers/cloudstack/cloudstack.go index b8abc551fe18..57230257d86b 100644 --- a/pkg/providers/cloudstack/cloudstack.go +++ b/pkg/providers/cloudstack/cloudstack.go @@ -702,7 +702,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec, datacenterConfigSpec v1alpha1 noProxyList = append(noProxyList, clusterSpec.Cluster.Spec.ProxyConfiguration.NoProxy...) // Add no-proxy defaults - noProxyList = append(noProxyList, common.NoProxyDefaults...) + noProxyList = append(noProxyList, clusterapi.NoProxyDefaults()...) cloudStackManagementApiEndpointHostname, err := getHostnameFromUrl(datacenterConfigSpec.ManagementApiEndpoint) if err == nil { noProxyList = append(noProxyList, cloudStackManagementApiEndpointHostname) @@ -786,7 +786,7 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, datacenterConfigSpec v1alpha1 noProxyList = append(noProxyList, clusterSpec.Cluster.Spec.ProxyConfiguration.NoProxy...) // Add no-proxy defaults - noProxyList = append(noProxyList, common.NoProxyDefaults...) + noProxyList = append(noProxyList, clusterapi.NoProxyDefaults()...) cloudStackManagementApiEndpointHostname, err := getHostnameFromUrl(datacenterConfigSpec.ManagementApiEndpoint) if err == nil { noProxyList = append(noProxyList, cloudStackManagementApiEndpointHostname) diff --git a/pkg/providers/common/common.go b/pkg/providers/common/common.go index 2bfa37dcbde6..216c5bbf4120 100644 --- a/pkg/providers/common/common.go +++ b/pkg/providers/common/common.go @@ -19,12 +19,6 @@ import ( //go:embed config/audit-policy.yaml var auditPolicy string -var NoProxyDefaults = []string{ - "localhost", - "127.0.0.1", - ".svc", -} - // TODO: Split out common into separate packages to avoid becoming a dumping ground const ( diff --git a/pkg/providers/snow/apibuilder.go b/pkg/providers/snow/apibuilder.go index 5ee1f9af8db0..ddbee7b549d9 100644 --- a/pkg/providers/snow/apibuilder.go +++ b/pkg/providers/snow/apibuilder.go @@ -28,7 +28,7 @@ func CAPICluster(clusterSpec *cluster.Spec, snowCluster *snowv1.AWSSnowCluster, func KubeadmControlPlane(clusterSpec *cluster.Spec, snowMachineTemplate *snowv1.AWSSnowMachineTemplate) (*controlplanev1.KubeadmControlPlane, error) { kcp, err := clusterapi.KubeadmControlPlane(clusterSpec, snowMachineTemplate) if err != nil { - return nil, fmt.Errorf("failed generating KubeadmControlPlane: %v", err) + return nil, fmt.Errorf("generating KubeadmControlPlane: %v", err) } // TODO: support unstacked etcd @@ -50,13 +50,24 @@ func KubeadmControlPlane(clusterSpec *cluster.Spec, snowMachineTemplate *snowv1. fmt.Sprintf("/etc/eks/bootstrap-after.sh %s %s", clusterSpec.VersionsBundle.Snow.KubeVip.VersionedImage(), clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host), ) + if err := clusterapi.SetRegistryMirrorInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil { + return nil, err + } + + if err := clusterapi.SetProxyConfigInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec); err != nil { + return nil, err + } + + clusterapi.CreateContainerdConfigFileInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec) + clusterapi.RestartContainerdInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec) + return kcp, nil } func kubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig v1alpha1.WorkerNodeGroupConfiguration) (*bootstrapv1.KubeadmConfigTemplate, error) { kct, err := clusterapi.KubeadmConfigTemplate(clusterSpec, workerNodeGroupConfig) if err != nil { - return nil, fmt.Errorf("failed generating KubeadmConfigTemplate: %v", err) + return nil, fmt.Errorf("generating KubeadmConfigTemplate: %v", err) } joinConfigKubeletExtraArg := kct.Spec.Template.Spec.JoinConfiguration.NodeRegistration.KubeletExtraArgs @@ -65,6 +76,18 @@ func kubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig v1al kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, "/etc/eks/bootstrap.sh", ) + + if err := clusterapi.SetRegistryMirrorInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil { + return nil, err + } + + if err := clusterapi.SetProxyConfigInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec); err != nil { + return nil, err + } + + clusterapi.CreateContainerdConfigFileInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec) + clusterapi.RestartContainerdInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec) + return kct, nil } diff --git a/pkg/providers/snow/apibuilder_test.go b/pkg/providers/snow/apibuilder_test.go index 0cbf4f6eb544..45c808c170a4 100644 --- a/pkg/providers/snow/apibuilder_test.go +++ b/pkg/providers/snow/apibuilder_test.go @@ -177,54 +177,100 @@ func TestKubeadmControlPlane(t *testing.T) { tt.Expect(got).To(Equal(want)) } -func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) { - tests := []struct { - name string - registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration - wantFiles []bootstrapv1.File - }{ - { - name: "with ca cert", - registryMirrorConfig: &v1alpha1.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4", - Port: "443", - CACertContent: "xyz", - }, - wantFiles: []bootstrapv1.File{ - { - Path: "/etc/containerd/config_append.toml", - Owner: "root:root", - Content: `[plugins."io.containerd.grpc.v1.cri".registry.mirrors] +var registryMirrorTests = []struct { + name string + registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration + wantFiles []bootstrapv1.File +}{ + { + name: "with ca cert", + registryMirrorConfig: &v1alpha1.RegistryMirrorConfiguration{ + Endpoint: "1.2.3.4", + Port: "443", + CACertContent: "xyz", + }, + wantFiles: []bootstrapv1.File{ + { + Path: "/etc/containerd/config_append.toml", + Owner: "root:root", + Content: `[plugins."io.containerd.grpc.v1.cri".registry.mirrors] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] endpoint = ["https://1.2.3.4:443"] [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:443".tls] ca_file = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt"`, - }, - { - Path: "/etc/containerd/certs.d/1.2.3.4:443/ca.crt", - Owner: "root:root", - Content: "xyz", - }, + }, + { + Path: "/etc/containerd/certs.d/1.2.3.4:443/ca.crt", + Owner: "root:root", + Content: "xyz", }, }, - { - name: "without ca cert", - registryMirrorConfig: &v1alpha1.RegistryMirrorConfiguration{ - Endpoint: "1.2.3.4", - Port: "443", + }, + { + name: "with insecure skip", + registryMirrorConfig: &v1alpha1.RegistryMirrorConfiguration{ + Endpoint: "1.2.3.4", + Port: "443", + InsecureSkipVerify: true, + }, + wantFiles: []bootstrapv1.File{ + { + Path: "/etc/containerd/config_append.toml", + Owner: "root:root", + Content: `[plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:443"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:443".tls] + insecure_skip_verify = true`, }, - wantFiles: []bootstrapv1.File{ - { - Path: "/etc/containerd/config_append.toml", - Owner: "root:root", - Content: `[plugins."io.containerd.grpc.v1.cri".registry.mirrors] + }, + }, + { + name: "without ca cert", + registryMirrorConfig: &v1alpha1.RegistryMirrorConfiguration{ + Endpoint: "1.2.3.4", + Port: "443", + }, + wantFiles: []bootstrapv1.File{ + { + Path: "/etc/containerd/config_append.toml", + Owner: "root:root", + Content: `[plugins."io.containerd.grpc.v1.cri".registry.mirrors] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] endpoint = ["https://1.2.3.4:443"]`, - }, }, }, - } - for _, tt := range tests { + }, + { + name: "with ca cert and insecure skip", + registryMirrorConfig: &v1alpha1.RegistryMirrorConfiguration{ + Endpoint: "1.2.3.4", + Port: "443", + CACertContent: "xyz", + InsecureSkipVerify: true, + }, + wantFiles: []bootstrapv1.File{ + { + Path: "/etc/containerd/config_append.toml", + Owner: "root:root", + Content: `[plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:443"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:443".tls] + ca_file = "/etc/containerd/certs.d/1.2.3.4:443/ca.crt" + insecure_skip_verify = true`, + }, + { + Path: "/etc/containerd/certs.d/1.2.3.4:443/ca.crt", + Owner: "root:root", + Content: "xyz", + }, + }, + }, +} + +func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) { + for _, tt := range registryMirrorTests { t.Run(tt.name, func(t *testing.T) { g := newApiBuilerTest(t) g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig @@ -233,12 +279,63 @@ func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) { g.Expect(err).To(Succeed()) want := wantKubeadmControlPlane() want.Spec.KubeadmConfigSpec.Files = tt.wantFiles - want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(wantRegistryMirrorCommands(), want.Spec.KubeadmConfigSpec.PreKubeadmCommands...) + want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(want.Spec.KubeadmConfigSpec.PreKubeadmCommands, wantRegistryMirrorCommands()...) g.Expect(got).To(Equal(want)) }) } } +func wantProxyConfigCommands() []string { + return []string{ + "sudo systemctl daemon-reload", + "sudo systemctl restart containerd", + } +} + +var proxyTests = []struct { + name string + proxy *v1alpha1.ProxyConfiguration + wantFiles []bootstrapv1.File +}{ + { + name: "with proxy, pods cidr, service cidr, cp endpoint", + proxy: &v1alpha1.ProxyConfiguration{ + HttpProxy: "1.2.3.4:8888", + HttpsProxy: "1.2.3.4:8888", + NoProxy: []string{ + "1.2.3.4/0", + "1.2.3.5/0", + }, + }, + wantFiles: []bootstrapv1.File{ + { + Path: "/etc/systemd/system/containerd.service.d/http-proxy.conf", + Owner: "root:root", + Content: `[Service] +Environment="HTTP_PROXY=1.2.3.4:8888" +Environment="HTTPS_PROXY=1.2.3.4:8888" +Environment="NO_PROXY=10.1.0.0/16,10.96.0.0/12,1.2.3.4/0,1.2.3.5/0,localhost,127.0.0.1,.svc,1.2.3.4"`, + }, + }, + }, +} + +func TestKubeadmControlPlaneWithProxyConfig(t *testing.T) { + for _, tt := range proxyTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy + controlPlaneMachineTemplate := snow.SnowMachineTemplate(g.machineConfigs[g.clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name]) + got, err := snow.KubeadmControlPlane(g.clusterSpec, controlPlaneMachineTemplate) + g.Expect(err).To(Succeed()) + want := wantKubeadmControlPlane() + want.Spec.KubeadmConfigSpec.Files = tt.wantFiles + want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(want.Spec.KubeadmConfigSpec.PreKubeadmCommands, wantProxyConfigCommands()...) + g.Expect(got.Spec.KubeadmConfigSpec.PreKubeadmCommands).To(Equal(want.Spec.KubeadmConfigSpec.PreKubeadmCommands)) + }) + } +} + func wantKubeadmConfigTemplate() *bootstrapv1.KubeadmConfigTemplate { return &bootstrapv1.KubeadmConfigTemplate{ TypeMeta: metav1.TypeMeta{ @@ -291,6 +388,40 @@ func TestKubeadmConfigTemplates(t *testing.T) { tt.Expect(got).To(Equal(want)) } +func TestKubeadmConfigTemplatesWithRegistryMirror(t *testing.T) { + for _, tt := range registryMirrorTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig + got, err := snow.KubeadmConfigTemplates(g.clusterSpec) + g.Expect(err).To(Succeed()) + want := map[string]*bootstrapv1.KubeadmConfigTemplate{ + "md-0": wantKubeadmConfigTemplate(), + } + want["md-0"].Spec.Template.Spec.Files = tt.wantFiles + want["md-0"].Spec.Template.Spec.PreKubeadmCommands = append(want["md-0"].Spec.Template.Spec.PreKubeadmCommands, wantRegistryMirrorCommands()...) + g.Expect(got).To(Equal(want)) + }) + } +} + +func TestKubeadmConfigTemplatesWithProxyConfig(t *testing.T) { + for _, tt := range proxyTests { + t.Run(tt.name, func(t *testing.T) { + g := newApiBuilerTest(t) + g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy + got, err := snow.KubeadmConfigTemplates(g.clusterSpec) + g.Expect(err).To(Succeed()) + want := map[string]*bootstrapv1.KubeadmConfigTemplate{ + "md-0": wantKubeadmConfigTemplate(), + } + want["md-0"].Spec.Template.Spec.Files = tt.wantFiles + want["md-0"].Spec.Template.Spec.PreKubeadmCommands = append(want["md-0"].Spec.Template.Spec.PreKubeadmCommands, wantProxyConfigCommands()...) + g.Expect(got).To(Equal(want)) + }) + } +} + func wantMachineDeployment() *clusterv1.MachineDeployment { wantVersion := "v1.21.5-eks-1-21-9" wantReplicas := int32(3) diff --git a/pkg/providers/vsphere/vsphere.go b/pkg/providers/vsphere/vsphere.go index 55d2bd58bb63..f127093e67e9 100644 --- a/pkg/providers/vsphere/vsphere.go +++ b/pkg/providers/vsphere/vsphere.go @@ -785,7 +785,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec, datacenterSpec v1alpha1.VSphe noProxyList = append(noProxyList, clusterSpec.Cluster.Spec.ProxyConfiguration.NoProxy...) // Add no-proxy defaults - noProxyList = append(noProxyList, common.NoProxyDefaults...) + noProxyList = append(noProxyList, clusterapi.NoProxyDefaults()...) noProxyList = append(noProxyList, datacenterSpec.Server, clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host, @@ -878,7 +878,7 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, datacenterSpec v1alpha1.VSphe noProxyList = append(noProxyList, clusterSpec.Cluster.Spec.ProxyConfiguration.NoProxy...) // Add no-proxy defaults - noProxyList = append(noProxyList, common.NoProxyDefaults...) + noProxyList = append(noProxyList, clusterapi.NoProxyDefaults()...) noProxyList = append(noProxyList, datacenterSpec.Server, clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host, diff --git a/test/e2e/proxy_test.go b/test/e2e/proxy_test.go index 0d3b10a14e87..8110696609dc 100644 --- a/test/e2e/proxy_test.go +++ b/test/e2e/proxy_test.go @@ -57,3 +57,16 @@ func TestCloudStackKubernetes121RedhatProxyConfig(t *testing.T) { ) runProxyConfigFlow(test) } + +func TestSnowKubernetes121UbuntuProxyConfig(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewSnow(t, framework.WithSnowUbuntu121()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), + framework.WithProxy(), + framework.WithEnvVar("SNOW_PROVIDER", "true"), + ) + runProxyConfigFlow(test) +}