Add proxy config setup when generating CAPI objects in apibuilder

pkg/clusterapi/apibuilder.go

@@ -162,10 +162,6 @@ func KubeadmControlPlane(clusterSpec *cluster.Spec, infrastructureObject APIObje
 		},
 	}
 
-	if err := SetRegistryMirrorInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil {
-		return nil, err
-	}
-
 	SetIdentityAuthInKubeadmControlPlane(kcp, clusterSpec)
 
 	return kcp, nil
@@ -207,10 +203,6 @@ func KubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig v1al
 		},
 	}
 
-	if err := SetRegistryMirrorInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil {
-		return nil, err
-	}
-
 	return kct, nil
 }
 

pkg/clusterapi/apibuilder_test.go

@@ -65,6 +65,9 @@ func newApiBuilerTest(t *testing.T) apiBuilerTest {
 					},
 				},
 				ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
+					Endpoint: &v1alpha1.Endpoint{
+						Host: "1.2.3.4",
+					},
 					Count: 3,
 				},
 				KubernetesVersion: "1.21",
@@ -254,21 +257,6 @@ func TestKubeadmControlPlane(t *testing.T) {
 	tt.Expect(got).To(Equal(want))
 }
 
-func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) {
-	for _, tt := range registryMirrorTests {
-		t.Run(tt.name, func(t *testing.T) {
-			g := newApiBuilerTest(t)
-			g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig
-			got, err := clusterapi.KubeadmControlPlane(g.clusterSpec, g.providerMachineTemplate)
-			g.Expect(err).To(Succeed())
-			want := wantKubeadmControlPlane()
-			want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
-			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands()
-			g.Expect(got).To(Equal(want))
-		})
-	}
-}
-
 func wantKubeadmConfigTemplate() *bootstrapv1.KubeadmConfigTemplate {
 	return &bootstrapv1.KubeadmConfigTemplate{
 		TypeMeta: metav1.TypeMeta{
@@ -314,21 +302,6 @@ func TestKubeadmConfigTemplate(t *testing.T) {
 	tt.Expect(got).To(Equal(want))
 }
 
-func TestKubeadmConfigTemplateWithRegistryMirror(t *testing.T) {
-	for _, tt := range registryMirrorTests {
-		t.Run(tt.name, func(t *testing.T) {
-			g := newApiBuilerTest(t)
-			g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig
-			got, err := clusterapi.KubeadmConfigTemplate(g.clusterSpec, *g.workerNodeGroupConfig)
-			g.Expect(err).To(Succeed())
-			want := wantKubeadmConfigTemplate()
-			want.Spec.Template.Spec.Files = tt.wantFiles
-			want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands()
-			g.Expect(got).To(Equal(want))
-		})
-	}
-}
-
 func TestMachineDeployment(t *testing.T) {
 	tt := newApiBuilerTest(t)
 	got := clusterapi.MachineDeployment(tt.clusterSpec, *tt.workerNodeGroupConfig, tt.kubeadmConfigTemplate, tt.providerMachineTemplate)

pkg/clusterapi/config/http-proxy.conf

@@ -0,0 +1,4 @@
+[Service]
+Environment="HTTP_PROXY={{.httpProxy}}"
+Environment="HTTPS_PROXY={{.httpsProxy}}"
+Environment="NO_PROXY={{ stringsJoin .noProxy "," }}"

pkg/clusterapi/proxy.go

@@ -0,0 +1,93 @@
+package clusterapi
+
+import (
+	_ "embed"
+	"fmt"
+
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/templater"
+)
+
+//go:embed config/http-proxy.conf
+var proxyConfig string
+
+func NoProxyDefaults() []string {
+	return []string{
+		"localhost",
+		"127.0.0.1",
+		".svc",
+	}
+}
+
+func proxyConfigContent(cluster v1alpha1.ClusterSpec) (string, error) {
+	capacity := len(cluster.ClusterNetwork.Pods.CidrBlocks) +
+		len(cluster.ClusterNetwork.Services.CidrBlocks) +
+		len(cluster.ProxyConfiguration.NoProxy) + 4
+
+	noProxyList := make([]string, 0, capacity)
+	noProxyList = append(noProxyList, cluster.ClusterNetwork.Pods.CidrBlocks...)
+	noProxyList = append(noProxyList, cluster.ClusterNetwork.Services.CidrBlocks...)
+	noProxyList = append(noProxyList, cluster.ProxyConfiguration.NoProxy...)
+
+	// Add no-proxy defaults
+	noProxyList = append(noProxyList, NoProxyDefaults()...)
+	noProxyList = append(noProxyList, cluster.ControlPlaneConfiguration.Endpoint.Host)
+
+	val := values{
+		"httpProxy":  cluster.ProxyConfiguration.HttpProxy,
+		"httpsProxy": cluster.ProxyConfiguration.HttpsProxy,
+		"noProxy":    noProxyList,
+	}
+
+	config, err := templater.Execute(proxyConfig, val)
+	if err != nil {
+		return "", fmt.Errorf("building http-proxy.conf file: %v", err)
+	}
+	return string(config), nil
+}
+
+func proxyConfigFile(cluster v1alpha1.ClusterSpec) (bootstrapv1.File, error) {
+	proxyConfig, err := proxyConfigContent(cluster)
+	if err != nil {
+		return bootstrapv1.File{}, err
+	}
+
+	return bootstrapv1.File{
+		Path:    "/etc/systemd/system/containerd.service.d/http-proxy.conf",
+		Owner:   "root:root",
+		Content: proxyConfig,
+	}, nil
+}
+
+func SetProxyConfigInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) error {
+	if cluster.ProxyConfiguration == nil {
+		return nil
+	}
+
+	proxyConfigFile, err := proxyConfigFile(cluster)
+	if err != nil {
+		return err
+	}
+
+	kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, proxyConfigFile)
+
+	return nil
+}
+
+func SetProxyConfigInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) error {
+	if cluster.ProxyConfiguration == nil {
+		return nil
+	}
+
+	proxyConfigFile, err := proxyConfigFile(cluster)
+	if err != nil {
+		return err
+	}
+
+	kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, proxyConfigFile)
+
+	return nil
+}

pkg/clusterapi/proxy_test.go

@@ -0,0 +1,82 @@
+package clusterapi_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/clusterapi"
+)
+
+var proxyTests = []struct {
+	name      string
+	proxy     *v1alpha1.ProxyConfiguration
+	wantFiles []bootstrapv1.File
+}{
+	{
+		name:      "proxy config nil",
+		proxy:     nil,
+		wantFiles: []bootstrapv1.File{},
+	},
+	{
+		name: "with proxy, pods cidr, service cidr, cp endpoint",
+		proxy: &v1alpha1.ProxyConfiguration{
+			HttpProxy:  "1.2.3.4:8888",
+			HttpsProxy: "1.2.3.4:8888",
+			NoProxy: []string{
+				"1.2.3.4/0",
+				"1.2.3.5/0",
+			},
+		},
+		wantFiles: []bootstrapv1.File{
+			{
+				Path:  "/etc/systemd/system/containerd.service.d/http-proxy.conf",
+				Owner: "root:root",
+				Content: `[Service]
+Environment="HTTP_PROXY=1.2.3.4:8888"
+Environment="HTTPS_PROXY=1.2.3.4:8888"
+Environment="NO_PROXY=1.2.3.4/5,1.2.3.4/5,1.2.3.4/0,1.2.3.5/0,localhost,127.0.0.1,.svc,1.2.3.4"`,
+			},
+		},
+	},
+}
+
+func TestSetProxyConfigInKubeadmControlPlane(t *testing.T) {
+	for _, tt := range proxyTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			got := wantKubeadmControlPlane()
+			g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
+			g.Expect(clusterapi.SetProxyConfigInKubeadmControlPlane(got, g.clusterSpec.Cluster.Spec)).To(Succeed())
+			want := wantKubeadmControlPlane()
+			want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}
+
+func TestSetProxyConfigInKubeadmConfigTemplate(t *testing.T) {
+	for _, tt := range proxyTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			got := wantKubeadmConfigTemplate()
+			g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
+			g.Expect(clusterapi.SetProxyConfigInKubeadmConfigTemplate(got, g.clusterSpec.Cluster.Spec)).To(Succeed())
+			want := wantKubeadmConfigTemplate()
+			want.Spec.Template.Spec.Files = tt.wantFiles
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}
+
+func TestNoProxyDefaults(t *testing.T) {
+	g := NewWithT(t)
+	want := []string{
+		"localhost",
+		"127.0.0.1",
+		".svc",
+	}
+	g.Expect(clusterapi.NoProxyDefaults()).To(Equal(want))
+}

pkg/clusterapi/registryMirror.go → pkg/clusterapi/registry_mirror.go

@@ -26,16 +26,16 @@ func registryMirrorConfigContent(registryAddress, registryCert string, insecureS
 
 	config, err := templater.Execute(containerdConfig, val)
 	if err != nil {
-		return "", fmt.Errorf("failed building containerd config file: %v", err)
+		return "", fmt.Errorf("building containerd config file: %v", err)
 	}
 	return string(config), nil
 }
 
-func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration) (files []bootstrapv1.File, preKubeadmCommands []string, err error) {
+func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration) (files []bootstrapv1.File, err error) {
 	registryAddress := net.JoinHostPort(registryMirrorConfig.Endpoint, registryMirrorConfig.Port)
 	registryConfig, err := registryMirrorConfigContent(registryAddress, registryMirrorConfig.CACertContent, registryMirrorConfig.InsecureSkipVerify)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	files = []bootstrapv1.File{
 		{
@@ -53,26 +53,20 @@ func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfigura
 		})
 	}
 
-	preKubeadmCommands = []string{
-		"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
-		"sudo systemctl daemon-reload",
-		"sudo systemctl restart containerd",
-	}
-	return files, preKubeadmCommands, nil
+	return files, nil
 }
 
 func SetRegistryMirrorInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, mirrorConfig *v1alpha1.RegistryMirrorConfiguration) error {
 	if mirrorConfig == nil {
 		return nil
 	}
 
-	containerdFiles, containerdCommands, err := registryMirrorConfig(mirrorConfig)
+	containerdFiles, err := registryMirrorConfig(mirrorConfig)
 	if err != nil {
 		return fmt.Errorf("setting registry mirror configuration: %v", err)
 	}
 
 	kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, containerdFiles...)
-	kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, containerdCommands...)
 
 	return nil
 }
@@ -82,13 +76,12 @@ func SetRegistryMirrorInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemp
 		return nil
 	}
 
-	containerdFiles, containerdCommands, err := registryMirrorConfig(mirrorConfig)
+	containerdFiles, err := registryMirrorConfig(mirrorConfig)
 	if err != nil {
 		return fmt.Errorf("setting registry mirror configuration: %v", err)
 	}
 
 	kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, containerdFiles...)
-	kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, containerdCommands...)
 
 	return nil
 }

pkg/clusterapi/registryMirror_test.go → pkg/clusterapi/registry_mirror_test.go

@@ -86,14 +86,6 @@ var registryMirrorTests = []struct {
 	},
 }
 
-func wantRegistryMirrorCommands() []string {
-	return []string{
-		"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
-		"sudo systemctl daemon-reload",
-		"sudo systemctl restart containerd",
-	}
-}
-
 func TestSetRegistryMirrorInKubeadmControlPlane(t *testing.T) {
 	for _, tt := range registryMirrorTests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -102,7 +94,6 @@ func TestSetRegistryMirrorInKubeadmControlPlane(t *testing.T) {
 			g.Expect(clusterapi.SetRegistryMirrorInKubeadmControlPlane(got, tt.registryMirrorConfig)).To(Succeed())
 			want := wantKubeadmControlPlane()
 			want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
-			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands()
 			g.Expect(got).To(Equal(want))
 		})
 	}
@@ -116,7 +107,6 @@ func TestSetRegistryMirrorInKubeadmConfigTemplate(t *testing.T) {
 			g.Expect(clusterapi.SetRegistryMirrorInKubeadmConfigTemplate(got, tt.registryMirrorConfig)).To(Succeed())
 			want := wantKubeadmConfigTemplate()
 			want.Spec.Template.Spec.Files = tt.wantFiles
-			want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands()
 			g.Expect(got).To(Equal(want))
 		})
 	}

pkg/clusterapi/systemctl.go

@@ -0,0 +1,45 @@
+package clusterapi
+
+import (
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+)
+
+var buildContainerdConfigCommands = []string{
+	"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
+}
+
+var restartContainerdCommands = []string{
+	"sudo systemctl daemon-reload",
+	"sudo systemctl restart containerd",
+}
+
+func CreateContainerdConfigFileInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) {
+	if cluster.RegistryMirrorConfiguration != nil {
+		kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, buildContainerdConfigCommands...)
+	}
+}
+
+func CreateContainerdConfigFileInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) {
+	if cluster.RegistryMirrorConfiguration != nil {
+		kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, buildContainerdConfigCommands...)
+	}
+}
+
+func RestartContainerdInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) {
+	if restartContainerdNeeded(cluster) {
+		kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, restartContainerdCommands...)
+	}
+}
+
+func RestartContainerdInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) {
+	if restartContainerdNeeded(cluster) {
+		kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, restartContainerdCommands...)
+	}
+}
+
+func restartContainerdNeeded(cluster v1alpha1.ClusterSpec) bool {
+	return cluster.RegistryMirrorConfiguration != nil || cluster.ProxyConfiguration != nil
+}