Add proxy config setup when generating CAPI objects in apibuilder

cmd/integration_test/build/buildspecs/test-eks-a-cli.yml

@@ -5,7 +5,7 @@ env:
     INTEGRATION_TEST_MAX_EC2_COUNT: 120
     INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 120
     T_CLOUDSTACK_CIDR: "10.11.101.0/16"
-    SKIPPED_TESTS: "TestVSphereKubernetes122BottlerocketAutoimport,TestTinkerbellKubernetes121ForceFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersForceFlow,TestTinkerbellKubernetes122ForceFlow,TestTinkerbellKubernetes120SimpleFlow,TestTinkerbellKubernetes121SimpleFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersSimpleFlow,TestTinkerbellKubernetes121DellSimpleFlow,TestTinkerbellKubernetes121HPSimpleFlow,TestTinkerbellKubernetes121SuperMicroSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdThreeReplicasTwoWorkersSimpleFlow,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC"
+    SKIPPED_TESTS: "TestVSphereKubernetes122BottlerocketAutoimport,TestTinkerbellKubernetes121ForceFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersForceFlow,TestTinkerbellKubernetes122ForceFlow,TestTinkerbellKubernetes120SimpleFlow,TestTinkerbellKubernetes121SimpleFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersSimpleFlow,TestTinkerbellKubernetes121DellSimpleFlow,TestTinkerbellKubernetes121HPSimpleFlow,TestTinkerbellKubernetes121SuperMicroSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdThreeReplicasTwoWorkersSimpleFlow,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig"
     CLOUDSTACK_PROVIDER: true
     EKSA_GIT_KNOWN_HOSTS: "/tmp/known_hosts"
     EKSA_GIT_PRIVATE_KEY: "/tmp/private-key"

pkg/clusterapi/apibuilder.go

@@ -166,8 +166,14 @@ func KubeadmControlPlane(clusterSpec *cluster.Spec, infrastructureObject APIObje
 		return nil, err
 	}
 
+	if err := SetProxyConfigInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec); err != nil {
+		return nil, err
+	}
+
 	SetIdentityAuthInKubeadmControlPlane(kcp, clusterSpec)
 
+	RestartContainerdInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec)
+
 	return kcp, nil
 }
 
@@ -211,6 +217,12 @@ func KubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig v1al
 		return nil, err
 	}
 
+	if err := SetProxyConfigInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec); err != nil {
+		return nil, err
+	}
+
+	RestartContainerdInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec)
+
 	return kct, nil
 }
 

pkg/clusterapi/apibuilder_test.go

@@ -17,6 +17,11 @@ import (
 	"github.com/aws/eks-anywhere/pkg/clusterapi"
 )
 
+var restartContainerdCommands = []string{
+	"sudo systemctl daemon-reload",
+	"sudo systemctl restart containerd",
+}
+
 type apiBuilerTest struct {
 	*WithT
 	clusterSpec             *cluster.Spec
@@ -65,6 +70,9 @@ func newApiBuilerTest(t *testing.T) apiBuilerTest {
 					},
 				},
 				ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
+					Endpoint: &v1alpha1.Endpoint{
+						Host: "1.2.3.4",
+					},
 					Count: 3,
 				},
 				KubernetesVersion: "1.21",
@@ -263,7 +271,22 @@ func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) {
 			g.Expect(err).To(Succeed())
 			want := wantKubeadmControlPlane()
 			want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
-			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands()
+			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(wantRegistryMirrorCommands(), restartContainerdCommands...)
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}
+
+func TestKubeadmControlPlaneWithProxyConfig(t *testing.T) {
+	for _, tt := range proxyTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
+			got, err := clusterapi.KubeadmControlPlane(g.clusterSpec, g.providerMachineTemplate)
+			g.Expect(err).To(Succeed())
+			want := wantKubeadmControlPlane()
+			want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
+			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = tt.wantCmd
 			g.Expect(got).To(Equal(want))
 		})
 	}
@@ -323,7 +346,7 @@ func TestKubeadmConfigTemplateWithRegistryMirror(t *testing.T) {
 			g.Expect(err).To(Succeed())
 			want := wantKubeadmConfigTemplate()
 			want.Spec.Template.Spec.Files = tt.wantFiles
-			want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands()
+			want.Spec.Template.Spec.PreKubeadmCommands = append(wantRegistryMirrorCommands(), restartContainerdCommands...)
 			g.Expect(got).To(Equal(want))
 		})
 	}

pkg/clusterapi/config/http-proxy.conf

@@ -0,0 +1,4 @@
+[Service]
+Environment="HTTP_PROXY={{.httpProxy}}"
+Environment="HTTPS_PROXY={{.httpsProxy}}"
+Environment="NO_PROXY={{ stringsJoin .noProxy "," }}"

pkg/clusterapi/kubeadm_commands.go

@@ -0,0 +1,29 @@
+package clusterapi
+
+import (
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+)
+
+var restartContainerdCommands = []string{
+	"sudo systemctl daemon-reload",
+	"sudo systemctl restart containerd",
+}
+
+func RestartContainerdInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) {
+	if restartContainerdNeeded(cluster) {
+		kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, restartContainerdCommands...)
+	}
+}
+
+func RestartContainerdInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) {
+	if restartContainerdNeeded(cluster) {
+		kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, restartContainerdCommands...)
+	}
+}
+
+func restartContainerdNeeded(cluster v1alpha1.ClusterSpec) bool {
+	return cluster.RegistryMirrorConfiguration != nil || cluster.ProxyConfiguration != nil
+}

pkg/clusterapi/kubeadm_commands_test.go

@@ -0,0 +1,75 @@
+package clusterapi_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/clusterapi"
+)
+
+var kubeadmCommandsTests = []struct {
+	name    string
+	cluster v1alpha1.ClusterSpec
+	want    []string
+}{
+	{
+		name: "registry mirror and proxy config both exist",
+		cluster: v1alpha1.ClusterSpec{
+			RegistryMirrorConfiguration: nil,
+			ProxyConfiguration:          nil,
+		},
+		want: []string{},
+	},
+	{
+		name: "registry mirror nil",
+		cluster: v1alpha1.ClusterSpec{
+			RegistryMirrorConfiguration: nil,
+			ProxyConfiguration: &v1alpha1.ProxyConfiguration{
+				HttpProxy:  "1.2.3.4:8888",
+				HttpsProxy: "1.2.3.4:8888",
+				NoProxy: []string{
+					"1.2.3.4/0",
+				},
+			},
+		},
+		want: restartContainerdCommands,
+	},
+	{
+		name: "proxy config nil",
+		cluster: v1alpha1.ClusterSpec{
+			RegistryMirrorConfiguration: &v1alpha1.RegistryMirrorConfiguration{
+				Endpoint: "1.2.3.4",
+			},
+			ProxyConfiguration: nil,
+		},
+		want: restartContainerdCommands,
+	},
+}
+
+func TestRestartContainerdInKubeadmControlPlane(t *testing.T) {
+	for _, tt := range kubeadmCommandsTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			got := wantKubeadmControlPlane()
+			clusterapi.RestartContainerdInKubeadmControlPlane(got, tt.cluster)
+			want := wantKubeadmControlPlane()
+			want.Spec.KubeadmConfigSpec.PreKubeadmCommands = tt.want
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}
+
+func TestRestartContainerdInKubeadmConfigTemplate(t *testing.T) {
+	for _, tt := range kubeadmCommandsTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			got := wantKubeadmConfigTemplate()
+			clusterapi.RestartContainerdInKubeadmConfigTemplate(got, tt.cluster)
+			want := wantKubeadmConfigTemplate()
+			want.Spec.Template.Spec.PreKubeadmCommands = tt.want
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}

pkg/clusterapi/proxy.go

@@ -0,0 +1,91 @@
+package clusterapi
+
+import (
+	_ "embed"
+	"fmt"
+
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/templater"
+)
+
+//go:embed config/http-proxy.conf
+var proxyConfig string
+
+var NoProxyDefaults = []string{
+	"localhost",
+	"127.0.0.1",
+	".svc",
+}
+
+func proxyConfigContent(cluster v1alpha1.ClusterSpec) (string, error) {
+	capacity := len(cluster.ClusterNetwork.Pods.CidrBlocks) +
+		len(cluster.ClusterNetwork.Services.CidrBlocks) +
+		len(cluster.ProxyConfiguration.NoProxy) + 4
+
+	noProxyList := make([]string, 0, capacity)
+	noProxyList = append(noProxyList, cluster.ClusterNetwork.Pods.CidrBlocks...)
+	noProxyList = append(noProxyList, cluster.ClusterNetwork.Services.CidrBlocks...)
+	noProxyList = append(noProxyList, cluster.ProxyConfiguration.NoProxy...)
+
+	// Add no-proxy defaults
+	noProxyList = append(noProxyList, NoProxyDefaults...)
+	noProxyList = append(noProxyList, cluster.ControlPlaneConfiguration.Endpoint.Host)
+
+	val := values{
+		"httpProxy":  cluster.ProxyConfiguration.HttpProxy,
+		"httpsProxy": cluster.ProxyConfiguration.HttpsProxy,
+		"noProxy":    noProxyList,
+	}
+
+	config, err := templater.Execute(proxyConfig, val)
+	if err != nil {
+		return "", fmt.Errorf("building http-proxy.conf file: %v", err)
+	}
+	return string(config), nil
+}
+
+func proxyConfigFile(cluster v1alpha1.ClusterSpec) (bootstrapv1.File, error) {
+	proxyConfig, err := proxyConfigContent(cluster)
+	if err != nil {
+		return bootstrapv1.File{}, err
+	}
+
+	return bootstrapv1.File{
+		Path:    "/etc/systemd/system/containerd.service.d/http-proxy.conf",
+		Owner:   "root:root",
+		Content: proxyConfig,
+	}, nil
+}
+
+func SetProxyConfigInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) error {
+	if cluster.ProxyConfiguration == nil {
+		return nil
+	}
+
+	proxyConfigFile, err := proxyConfigFile(cluster)
+	if err != nil {
+		return err
+	}
+
+	kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, proxyConfigFile)
+
+	return nil
+}
+
+func SetProxyConfigInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) error {
+	if cluster.ProxyConfiguration == nil {
+		return nil
+	}
+
+	proxyConfigFile, err := proxyConfigFile(cluster)
+	if err != nil {
+		return err
+	}
+
+	kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, proxyConfigFile)
+
+	return nil
+}

pkg/clusterapi/proxy_test.go

@@ -0,0 +1,75 @@
+package clusterapi_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+
+	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/clusterapi"
+)
+
+var proxyTests = []struct {
+	name      string
+	proxy     *v1alpha1.ProxyConfiguration
+	wantFiles []bootstrapv1.File
+	wantCmd   []string
+}{
+	{
+		name:      "proxy config nil",
+		proxy:     nil,
+		wantFiles: []bootstrapv1.File{},
+		wantCmd:   []string{},
+	},
+	{
+		name: "with proxy, pods cidr, service cidr, cp endpoint",
+		proxy: &v1alpha1.ProxyConfiguration{
+			HttpProxy:  "1.2.3.4:8888",
+			HttpsProxy: "1.2.3.4:8888",
+			NoProxy: []string{
+				"1.2.3.4/0",
+				"1.2.3.5/0",
+			},
+		},
+		wantFiles: []bootstrapv1.File{
+			{
+				Path:  "/etc/systemd/system/containerd.service.d/http-proxy.conf",
+				Owner: "root:root",
+				Content: `[Service]
+Environment="HTTP_PROXY=1.2.3.4:8888"
+Environment="HTTPS_PROXY=1.2.3.4:8888"
+Environment="NO_PROXY=1.2.3.4/5,1.2.3.4/5,1.2.3.4/0,1.2.3.5/0,localhost,127.0.0.1,.svc,1.2.3.4"`,
+			},
+		},
+		wantCmd: restartContainerdCommands,
+	},
+}
+
+func TestSetProxyConfigInKubeadmControlPlane(t *testing.T) {
+	for _, tt := range proxyTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			got := wantKubeadmControlPlane()
+			g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
+			g.Expect(clusterapi.SetProxyConfigInKubeadmControlPlane(got, g.clusterSpec.Cluster.Spec)).To(Succeed())
+			want := wantKubeadmControlPlane()
+			want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}
+
+func TestSetProxyConfigInKubeadmConfigTemplate(t *testing.T) {
+	for _, tt := range proxyTests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := newApiBuilerTest(t)
+			got := wantKubeadmConfigTemplate()
+			g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
+			g.Expect(clusterapi.SetProxyConfigInKubeadmConfigTemplate(got, g.clusterSpec.Cluster.Spec)).To(Succeed())
+			want := wantKubeadmConfigTemplate()
+			want.Spec.Template.Spec.Files = tt.wantFiles
+			g.Expect(got).To(Equal(want))
+		})
+	}
+}

pkg/clusterapi/registryMirror.go → pkg/clusterapi/registry_mirror.go

@@ -26,7 +26,7 @@ func registryMirrorConfigContent(registryAddress, registryCert string, insecureS
 
 	config, err := templater.Execute(containerdConfig, val)
 	if err != nil {
-		return "", fmt.Errorf("failed building containerd config file: %v", err)
+		return "", fmt.Errorf("building containerd config file: %v", err)
 	}
 	return string(config), nil
 }
@@ -55,8 +55,6 @@ func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfigura
 
 	preKubeadmCommands = []string{
 		"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
-		"sudo systemctl daemon-reload",
-		"sudo systemctl restart containerd",
 	}
 	return files, preKubeadmCommands, nil
 }

pkg/clusterapi/registryMirror_test.go → pkg/clusterapi/registry_mirror_test.go

@@ -89,8 +89,6 @@ var registryMirrorTests = []struct {
 func wantRegistryMirrorCommands() []string {
 	return []string{
 		"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
-		"sudo systemctl daemon-reload",
-		"sudo systemctl restart containerd",
 	}
 }