Skip to content

Commit

Permalink
Add proxy config setup when generating CAPI objects in apibuilder (#2262
Browse files Browse the repository at this point in the history
)

* Add http proxy support in apibuilder

* Make NoProxyDefaults func

* Move OS specific cmds to snow provider
  • Loading branch information
jiayiwang7 authored Jun 1, 2022
1 parent adf3fe1 commit ccc024f
Show file tree
Hide file tree
Showing 15 changed files with 577 additions and 111 deletions.
8 changes: 0 additions & 8 deletions pkg/clusterapi/apibuilder.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,10 +162,6 @@ func KubeadmControlPlane(clusterSpec *cluster.Spec, infrastructureObject APIObje
},
}

if err := SetRegistryMirrorInKubeadmControlPlane(kcp, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil {
return nil, err
}

SetIdentityAuthInKubeadmControlPlane(kcp, clusterSpec)

return kcp, nil
Expand Down Expand Up @@ -207,10 +203,6 @@ func KubeadmConfigTemplate(clusterSpec *cluster.Spec, workerNodeGroupConfig v1al
},
}

if err := SetRegistryMirrorInKubeadmConfigTemplate(kct, clusterSpec.Cluster.Spec.RegistryMirrorConfiguration); err != nil {
return nil, err
}

return kct, nil
}

Expand Down
33 changes: 3 additions & 30 deletions pkg/clusterapi/apibuilder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ func newApiBuilerTest(t *testing.T) apiBuilerTest {
},
},
ControlPlaneConfiguration: v1alpha1.ControlPlaneConfiguration{
Endpoint: &v1alpha1.Endpoint{
Host: "1.2.3.4",
},
Count: 3,
},
KubernetesVersion: "1.21",
Expand Down Expand Up @@ -254,21 +257,6 @@ func TestKubeadmControlPlane(t *testing.T) {
tt.Expect(got).To(Equal(want))
}

func TestKubeadmControlPlaneWithRegistryMirror(t *testing.T) {
for _, tt := range registryMirrorTests {
t.Run(tt.name, func(t *testing.T) {
g := newApiBuilerTest(t)
g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig
got, err := clusterapi.KubeadmControlPlane(g.clusterSpec, g.providerMachineTemplate)
g.Expect(err).To(Succeed())
want := wantKubeadmControlPlane()
want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands()
g.Expect(got).To(Equal(want))
})
}
}

func wantKubeadmConfigTemplate() *bootstrapv1.KubeadmConfigTemplate {
return &bootstrapv1.KubeadmConfigTemplate{
TypeMeta: metav1.TypeMeta{
Expand Down Expand Up @@ -314,21 +302,6 @@ func TestKubeadmConfigTemplate(t *testing.T) {
tt.Expect(got).To(Equal(want))
}

func TestKubeadmConfigTemplateWithRegistryMirror(t *testing.T) {
for _, tt := range registryMirrorTests {
t.Run(tt.name, func(t *testing.T) {
g := newApiBuilerTest(t)
g.clusterSpec.Cluster.Spec.RegistryMirrorConfiguration = tt.registryMirrorConfig
got, err := clusterapi.KubeadmConfigTemplate(g.clusterSpec, *g.workerNodeGroupConfig)
g.Expect(err).To(Succeed())
want := wantKubeadmConfigTemplate()
want.Spec.Template.Spec.Files = tt.wantFiles
want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands()
g.Expect(got).To(Equal(want))
})
}
}

func TestMachineDeployment(t *testing.T) {
tt := newApiBuilerTest(t)
got := clusterapi.MachineDeployment(tt.clusterSpec, *tt.workerNodeGroupConfig, tt.kubeadmConfigTemplate, tt.providerMachineTemplate)
Expand Down
4 changes: 4 additions & 0 deletions pkg/clusterapi/config/http-proxy.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[Service]
Environment="HTTP_PROXY={{.httpProxy}}"
Environment="HTTPS_PROXY={{.httpsProxy}}"
Environment="NO_PROXY={{ stringsJoin .noProxy "," }}"
93 changes: 93 additions & 0 deletions pkg/clusterapi/proxy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package clusterapi

import (
_ "embed"
"fmt"

bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"

"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/templater"
)

//go:embed config/http-proxy.conf
var proxyConfig string

func NoProxyDefaults() []string {
return []string{
"localhost",
"127.0.0.1",
".svc",
}
}

func proxyConfigContent(cluster v1alpha1.ClusterSpec) (string, error) {
capacity := len(cluster.ClusterNetwork.Pods.CidrBlocks) +
len(cluster.ClusterNetwork.Services.CidrBlocks) +
len(cluster.ProxyConfiguration.NoProxy) + 4

noProxyList := make([]string, 0, capacity)
noProxyList = append(noProxyList, cluster.ClusterNetwork.Pods.CidrBlocks...)
noProxyList = append(noProxyList, cluster.ClusterNetwork.Services.CidrBlocks...)
noProxyList = append(noProxyList, cluster.ProxyConfiguration.NoProxy...)

// Add no-proxy defaults
noProxyList = append(noProxyList, NoProxyDefaults()...)
noProxyList = append(noProxyList, cluster.ControlPlaneConfiguration.Endpoint.Host)

val := values{
"httpProxy": cluster.ProxyConfiguration.HttpProxy,
"httpsProxy": cluster.ProxyConfiguration.HttpsProxy,
"noProxy": noProxyList,
}

config, err := templater.Execute(proxyConfig, val)
if err != nil {
return "", fmt.Errorf("building http-proxy.conf file: %v", err)
}
return string(config), nil
}

func proxyConfigFile(cluster v1alpha1.ClusterSpec) (bootstrapv1.File, error) {
proxyConfig, err := proxyConfigContent(cluster)
if err != nil {
return bootstrapv1.File{}, err
}

return bootstrapv1.File{
Path: "/etc/systemd/system/containerd.service.d/http-proxy.conf",
Owner: "root:root",
Content: proxyConfig,
}, nil
}

func SetProxyConfigInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) error {
if cluster.ProxyConfiguration == nil {
return nil
}

proxyConfigFile, err := proxyConfigFile(cluster)
if err != nil {
return err
}

kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, proxyConfigFile)

return nil
}

func SetProxyConfigInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) error {
if cluster.ProxyConfiguration == nil {
return nil
}

proxyConfigFile, err := proxyConfigFile(cluster)
if err != nil {
return err
}

kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, proxyConfigFile)

return nil
}
82 changes: 82 additions & 0 deletions pkg/clusterapi/proxy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package clusterapi_test

import (
"testing"

. "github.com/onsi/gomega"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"

"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/clusterapi"
)

var proxyTests = []struct {
name string
proxy *v1alpha1.ProxyConfiguration
wantFiles []bootstrapv1.File
}{
{
name: "proxy config nil",
proxy: nil,
wantFiles: []bootstrapv1.File{},
},
{
name: "with proxy, pods cidr, service cidr, cp endpoint",
proxy: &v1alpha1.ProxyConfiguration{
HttpProxy: "1.2.3.4:8888",
HttpsProxy: "1.2.3.4:8888",
NoProxy: []string{
"1.2.3.4/0",
"1.2.3.5/0",
},
},
wantFiles: []bootstrapv1.File{
{
Path: "/etc/systemd/system/containerd.service.d/http-proxy.conf",
Owner: "root:root",
Content: `[Service]
Environment="HTTP_PROXY=1.2.3.4:8888"
Environment="HTTPS_PROXY=1.2.3.4:8888"
Environment="NO_PROXY=1.2.3.4/5,1.2.3.4/5,1.2.3.4/0,1.2.3.5/0,localhost,127.0.0.1,.svc,1.2.3.4"`,
},
},
},
}

func TestSetProxyConfigInKubeadmControlPlane(t *testing.T) {
for _, tt := range proxyTests {
t.Run(tt.name, func(t *testing.T) {
g := newApiBuilerTest(t)
got := wantKubeadmControlPlane()
g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
g.Expect(clusterapi.SetProxyConfigInKubeadmControlPlane(got, g.clusterSpec.Cluster.Spec)).To(Succeed())
want := wantKubeadmControlPlane()
want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
g.Expect(got).To(Equal(want))
})
}
}

func TestSetProxyConfigInKubeadmConfigTemplate(t *testing.T) {
for _, tt := range proxyTests {
t.Run(tt.name, func(t *testing.T) {
g := newApiBuilerTest(t)
got := wantKubeadmConfigTemplate()
g.clusterSpec.Cluster.Spec.ProxyConfiguration = tt.proxy
g.Expect(clusterapi.SetProxyConfigInKubeadmConfigTemplate(got, g.clusterSpec.Cluster.Spec)).To(Succeed())
want := wantKubeadmConfigTemplate()
want.Spec.Template.Spec.Files = tt.wantFiles
g.Expect(got).To(Equal(want))
})
}
}

func TestNoProxyDefaults(t *testing.T) {
g := NewWithT(t)
want := []string{
"localhost",
"127.0.0.1",
".svc",
}
g.Expect(clusterapi.NoProxyDefaults()).To(Equal(want))
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,16 @@ func registryMirrorConfigContent(registryAddress, registryCert string, insecureS

config, err := templater.Execute(containerdConfig, val)
if err != nil {
return "", fmt.Errorf("failed building containerd config file: %v", err)
return "", fmt.Errorf("building containerd config file: %v", err)
}
return string(config), nil
}

func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration) (files []bootstrapv1.File, preKubeadmCommands []string, err error) {
func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfiguration) (files []bootstrapv1.File, err error) {
registryAddress := net.JoinHostPort(registryMirrorConfig.Endpoint, registryMirrorConfig.Port)
registryConfig, err := registryMirrorConfigContent(registryAddress, registryMirrorConfig.CACertContent, registryMirrorConfig.InsecureSkipVerify)
if err != nil {
return nil, nil, err
return nil, err
}
files = []bootstrapv1.File{
{
Expand All @@ -53,26 +53,20 @@ func registryMirrorConfig(registryMirrorConfig *v1alpha1.RegistryMirrorConfigura
})
}

preKubeadmCommands = []string{
"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
"sudo systemctl daemon-reload",
"sudo systemctl restart containerd",
}
return files, preKubeadmCommands, nil
return files, nil
}

func SetRegistryMirrorInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, mirrorConfig *v1alpha1.RegistryMirrorConfiguration) error {
if mirrorConfig == nil {
return nil
}

containerdFiles, containerdCommands, err := registryMirrorConfig(mirrorConfig)
containerdFiles, err := registryMirrorConfig(mirrorConfig)
if err != nil {
return fmt.Errorf("setting registry mirror configuration: %v", err)
}

kcp.Spec.KubeadmConfigSpec.Files = append(kcp.Spec.KubeadmConfigSpec.Files, containerdFiles...)
kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, containerdCommands...)

return nil
}
Expand All @@ -82,13 +76,12 @@ func SetRegistryMirrorInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemp
return nil
}

containerdFiles, containerdCommands, err := registryMirrorConfig(mirrorConfig)
containerdFiles, err := registryMirrorConfig(mirrorConfig)
if err != nil {
return fmt.Errorf("setting registry mirror configuration: %v", err)
}

kct.Spec.Template.Spec.Files = append(kct.Spec.Template.Spec.Files, containerdFiles...)
kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, containerdCommands...)

return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,6 @@ var registryMirrorTests = []struct {
},
}

func wantRegistryMirrorCommands() []string {
return []string{
"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
"sudo systemctl daemon-reload",
"sudo systemctl restart containerd",
}
}

func TestSetRegistryMirrorInKubeadmControlPlane(t *testing.T) {
for _, tt := range registryMirrorTests {
t.Run(tt.name, func(t *testing.T) {
Expand All @@ -102,7 +94,6 @@ func TestSetRegistryMirrorInKubeadmControlPlane(t *testing.T) {
g.Expect(clusterapi.SetRegistryMirrorInKubeadmControlPlane(got, tt.registryMirrorConfig)).To(Succeed())
want := wantKubeadmControlPlane()
want.Spec.KubeadmConfigSpec.Files = tt.wantFiles
want.Spec.KubeadmConfigSpec.PreKubeadmCommands = wantRegistryMirrorCommands()
g.Expect(got).To(Equal(want))
})
}
Expand All @@ -116,7 +107,6 @@ func TestSetRegistryMirrorInKubeadmConfigTemplate(t *testing.T) {
g.Expect(clusterapi.SetRegistryMirrorInKubeadmConfigTemplate(got, tt.registryMirrorConfig)).To(Succeed())
want := wantKubeadmConfigTemplate()
want.Spec.Template.Spec.Files = tt.wantFiles
want.Spec.Template.Spec.PreKubeadmCommands = wantRegistryMirrorCommands()
g.Expect(got).To(Equal(want))
})
}
Expand Down
45 changes: 45 additions & 0 deletions pkg/clusterapi/systemctl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package clusterapi

import (
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"

"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
)

var buildContainerdConfigCommands = []string{
"cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml",
}

var restartContainerdCommands = []string{
"sudo systemctl daemon-reload",
"sudo systemctl restart containerd",
}

func CreateContainerdConfigFileInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) {
if cluster.RegistryMirrorConfiguration != nil {
kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, buildContainerdConfigCommands...)
}
}

func CreateContainerdConfigFileInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) {
if cluster.RegistryMirrorConfiguration != nil {
kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, buildContainerdConfigCommands...)
}
}

func RestartContainerdInKubeadmControlPlane(kcp *controlplanev1.KubeadmControlPlane, cluster v1alpha1.ClusterSpec) {
if restartContainerdNeeded(cluster) {
kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(kcp.Spec.KubeadmConfigSpec.PreKubeadmCommands, restartContainerdCommands...)
}
}

func RestartContainerdInKubeadmConfigTemplate(kct *bootstrapv1.KubeadmConfigTemplate, cluster v1alpha1.ClusterSpec) {
if restartContainerdNeeded(cluster) {
kct.Spec.Template.Spec.PreKubeadmCommands = append(kct.Spec.Template.Spec.PreKubeadmCommands, restartContainerdCommands...)
}
}

func restartContainerdNeeded(cluster v1alpha1.ClusterSpec) bool {
return cluster.RegistryMirrorConfiguration != nil || cluster.ProxyConfiguration != nil
}
Loading

0 comments on commit ccc024f

Please sign in to comment.