Skip to content

Commit

Permalink
refactor: move more nodeup script logic to NodeUpScript
Browse files Browse the repository at this point in the history 
This enables simpler reuse.
  • Loading branch information
@justinsb
justinsb committed Aug 30, 2024
1 parent d5c56dc commit c9eb14a
Show file tree
Hide file tree
Showing 4 changed files with 219 additions and 244 deletions.
38 changes: 3 additions & 35 deletions pkg/commands/toolbox_enroll.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -477,41 +477,9 @@ func buildBootstrapData(ctx context.Context, clientset simple.Clientset, cluster
nodeupScript.NodeUpAssets = nodeUpAssets
nodeupScript.BootConfig = bootConfig

{
nodeupScript.EnvironmentVariables = func() (string, error) {
env := make(map[string]string)

// TODO: Support the full set of environment variables?
// env, err := b.buildEnvironmentVariables()
// if err != nil {
// return "", err
// }

// Sort keys to have a stable sequence of "export xx=xxx"" statements
var keys []string
for k := range env {
keys = append(keys, k)
}
sort.Strings(keys)

var b bytes.Buffer
for _, k := range keys {
b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k]))
}
return b.String(), nil
}

nodeupScript.ProxyEnv = func() (string, error) {
return "", nil
// TODO: Support proxy?
// return b.createProxyEnv(cluster.Spec.Networking.EgressProxy)
}
}

// TODO: Support sysctls?
// By setting some sysctls early, we avoid broken configurations that prevent nodeup download.
// See https://github.com/kubernetes/kops/issues/10206 for details.
// nodeupScript.SetSysctls = setSysctls()
nodeupScript.withEnvironmentVariables(cluster, ig)
nodeupScript.WithProxyEnv(cluster)
nodeupScript.WithSysctls()

nodeupScript.CloudProvider = string(cluster.GetCloudProvider())

Expand Down
204 changes: 3 additions & 201 deletions pkg/model/bootstrapscript.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,10 @@ limitations under the License.
package model

import (
"bytes"
"crypto/sha256"
"encoding/base64"
"fmt"
"os"
"sort"
"strconv"
"strings"

"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
Expand All @@ -33,8 +29,6 @@ import (
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
"k8s.io/kops/upup/pkg/fi/fitasks"
"k8s.io/kops/upup/pkg/fi/utils"
"k8s.io/kops/util/pkg/architectures"
Expand Down Expand Up @@ -130,110 +124,6 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.CloudupContext)
return bootConfig, nil
}

func (b *BootstrapScript) buildEnvironmentVariables() (map[string]string, error) {
cluster := b.cluster

env := make(map[string]string)

if os.Getenv("GOSSIP_DNS_CONN_LIMIT") != "" {
env["GOSSIP_DNS_CONN_LIMIT"] = os.Getenv("GOSSIP_DNS_CONN_LIMIT")
}

if os.Getenv("S3_ENDPOINT") != "" {
if b.ig.IsControlPlane() {
env["S3_ENDPOINT"] = os.Getenv("S3_ENDPOINT")
env["S3_REGION"] = os.Getenv("S3_REGION")
env["S3_ACCESS_KEY_ID"] = os.Getenv("S3_ACCESS_KEY_ID")
env["S3_SECRET_ACCESS_KEY"] = os.Getenv("S3_SECRET_ACCESS_KEY")
}
}

if cluster.GetCloudProvider() == kops.CloudProviderOpenstack {

osEnvs := []string{
"OS_TENANT_ID", "OS_TENANT_NAME", "OS_PROJECT_ID", "OS_PROJECT_NAME",
"OS_PROJECT_DOMAIN_NAME", "OS_PROJECT_DOMAIN_ID",
"OS_DOMAIN_NAME", "OS_DOMAIN_ID",
"OS_AUTH_URL",
"OS_REGION_NAME",
}

appCreds := os.Getenv("OS_APPLICATION_CREDENTIAL_ID") != "" && os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET") != ""
if appCreds {
osEnvs = append(osEnvs,
"OS_APPLICATION_CREDENTIAL_ID",
"OS_APPLICATION_CREDENTIAL_SECRET",
)
} else {
klog.Warning("exporting username and password. Consider using application credentials instead.")
osEnvs = append(osEnvs,
"OS_USERNAME",
"OS_PASSWORD",
)
}

// credentials needed always in control-plane and when using gossip also in nodes
passEnvs := false
if b.ig.IsControlPlane() || cluster.UsesLegacyGossip() {
passEnvs = true
}
// Pass in required credentials when using user-defined swift endpoint
if os.Getenv("OS_AUTH_URL") != "" && passEnvs {
for _, envVar := range osEnvs {
env[envVar] = fmt.Sprintf("'%s'", os.Getenv(envVar))
}
}
}

if cluster.GetCloudProvider() == kops.CloudProviderDO {
if b.ig.IsControlPlane() {
doToken := os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
if doToken != "" {
env["DIGITALOCEAN_ACCESS_TOKEN"] = doToken
}
}
}

if cluster.GetCloudProvider() == kops.CloudProviderHetzner && (b.ig.IsControlPlane() || cluster.UsesLegacyGossip()) {
hcloudToken := os.Getenv("HCLOUD_TOKEN")
if hcloudToken != "" {
env["HCLOUD_TOKEN"] = hcloudToken
}
}

if cluster.GetCloudProvider() == kops.CloudProviderAWS {
region, err := awsup.FindRegion(cluster)
if err != nil {
return nil, err
}
if region == "" {
klog.Warningf("unable to determine cluster region")
} else {
env["AWS_REGION"] = region
}
}

if cluster.GetCloudProvider() == kops.CloudProviderAzure {
env["AZURE_STORAGE_ACCOUNT"] = os.Getenv("AZURE_STORAGE_ACCOUNT")
azureEnv := os.Getenv("AZURE_ENVIRONMENT")
if azureEnv != "" {
env["AZURE_ENVIRONMENT"] = os.Getenv("AZURE_ENVIRONMENT")
}
}

if cluster.GetCloudProvider() == kops.CloudProviderScaleway && (b.ig.IsControlPlane() || cluster.UsesLegacyGossip()) {
profile, err := scaleway.CreateValidScalewayProfile()
if err != nil {
return nil, err
}
env["SCW_ACCESS_KEY"] = fi.ValueOf(profile.AccessKey)
env["SCW_SECRET_KEY"] = fi.ValueOf(profile.SecretKey)
env["SCW_DEFAULT_PROJECT_ID"] = fi.ValueOf(profile.DefaultProjectID)
}

return env, nil
}

// ResourceNodeUp generates and returns a nodeup (bootstrap) script from a
// template file, substituting in specific env vars & cluster spec configuration
func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.CloudupModelBuilderContext, ig *kops.InstanceGroup) (fi.Resource, error) {
Expand Down Expand Up @@ -324,38 +214,12 @@ func (b *BootstrapScript) Run(c *fi.CloudupContext) error {
nodeupScript.NodeUpAssets = b.builder.NodeUpAssets
nodeupScript.BootConfig = bootConfig

{
nodeupScript.EnvironmentVariables = func() (string, error) {
env, err := b.buildEnvironmentVariables()
if err != nil {
return "", err
}

// Sort keys to have a stable sequence of "export xx=xxx"" statements
var keys []string
for k := range env {
keys = append(keys, k)
}
sort.Strings(keys)

var b bytes.Buffer
for _, k := range keys {
b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k]))
}
return b.String(), nil
}

nodeupScript.ProxyEnv = func() (string, error) {
return b.createProxyEnv(c.T.Cluster.Spec.Networking.EgressProxy)
}
}
nodeupScript.WithEnvironmentVariables(b.cluster, b.ig)
nodeupScript.WithProxyEnv(b.cluster)
nodeupScript.WithSysctls()

nodeupScript.CompressUserData = fi.ValueOf(b.ig.Spec.CompressUserData)

// By setting some sysctls early, we avoid broken configurations that prevent nodeup download.
// See https://github.com/kubernetes/kops/issues/10206 for details.
nodeupScript.SetSysctls = setSysctls()

nodeupScript.CloudProvider = string(c.T.Cluster.GetCloudProvider())

nodeupScriptResource, err := nodeupScript.Build()
Expand All @@ -378,65 +242,3 @@ func (b *BootstrapScript) Run(c *fi.CloudupContext) error {
})
return nil
}

func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) (string, error) {
var buffer bytes.Buffer

if ps != nil && ps.HTTPProxy.Host != "" {
var httpProxyURL string

// TODO double check that all the code does this
// TODO move this into a validate so we can enforce the string syntax
if !strings.HasPrefix(ps.HTTPProxy.Host, "http://") {
httpProxyURL = "http://"
}

if ps.HTTPProxy.Port != 0 {
httpProxyURL += ps.HTTPProxy.Host + ":" + strconv.Itoa(ps.HTTPProxy.Port)
} else {
httpProxyURL += ps.HTTPProxy.Host
}

// Set env variables for base environment
buffer.WriteString(`{` + "\n")
buffer.WriteString(` echo "http_proxy=` + httpProxyURL + `"` + "\n")
buffer.WriteString(` echo "https_proxy=` + httpProxyURL + `"` + "\n")
buffer.WriteString(` echo "no_proxy=` + ps.ProxyExcludes + `"` + "\n")
buffer.WriteString(` echo "NO_PROXY=` + ps.ProxyExcludes + `"` + "\n")
buffer.WriteString(`} >> /etc/environment` + "\n")

// Load the proxy environment variables
buffer.WriteString("while read -r in; do export \"${in?}\"; done < /etc/environment\n")

// Set env variables for package manager depending on OS Distribution (N/A for Flatcar)
// Note: Nodeup will source the `/etc/environment` file within docker config in the correct location
buffer.WriteString("case $(cat /proc/version) in\n")
buffer.WriteString("*[Dd]ebian* | *[Uu]buntu*)\n")
buffer.WriteString(` echo "Acquire::http::Proxy \"` + httpProxyURL + `\";" > /etc/apt/apt.conf.d/30proxy ;;` + "\n")
buffer.WriteString("*[Rr]ed[Hh]at*)\n")
buffer.WriteString(` echo "proxy=` + httpProxyURL + `" >> /etc/yum.conf ;;` + "\n")
buffer.WriteString("esac\n")

// Set env variables for systemd
buffer.WriteString(`echo "DefaultEnvironment=\"http_proxy=` + httpProxyURL + `\" \"https_proxy=` + httpProxyURL + `\"`)
buffer.WriteString(` \"NO_PROXY=` + ps.ProxyExcludes + `\" \"no_proxy=` + ps.ProxyExcludes + `\""`)
buffer.WriteString(" >> /etc/systemd/system.conf\n")

// Restart stuff
buffer.WriteString("systemctl daemon-reload\n")
buffer.WriteString("systemctl daemon-reexec\n")
}
return buffer.String(), nil
}

func setSysctls() string {
var b bytes.Buffer

// Based on https://github.com/kubernetes/kops/issues/10206#issuecomment-766852332
b.WriteString("sysctl -w net.core.rmem_max=16777216 || true\n")
b.WriteString("sysctl -w net.core.wmem_max=16777216 || true\n")
b.WriteString("sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true\n")
b.WriteString("sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true\n")

return b.String()
}
20 changes: 12 additions & 8 deletions pkg/model/bootstrapscript_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/testutils/golden"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/fitasks"
Expand All @@ -36,17 +37,20 @@ import (
)

func Test_ProxyFunc(t *testing.T) {
b := &BootstrapScript{}
ps := &kops.EgressProxySpec{
cluster := &kops.Cluster{}
cluster.Spec.Networking.EgressProxy = &kops.EgressProxySpec{
HTTPProxy: kops.HTTPProxy{
Host: "example.com",
Port: 80,
},
}

script, err := b.createProxyEnv(ps)
nodeupScript := &resources.NodeUpScript{}
nodeupScript.WithProxyEnv(cluster)

script, err := nodeupScript.ProxyEnv()
if err != nil {
t.Fatalf("createProxyEnv failed: %v", err)
t.Fatalf("ProxyEnv failed: %v", err)
}
if script == "" {
t.Fatalf("script cannot be empty")
Expand All @@ -56,14 +60,14 @@ func Test_ProxyFunc(t *testing.T) {
t.Fatalf("script not setting http_proxy properly")
}

ps.ProxyExcludes = "www.google.com,www.kubernetes.io"
cluster.Spec.Networking.EgressProxy.ProxyExcludes = "www.google.com,www.kubernetes.io"

script, err = b.createProxyEnv(ps)
script, err = nodeupScript.ProxyEnv()
if err != nil {
t.Fatalf("createProxyEnv failed: %v", err)
t.Fatalf("ProxyEnv failed: %v", err)
}

if !strings.Contains(script, "no_proxy="+ps.ProxyExcludes) {
if !strings.Contains(script, "no_proxy="+cluster.Spec.Networking.EgressProxy.ProxyExcludes) {
t.Fatalf("script not setting no_proxy properly")
}
}
Expand Down
Loading

0 comments on commit c9eb14a

Please sign in to comment.