Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: move more nodeup script logic to NodeUpScript #16793

Merged
merged 1 commit into from
Aug 30, 2024
Merged

refactor: move more nodeup script logic to NodeUpScript #16793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 3 additions & 35 deletions pkg/commands/toolbox_enroll.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,41 +477,9 @@ func buildBootstrapData(ctx context.Context, clientset simple.Clientset, cluster
nodeupScript.NodeUpAssets = nodeUpAssets
nodeupScript.BootConfig = bootConfig

{
nodeupScript.EnvironmentVariables = func() (string, error) {
env := make(map[string]string)

// TODO: Support the full set of environment variables?
// env, err := b.buildEnvironmentVariables()
// if err != nil {
// return "", err
// }

// Sort keys to have a stable sequence of "export xx=xxx"" statements
var keys []string
for k := range env {
keys = append(keys, k)
}
sort.Strings(keys)

var b bytes.Buffer
for _, k := range keys {
b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k]))
}
return b.String(), nil
}

nodeupScript.ProxyEnv = func() (string, error) {
return "", nil
// TODO: Support proxy?
// return b.createProxyEnv(cluster.Spec.Networking.EgressProxy)
}
}

// TODO: Support sysctls?
// By setting some sysctls early, we avoid broken configurations that prevent nodeup download.
// See https://github.com/kubernetes/kops/issues/10206 for details.
// nodeupScript.SetSysctls = setSysctls()
nodeupScript.WithEnvironmentVariables(cluster, ig)
nodeupScript.WithProxyEnv(cluster)
nodeupScript.WithSysctls()

nodeupScript.CloudProvider = string(cluster.GetCloudProvider())

Expand Down
204 changes: 3 additions & 201 deletions pkg/model/bootstrapscript.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,10 @@ limitations under the License.
package model

import (
"bytes"
"crypto/sha256"
"encoding/base64"
"fmt"
"os"
"sort"
"strconv"
"strings"

"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
Expand All @@ -33,8 +29,6 @@ import (
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/wellknownservices"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
"k8s.io/kops/upup/pkg/fi/fitasks"
"k8s.io/kops/upup/pkg/fi/utils"
"k8s.io/kops/util/pkg/architectures"
Expand Down Expand Up @@ -130,110 +124,6 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.CloudupContext)
return bootConfig, nil
}

func (b *BootstrapScript) buildEnvironmentVariables() (map[string]string, error) {
cluster := b.cluster

env := make(map[string]string)

if os.Getenv("GOSSIP_DNS_CONN_LIMIT") != "" {
env["GOSSIP_DNS_CONN_LIMIT"] = os.Getenv("GOSSIP_DNS_CONN_LIMIT")
}

if os.Getenv("S3_ENDPOINT") != "" {
if b.ig.IsControlPlane() {
env["S3_ENDPOINT"] = os.Getenv("S3_ENDPOINT")
env["S3_REGION"] = os.Getenv("S3_REGION")
env["S3_ACCESS_KEY_ID"] = os.Getenv("S3_ACCESS_KEY_ID")
env["S3_SECRET_ACCESS_KEY"] = os.Getenv("S3_SECRET_ACCESS_KEY")
}
}

if cluster.GetCloudProvider() == kops.CloudProviderOpenstack {

osEnvs := []string{
"OS_TENANT_ID", "OS_TENANT_NAME", "OS_PROJECT_ID", "OS_PROJECT_NAME",
"OS_PROJECT_DOMAIN_NAME", "OS_PROJECT_DOMAIN_ID",
"OS_DOMAIN_NAME", "OS_DOMAIN_ID",
"OS_AUTH_URL",
"OS_REGION_NAME",
}

appCreds := os.Getenv("OS_APPLICATION_CREDENTIAL_ID") != "" && os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET") != ""
if appCreds {
osEnvs = append(osEnvs,
"OS_APPLICATION_CREDENTIAL_ID",
"OS_APPLICATION_CREDENTIAL_SECRET",
)
} else {
klog.Warning("exporting username and password. Consider using application credentials instead.")
osEnvs = append(osEnvs,
"OS_USERNAME",
"OS_PASSWORD",
)
}

// credentials needed always in control-plane and when using gossip also in nodes
passEnvs := false
if b.ig.IsControlPlane() || cluster.UsesLegacyGossip() {
passEnvs = true
}
// Pass in required credentials when using user-defined swift endpoint
if os.Getenv("OS_AUTH_URL") != "" && passEnvs {
for _, envVar := range osEnvs {
env[envVar] = fmt.Sprintf("'%s'", os.Getenv(envVar))
}
}
}

if cluster.GetCloudProvider() == kops.CloudProviderDO {
if b.ig.IsControlPlane() {
doToken := os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
if doToken != "" {
env["DIGITALOCEAN_ACCESS_TOKEN"] = doToken
}
}
}

if cluster.GetCloudProvider() == kops.CloudProviderHetzner && (b.ig.IsControlPlane() || cluster.UsesLegacyGossip()) {
hcloudToken := os.Getenv("HCLOUD_TOKEN")
if hcloudToken != "" {
env["HCLOUD_TOKEN"] = hcloudToken
}
}

if cluster.GetCloudProvider() == kops.CloudProviderAWS {
region, err := awsup.FindRegion(cluster)
if err != nil {
return nil, err
}
if region == "" {
klog.Warningf("unable to determine cluster region")
} else {
env["AWS_REGION"] = region
}
}

if cluster.GetCloudProvider() == kops.CloudProviderAzure {
env["AZURE_STORAGE_ACCOUNT"] = os.Getenv("AZURE_STORAGE_ACCOUNT")
azureEnv := os.Getenv("AZURE_ENVIRONMENT")
if azureEnv != "" {
env["AZURE_ENVIRONMENT"] = os.Getenv("AZURE_ENVIRONMENT")
}
}

if cluster.GetCloudProvider() == kops.CloudProviderScaleway && (b.ig.IsControlPlane() || cluster.UsesLegacyGossip()) {
profile, err := scaleway.CreateValidScalewayProfile()
if err != nil {
return nil, err
}
env["SCW_ACCESS_KEY"] = fi.ValueOf(profile.AccessKey)
env["SCW_SECRET_KEY"] = fi.ValueOf(profile.SecretKey)
env["SCW_DEFAULT_PROJECT_ID"] = fi.ValueOf(profile.DefaultProjectID)
}

return env, nil
}

// ResourceNodeUp generates and returns a nodeup (bootstrap) script from a
// template file, substituting in specific env vars & cluster spec configuration
func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.CloudupModelBuilderContext, ig *kops.InstanceGroup) (fi.Resource, error) {
Expand Down Expand Up @@ -324,38 +214,12 @@ func (b *BootstrapScript) Run(c *fi.CloudupContext) error {
nodeupScript.NodeUpAssets = b.builder.NodeUpAssets
nodeupScript.BootConfig = bootConfig

{
nodeupScript.EnvironmentVariables = func() (string, error) {
env, err := b.buildEnvironmentVariables()
if err != nil {
return "", err
}

// Sort keys to have a stable sequence of "export xx=xxx"" statements
var keys []string
for k := range env {
keys = append(keys, k)
}
sort.Strings(keys)

var b bytes.Buffer
for _, k := range keys {
b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k]))
}
return b.String(), nil
}

nodeupScript.ProxyEnv = func() (string, error) {
return b.createProxyEnv(c.T.Cluster.Spec.Networking.EgressProxy)
}
}
nodeupScript.WithEnvironmentVariables(b.cluster, b.ig)
nodeupScript.WithProxyEnv(b.cluster)
nodeupScript.WithSysctls()

nodeupScript.CompressUserData = fi.ValueOf(b.ig.Spec.CompressUserData)

// By setting some sysctls early, we avoid broken configurations that prevent nodeup download.
// See https://github.com/kubernetes/kops/issues/10206 for details.
nodeupScript.SetSysctls = setSysctls()

nodeupScript.CloudProvider = string(c.T.Cluster.GetCloudProvider())

nodeupScriptResource, err := nodeupScript.Build()
Expand All @@ -378,65 +242,3 @@ func (b *BootstrapScript) Run(c *fi.CloudupContext) error {
})
return nil
}

func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) (string, error) {
var buffer bytes.Buffer

if ps != nil && ps.HTTPProxy.Host != "" {
var httpProxyURL string

// TODO double check that all the code does this
// TODO move this into a validate so we can enforce the string syntax
if !strings.HasPrefix(ps.HTTPProxy.Host, "http://") {
httpProxyURL = "http://"
}

if ps.HTTPProxy.Port != 0 {
httpProxyURL += ps.HTTPProxy.Host + ":" + strconv.Itoa(ps.HTTPProxy.Port)
} else {
httpProxyURL += ps.HTTPProxy.Host
}

// Set env variables for base environment
buffer.WriteString(`{` + "\n")
buffer.WriteString(` echo "http_proxy=` + httpProxyURL + `"` + "\n")
buffer.WriteString(` echo "https_proxy=` + httpProxyURL + `"` + "\n")
buffer.WriteString(` echo "no_proxy=` + ps.ProxyExcludes + `"` + "\n")
buffer.WriteString(` echo "NO_PROXY=` + ps.ProxyExcludes + `"` + "\n")
buffer.WriteString(`} >> /etc/environment` + "\n")

// Load the proxy environment variables
buffer.WriteString("while read -r in; do export \"${in?}\"; done < /etc/environment\n")

// Set env variables for package manager depending on OS Distribution (N/A for Flatcar)
// Note: Nodeup will source the `/etc/environment` file within docker config in the correct location
buffer.WriteString("case $(cat /proc/version) in\n")
buffer.WriteString("*[Dd]ebian* | *[Uu]buntu*)\n")
buffer.WriteString(` echo "Acquire::http::Proxy \"` + httpProxyURL + `\";" > /etc/apt/apt.conf.d/30proxy ;;` + "\n")
buffer.WriteString("*[Rr]ed[Hh]at*)\n")
buffer.WriteString(` echo "proxy=` + httpProxyURL + `" >> /etc/yum.conf ;;` + "\n")
buffer.WriteString("esac\n")

// Set env variables for systemd
buffer.WriteString(`echo "DefaultEnvironment=\"http_proxy=` + httpProxyURL + `\" \"https_proxy=` + httpProxyURL + `\"`)
buffer.WriteString(` \"NO_PROXY=` + ps.ProxyExcludes + `\" \"no_proxy=` + ps.ProxyExcludes + `\""`)
buffer.WriteString(" >> /etc/systemd/system.conf\n")

// Restart stuff
buffer.WriteString("systemctl daemon-reload\n")
buffer.WriteString("systemctl daemon-reexec\n")
}
return buffer.String(), nil
}

func setSysctls() string {
var b bytes.Buffer

// Based on https://github.com/kubernetes/kops/issues/10206#issuecomment-766852332
b.WriteString("sysctl -w net.core.rmem_max=16777216 || true\n")
b.WriteString("sysctl -w net.core.wmem_max=16777216 || true\n")
b.WriteString("sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true\n")
b.WriteString("sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true\n")

return b.String()
}
20 changes: 12 additions & 8 deletions pkg/model/bootstrapscript_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/kops/pkg/apis/nodeup"
"k8s.io/kops/pkg/assets"
"k8s.io/kops/pkg/model/iam"
"k8s.io/kops/pkg/model/resources"
"k8s.io/kops/pkg/testutils/golden"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/fitasks"
Expand All @@ -36,17 +37,20 @@ import (
)

func Test_ProxyFunc(t *testing.T) {
b := &BootstrapScript{}
ps := &kops.EgressProxySpec{
cluster := &kops.Cluster{}
cluster.Spec.Networking.EgressProxy = &kops.EgressProxySpec{
HTTPProxy: kops.HTTPProxy{
Host: "example.com",
Port: 80,
},
}

script, err := b.createProxyEnv(ps)
nodeupScript := &resources.NodeUpScript{}
nodeupScript.WithProxyEnv(cluster)

script, err := nodeupScript.ProxyEnv()
if err != nil {
t.Fatalf("createProxyEnv failed: %v", err)
t.Fatalf("ProxyEnv failed: %v", err)
}
if script == "" {
t.Fatalf("script cannot be empty")
Expand All @@ -56,14 +60,14 @@ func Test_ProxyFunc(t *testing.T) {
t.Fatalf("script not setting http_proxy properly")
}

ps.ProxyExcludes = "www.google.com,www.kubernetes.io"
cluster.Spec.Networking.EgressProxy.ProxyExcludes = "www.google.com,www.kubernetes.io"

script, err = b.createProxyEnv(ps)
script, err = nodeupScript.ProxyEnv()
if err != nil {
t.Fatalf("createProxyEnv failed: %v", err)
t.Fatalf("ProxyEnv failed: %v", err)
}

if !strings.Contains(script, "no_proxy="+ps.ProxyExcludes) {
if !strings.Contains(script, "no_proxy="+cluster.Spec.Networking.EgressProxy.ProxyExcludes) {
t.Fatalf("script not setting no_proxy properly")
}
}
Expand Down
Loading
Loading