[Feature] Enhanced k3d-proxy configuration

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v5.0.0
+
+### Fixes
+
+- cleaned up and properly sorted the sanitization of existing resources used to create new nodes (#638)
+
+### Features & Enhancements
+
+- new command: `k3d node edit` to edit existing nodes (#615)
+  - currently only allows `k3d node edit NODE --port-add HOSTPORT:CONTAINERPORT` for the serverlb/loadbalancer to add new ports
+  - pkg: new `NodeEdit` function
+- new (hidden) command: `k3d debug` with some options for debugging k3d resources (#638)
+  - e.g. `k3d debug loadbalancer get-config` to get the current loadbalancer configuration
+- loadbalancer / k3d-proxy (#638)
+  - updated fork of `confd` to make usage of the file backend including a file watcher for auto-reloads
+    - this also checks the config before applying it, so the lb doesn't crash on a faulty config
+  - updating the loadbalancer writes the new config file and also checks if everything's going fine afterwards
+- helper images can now be set explicitly via environment variables: `K3D_IMAGE_LOADBALANCER` & `K3D_IMAGE_TOOLS` (#638)
+
+### Misc
+
+- tests/e2e: timeouts everywhere to avoid killing DroneCI (#638)
+
 ## v4.4.6
 
 ### Fixes

cmd/debug/debug.go

@@ -0,0 +1,91 @@
+/*
+Copyright © 2020-2021 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package debug
+
+import (
+	"fmt"
+
+	"github.com/rancher/k3d/v4/pkg/client"
+	"github.com/rancher/k3d/v4/pkg/runtimes"
+	"github.com/rancher/k3d/v4/pkg/types"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+// NewCmdDebug returns a new cobra command
+func NewCmdDebug() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:    "debug",
+		Hidden: true,
+		Short:  "Debug k3d cluster(s)",
+		Long:   `Debug k3d cluster(s)`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cmd.Help(); err != nil {
+				log.Errorln("Couldn't get help text")
+				log.Fatalln(err)
+			}
+		},
+	}
+
+	cmd.AddCommand(NewCmdDebugLoadbalancer())
+
+	return cmd
+}
+
+func NewCmdDebugLoadbalancer() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "loadbalancer",
+		Aliases: []string{"lb"},
+		Short:   "Debug the loadbalancer",
+		Long:    `Debug the loadbalancer`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cmd.Help(); err != nil {
+				log.Errorln("Couldn't get help text")
+				log.Fatalln(err)
+			}
+		},
+	}
+
+	cmd.AddCommand(&cobra.Command{
+		Use:  "get-config",
+		Args: cobra.ExactArgs(1), // cluster name
+		Run: func(cmd *cobra.Command, args []string) {
+			c, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &types.Cluster{Name: args[0]})
+			if err != nil {
+				log.Fatalln(err)
+			}
+
+			lbconf, err := client.GetLoadbalancerConfig(cmd.Context(), runtimes.SelectedRuntime, c)
+			if err != nil {
+				log.Fatalln(err)
+			}
+			yamlized, err := yaml.Marshal(lbconf)
+			if err != nil {
+				log.Fatalln(err)
+			}
+			fmt.Println(string(yamlized))
+		},
+	})
+
+	return cmd
+}

cmd/root.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/rancher/k3d/v4/cmd/cluster"
 	cfg "github.com/rancher/k3d/v4/cmd/config"
+	"github.com/rancher/k3d/v4/cmd/debug"
 	"github.com/rancher/k3d/v4/cmd/image"
 	"github.com/rancher/k3d/v4/cmd/kubeconfig"
 	"github.com/rancher/k3d/v4/cmd/node"
@@ -116,6 +117,7 @@ func init() {
 	rootCmd.AddCommand(image.NewCmdImage())
 	rootCmd.AddCommand(cfg.NewCmdConfig())
 	rootCmd.AddCommand(registry.NewCmdRegistry())
+	rootCmd.AddCommand(debug.NewCmdDebug())
 
 	rootCmd.AddCommand(&cobra.Command{
 		Use:   "version",

pkg/client/cluster.go

@@ -28,7 +28,6 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
-	"strings"
 	"time"
 
 	gort "runtime"
@@ -44,7 +43,6 @@ import (
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	"github.com/rancher/k3d/v4/pkg/types/k3s"
 	"github.com/rancher/k3d/v4/pkg/util"
-	"github.com/rancher/k3d/v4/version"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
@@ -509,59 +507,41 @@ ClusterCreatOpts:
 	 */
 	// *** ServerLoadBalancer ***
 	if !clusterCreateOpts.DisableLoadBalancer {
-		// Generate a comma-separated list of server/server names to pass to the LB container
-		servers := ""
-		for _, node := range cluster.Nodes {
-			if node.Role == k3d.ServerRole {
-				if servers == "" {
-					servers = node.Name
-				} else {
-					servers = fmt.Sprintf("%s,%s", servers, node.Name)
-				}
-			}
+		lbNode, err := LoadbalancerPrepare(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels})
+		if err != nil {
+			return err
 		}
+		cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
 
-		// generate comma-separated list of extra ports to forward
-		ports := []string{k3d.DefaultAPIPort}
-		var udp_ports []string
-		for exposedPort := range cluster.ServerLoadBalancer.Ports {
-			if exposedPort.Proto() == "udp" {
-				udp_ports = append(udp_ports, exposedPort.Port())
-				continue
-			}
-			ports = append(ports, exposedPort.Port())
+		lbConfig, err := LoadbalancerGenerateConfig(cluster)
+		if err != nil {
+			return fmt.Errorf("error generating loadbalancer config: %v", err)
 		}
 
-		if cluster.ServerLoadBalancer.Ports == nil {
-			cluster.ServerLoadBalancer.Ports = nat.PortMap{}
+		// prepare to write config to lb container
+		configyaml, err := yaml.Marshal(lbConfig)
+		if err != nil {
+			return err
 		}
-		cluster.ServerLoadBalancer.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
 
-		// Create LB as a modified node with loadbalancerRole
-		lbNode := &k3d.Node{
-			Name:  fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name),
-			Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()),
-			Ports: cluster.ServerLoadBalancer.Ports,
-			Env: []string{
-				fmt.Sprintf("SERVERS=%s", servers),
-				fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")),
-				fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)),
+		writeLbConfigAction := k3d.NodeHook{
+			Stage: k3d.LifecycleStagePreStart,
+			Action: actions.WriteFileAction{
+				Runtime: runtime,
+				Dest:    k3d.DefaultLoadbalancerConfigPath,
+				Mode:    0744,
+				Content: configyaml,
 			},
-			Role:          k3d.LoadBalancerRole,
-			RuntimeLabels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
-			Networks:      []string{cluster.Network.Name},
-			Restart:       true,
-		}
-		if len(udp_ports) > 0 {
-			lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
 		}
-		cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
+
+		lbNode.HookActions = append(lbNode.HookActions, writeLbConfigAction)
+
 		log.Infof("Creating LoadBalancer '%s'", lbNode.Name)
-		if err := NodeCreate(clusterCreateCtx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil {
-			log.Errorln("Failed to create loadbalancer")
-			return err
+		if err := NodeCreate(ctx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil {
+			return fmt.Errorf("error creating loadbalancer: %v", err)
 		}
 		log.Debugf("Created loadbalancer '%s'", lbNode.Name)
+		return err
 	}
 
 	return nil
@@ -910,7 +890,9 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	log.Infoln("Starting helpers...")
 	failedHelpers := 0
 	for _, helperNode := range aux {
-		nodeStartOpts := k3d.NodeStartOpts{}
+		nodeStartOpts := k3d.NodeStartOpts{
+			NodeHooks: helperNode.HookActions,
+		}
 		if helperNode.Role == k3d.LoadBalancerRole {
 			nodeStartOpts.Wait = true
 		}