Skip to content

Commit

Permalink
[BREAKING] Config File Enhancements: v1alpha3, migrations, generic k3…
Browse files Browse the repository at this point in the history
…s-args (#605)

Excerpt:
- new version v1alpha3 with k3s extraArgs using node filters
  - reflected in CLI via --k3s-arg
- new migration option to migrate (internally and via cli) from v1alpha2
to v1alpha3
- enhancements to how config files are being read
- e2e tests for config file migration
  • Loading branch information
iwilltry42 committed Jul 9, 2021
1 parent 296f24c commit 261ac0f
Show file tree
Hide file tree
Showing 40 changed files with 1,126 additions and 161 deletions.
86 changes: 64 additions & 22 deletions cmd/cluster/clusterCreate.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
cliutil "github.com/rancher/k3d/v4/cmd/util"
k3dCluster "github.com/rancher/k3d/v4/pkg/client"
"github.com/rancher/k3d/v4/pkg/config"
conf "github.com/rancher/k3d/v4/pkg/config/v1alpha2"
conf "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
"github.com/rancher/k3d/v4/pkg/runtimes"
k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/version"
Expand Down Expand Up @@ -77,11 +77,6 @@ func initConfig() {
if _, err := os.Stat(configFile); err != nil {
log.Fatalf("Failed to stat config file %s: %+v", configFile, err)
}
log.Tracef("Schema: %+v", conf.JSONSchema)

if err := config.ValidateSchemaFile(configFile, []byte(conf.JSONSchema)); err != nil {
log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

// try to read config into memory (viper map structure)
if err := cfgViper.ReadInConfig(); err != nil {
Expand All @@ -92,7 +87,16 @@ func initConfig() {
log.Fatalf("Failed to read config file %s: %+v", configFile, err)
}

log.Infof("Using config file %s", cfgViper.ConfigFileUsed())
schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
if err != nil {
log.Fatalf("Cannot validate config file %s: %+v", configFile, err)
}

if err := config.ValidateSchemaFile(configFile, schema); err != nil {
log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

log.Infof("Using config file %s (%s#%s)", cfgViper.ConfigFileUsed(), strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))
}
if log.GetLevel() >= log.DebugLevel {
c, _ := yaml.Marshal(cfgViper.AllSettings())
Expand Down Expand Up @@ -121,30 +125,46 @@ func NewCmdClusterCreate() *cobra.Command {
/*************************
* Compute Configuration *
*************************/
cfg, err := config.FromViperSimple(cfgViper)
if cfgViper.GetString("apiversion") == "" {
cfgViper.Set("apiversion", config.DefaultConfigApiVersion)
}
if cfgViper.GetString("kind") == "" {
cfgViper.Set("kind", "Simple")
}
cfg, err := config.FromViper(cfgViper)
if err != nil {
log.Fatalln(err)
}

log.Debugf("========== Simple Config ==========\n%+v\n==========================\n", cfg)
if cfg.GetAPIVersion() != config.DefaultConfigApiVersion {
log.Warnf("Default config apiVersion is '%s', but you're using '%s': consider migrating.", config.DefaultConfigApiVersion, cfg.GetAPIVersion())
cfg, err = config.Migrate(cfg, config.DefaultConfigApiVersion)
if err != nil {
log.Fatalln(err)
}
}

simpleCfg := cfg.(conf.SimpleConfig)

log.Debugf("========== Simple Config ==========\n%+v\n==========================\n", simpleCfg)

cfg, err = applyCLIOverrides(cfg)
simpleCfg, err = applyCLIOverrides(simpleCfg)
if err != nil {
log.Fatalf("Failed to apply CLI overrides: %+v", err)
}

log.Debugf("========== Merged Simple Config ==========\n%+v\n==========================\n", cfg)
log.Debugf("========== Merged Simple Config ==========\n%+v\n==========================\n", simpleCfg)

/**************************************
* Transform, Process & Validate Configuration *
**************************************/

// Set the name
if len(args) != 0 {
cfg.Name = args[0]
simpleCfg.Name = args[0]
}

clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, cfg)
clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, simpleCfg)
if err != nil {
log.Fatalln(err)
}
Expand Down Expand Up @@ -178,7 +198,7 @@ func NewCmdClusterCreate() *cobra.Command {
if err := k3dCluster.ClusterRun(cmd.Context(), runtimes.SelectedRuntime, clusterConfig); err != nil {
// rollback if creation failed
log.Errorln(err)
if cfg.Options.K3dOptions.NoRollback { // TODO: move rollback mechanics to pkg/
if simpleCfg.Options.K3dOptions.NoRollback { // TODO: move rollback mechanics to pkg/
log.Fatalln("Cluster creation FAILED, rollback deactivated.")
}
// rollback if creation failed
Expand All @@ -202,7 +222,7 @@ func NewCmdClusterCreate() *cobra.Command {

if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig {
log.Debugf("Updating default kubeconfig with a new context for cluster %s", clusterConfig.Cluster.Name)
if _, err := k3dCluster.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, "", &k3dCluster.WriteKubeConfigOptions{UpdateExisting: true, OverwriteExisting: false, UpdateCurrentContext: cfg.Options.KubeconfigOptions.SwitchCurrentContext}); err != nil {
if _, err := k3dCluster.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, "", &k3dCluster.WriteKubeConfigOptions{UpdateExisting: true, OverwriteExisting: false, UpdateCurrentContext: simpleCfg.Options.KubeconfigOptions.SwitchCurrentContext}); err != nil {
log.Warningln(err)
}
}
Expand Down Expand Up @@ -266,6 +286,10 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`")
_ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label"))

/* k3s */
cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server[0]\"")
_ = cfgViper.BindPFlag("cli.k3sargs", cmd.Flags().Lookup("k3s-arg"))

/******************
* "Normal" Flags *
******************
Expand Down Expand Up @@ -340,13 +364,6 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().String("registry-config", "", "Specify path to an extra registries.yaml file")
_ = cfgViper.BindPFlag("registries.config", cmd.Flags().Lookup("registry-config"))

/* k3s */
cmd.Flags().StringArray("k3s-server-arg", nil, "Additional args passed to the `k3s server` command on server nodes (new flag per arg)")
_ = cfgViper.BindPFlag("options.k3s.extraserverargs", cmd.Flags().Lookup("k3s-server-arg"))

cmd.Flags().StringArray("k3s-agent-arg", nil, "Additional args passed to the `k3s agent` command on agent nodes (new flag per arg)")
_ = cfgViper.BindPFlag("options.k3s.extraagentargs", cmd.Flags().Lookup("k3s-agent-arg"))

/* Subcommands */

// done
Expand Down Expand Up @@ -520,5 +537,30 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {

log.Tracef("EnvFilterMap: %+v", envFilterMap)

// --k3s-arg
argFilterMap := make(map[string][]string, 1)
for _, argFlag := range ppViper.GetStringSlice("cli.k3sargs") {

// split node filter from the specified arg
arg, filters, err := cliutil.SplitFiltersFromFlag(argFlag)
if err != nil {
log.Fatalln(err)
}

// create new entry or append filter to existing entry
if _, exists := argFilterMap[arg]; exists {
argFilterMap[arg] = append(argFilterMap[arg], filters...)
} else {
argFilterMap[arg] = filters
}
}

for arg, nodeFilters := range argFilterMap {
cfg.Options.K3sOptions.ExtraArgs = append(cfg.Options.K3sOptions.ExtraArgs, conf.K3sArgWithNodeFilters{
Arg: arg,
NodeFilters: nodeFilters,
})
}

return cfg, nil
}
1 change: 1 addition & 0 deletions cmd/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func NewCmdConfig() *cobra.Command {
}

cmd.AddCommand(NewCmdConfigInit())
cmd.AddCommand(NewCmdConfigMigrate())

return cmd
}
2 changes: 1 addition & 1 deletion cmd/config/configInit.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"fmt"
"os"

config "github.com/rancher/k3d/v4/pkg/config/v1alpha2"
config "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand Down
111 changes: 111 additions & 0 deletions cmd/config/configMigrate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
Copyright © 2020 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package config

import (
"os"
"strings"

"github.com/rancher/k3d/v4/pkg/config"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"
)

// NewCmdConfigMigrate returns a new cobra command
func NewCmdConfigMigrate() *cobra.Command {

cmd := &cobra.Command{
Use: "migrate INPUT OUTPUT",
Args: cobra.RangeArgs(1, 2),
Run: func(cmd *cobra.Command, args []string) {

configFile := args[0]

if _, err := os.Stat(configFile); err != nil {
log.Fatalf("Failed to stat config file %s: %+v", configFile, err)
}

cfgViper := viper.New()
cfgViper.SetConfigType("yaml")

cfgViper.SetConfigFile(configFile)

// try to read config into memory (viper map structure)
if err := cfgViper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
log.Fatalf("Config file %s not found: %+v", configFile, err)
}
// config file found but some other error happened
log.Fatalf("Failed to read config file %s: %+v", configFile, err)
}

schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
if err != nil {
log.Fatalf("Cannot validate config file %s: %+v", configFile, err)
}

if err := config.ValidateSchemaFile(configFile, schema); err != nil {
log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

log.Infof("Using config file %s (%s#%s)", cfgViper.ConfigFileUsed(), strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))

cfg, err := config.FromViper(cfgViper)
if err != nil {
log.Fatalln(err)
}

if cfg.GetAPIVersion() != config.DefaultConfigApiVersion {
cfg, err = config.Migrate(cfg, config.DefaultConfigApiVersion)
if err != nil {
log.Fatalln(err)
}
}

yamlout, err := yaml.Marshal(cfg)
if err != nil {
log.Fatalln(err)
}

output := "-"

if len(args) > 1 {
output = args[1]
}

if output == "-" {
if _, err := os.Stdout.Write(yamlout); err != nil {
log.Fatalln(err)
}
} else {
if err := os.WriteFile(output, yamlout, os.ModeAppend); err != nil {
log.Fatalln(err)
}
}

},
}

return cmd
}
2 changes: 2 additions & 0 deletions docgen/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ The code will output files in [`../docs/usage/commands/`](../docs/usage/commands

## Run

- may required a `replace github.com/rancher/k3d/v4 => PATH/TO/LOCAL/REPO` in the `go.mod`

```bash
# ensure that you're in the docgen dir, as the relative path to the docs/ dir is hardcoded
cd docgen
Expand Down
4 changes: 2 additions & 2 deletions docs/usage/commands/k3d_cluster_create.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ k3d cluster create NAME [flags]
--gpus string GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]
-h, --help help for create
-i, --image string Specify k3s image that you want to use for the nodes
--k3s-agent-arg k3s agent Additional args passed to the k3s agent command on agent nodes (new flag per arg)
--k3s-server-arg k3s server Additional args passed to the k3s server command on server nodes (new flag per arg)
--k3s-arg ARG@NODEFILTER[;@NODEFILTER] Additional args passed to k3s command (Format: ARG@NODEFILTER[;@NODEFILTER])
- Example: `k3d cluster create --k3s-arg "--disable=traefik@server[0]"
--kubeconfig-switch-context Directly switch the default kubeconfig's current-context to the new cluster's context (requires --kubeconfig-update-default) (default true)
--kubeconfig-update-default Directly update the default kubeconfig with the new cluster's context (default true)
-l, --label KEY[=VALUE][@NODEFILTER[;NODEFILTER...]] Add label to node container (Format: KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]
Expand Down
1 change: 1 addition & 0 deletions docs/usage/commands/k3d_config.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ k3d config [flags]

* [k3d](k3d.md) - https://k3d.io/ -> Run k3s in Docker!
* [k3d config init](k3d_config_init.md) -
* [k3d config migrate](k3d_config_migrate.md) -

Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
## k3d docgen
## k3d config migrate


Generate command docs

```
k3d docgen [flags]
k3d config migrate INPUT OUTPUT [flags]
```

### Options

```
-h, --help help for docgen
-h, --help help for migrate
```

### Options inherited from parent commands
Expand All @@ -22,5 +22,5 @@ k3d docgen [flags]

### SEE ALSO

* [k3d](k3d.md) - https://k3d.io/ -> Run k3s in Docker!
* [k3d config](k3d_config.md) - Work with config file(s)

17 changes: 9 additions & 8 deletions docs/usage/commands/k3d_node_create.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,15 @@ k3d node create NAME [flags]
### Options

```
-c, --cluster string Select the cluster that the node shall connect to. (default "k3s-default")
-h, --help help for create
-i, --image string Specify k3s image used for the node(s) (default "docker.io/rancher/k3s:v1.20.0-k3s2")
--memory string Memory limit imposed on the node [From docker]
--replicas int Number of replicas of this node specification. (default 1)
--role string Specify node role [server, agent] (default "agent")
--timeout duration Maximum waiting time for '--wait' before canceling/returning.
--wait Wait for the node(s) to be ready before returning.
-c, --cluster string Select the cluster that the node shall connect to. (default "k3s-default")
-h, --help help for create
-i, --image string Specify k3s image used for the node(s) (default "docker.io/rancher/k3s:v1.20.0-k3s2")
--k3s-node-label strings Specify k3s node labels in format "foo=bar"
--memory string Memory limit imposed on the node [From docker]
--replicas int Number of replicas of this node specification. (default 1)
--role string Specify node role [server, agent] (default "agent")
--timeout duration Maximum waiting time for '--wait' before canceling/returning.
--wait Wait for the node(s) to be ready before returning.
```

### Options inherited from parent commands
Expand Down
Loading

0 comments on commit 261ac0f

Please sign in to comment.