Skip to content

Commit

Permalink
[XXL] Config File Enhancements
Browse files Browse the repository at this point in the history
- new version v1alpha3 with k3s extraArgs using node filters
  - reflected in CLI via --k3s-arg
- new migration option to migrate (internally and via cli) from v1alpha2
to v1alpha3
- enhancements to how config files are being read
- e2e tests for config file migration
  • Loading branch information
iwilltry42 committed May 14, 2021
1 parent 770aaa4 commit cef3b17
Show file tree
Hide file tree
Showing 35 changed files with 1,108 additions and 146 deletions.
86 changes: 64 additions & 22 deletions cmd/cluster/clusterCreate.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
cliutil "github.com/rancher/k3d/v4/cmd/util"
k3dCluster "github.com/rancher/k3d/v4/pkg/client"
"github.com/rancher/k3d/v4/pkg/config"
conf "github.com/rancher/k3d/v4/pkg/config/v1alpha2"
conf "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
"github.com/rancher/k3d/v4/pkg/runtimes"
k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/version"
Expand Down Expand Up @@ -77,11 +77,6 @@ func initConfig() {
if _, err := os.Stat(configFile); err != nil {
log.Fatalf("Failed to stat config file %s: %+v", configFile, err)
}
log.Tracef("Schema: %+v", conf.JSONSchema)

if err := config.ValidateSchemaFile(configFile, []byte(conf.JSONSchema)); err != nil {
log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

// try to read config into memory (viper map structure)
if err := cfgViper.ReadInConfig(); err != nil {
Expand All @@ -92,7 +87,16 @@ func initConfig() {
log.Fatalf("Failed to read config file %s: %+v", configFile, err)
}

log.Infof("Using config file %s", cfgViper.ConfigFileUsed())
schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
if err != nil {
log.Fatalf("Cannot validate config file %s: %+v", configFile, err)
}

if err := config.ValidateSchemaFile(configFile, schema); err != nil {
log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

log.Infof("Using config file %s (%s#%s)", cfgViper.ConfigFileUsed(), strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))
}
if log.GetLevel() >= log.DebugLevel {
c, _ := yaml.Marshal(cfgViper.AllSettings())
Expand Down Expand Up @@ -121,30 +125,46 @@ func NewCmdClusterCreate() *cobra.Command {
/*************************
* Compute Configuration *
*************************/
cfg, err := config.FromViperSimple(cfgViper)
if cfgViper.GetString("apiversion") == "" {
cfgViper.Set("apiversion", config.DefaultConfigApiVersion)
}
if cfgViper.GetString("kind") == "" {
cfgViper.Set("kind", "Simple")
}
cfg, err := config.FromViper(cfgViper)
if err != nil {
log.Fatalln(err)
}

log.Debugf("========== Simple Config ==========\n%+v\n==========================\n", cfg)
if cfg.GetAPIVersion() != config.DefaultConfigApiVersion {
log.Warnf("Default config apiVersion is '%s', but you're using '%s': consider migrating.", config.DefaultConfigApiVersion, cfg.GetAPIVersion())
cfg, err = config.Migrate(cfg, config.DefaultConfigApiVersion)
if err != nil {
log.Fatalln(err)
}
}

simpleCfg := cfg.(conf.SimpleConfig)

log.Debugf("========== Simple Config ==========\n%+v\n==========================\n", simpleCfg)

cfg, err = applyCLIOverrides(cfg)
simpleCfg, err = applyCLIOverrides(simpleCfg)
if err != nil {
log.Fatalf("Failed to apply CLI overrides: %+v", err)
}

log.Debugf("========== Merged Simple Config ==========\n%+v\n==========================\n", cfg)
log.Debugf("========== Merged Simple Config ==========\n%+v\n==========================\n", simpleCfg)

/**************************************
* Transform, Process & Validate Configuration *
**************************************/

// Set the name
if len(args) != 0 {
cfg.Name = args[0]
simpleCfg.Name = args[0]
}

clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, cfg)
clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, simpleCfg)
if err != nil {
log.Fatalln(err)
}
Expand Down Expand Up @@ -178,7 +198,7 @@ func NewCmdClusterCreate() *cobra.Command {
if err := k3dCluster.ClusterRun(cmd.Context(), runtimes.SelectedRuntime, clusterConfig); err != nil {
// rollback if creation failed
log.Errorln(err)
if cfg.Options.K3dOptions.NoRollback { // TODO: move rollback mechanics to pkg/
if simpleCfg.Options.K3dOptions.NoRollback { // TODO: move rollback mechanics to pkg/
log.Fatalln("Cluster creation FAILED, rollback deactivated.")
}
// rollback if creation failed
Expand All @@ -202,7 +222,7 @@ func NewCmdClusterCreate() *cobra.Command {

if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig {
log.Debugf("Updating default kubeconfig with a new context for cluster %s", clusterConfig.Cluster.Name)
if _, err := k3dCluster.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, "", &k3dCluster.WriteKubeConfigOptions{UpdateExisting: true, OverwriteExisting: false, UpdateCurrentContext: cfg.Options.KubeconfigOptions.SwitchCurrentContext}); err != nil {
if _, err := k3dCluster.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, "", &k3dCluster.WriteKubeConfigOptions{UpdateExisting: true, OverwriteExisting: false, UpdateCurrentContext: simpleCfg.Options.KubeconfigOptions.SwitchCurrentContext}); err != nil {
log.Warningln(err)
}
}
Expand Down Expand Up @@ -266,6 +286,10 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`")
_ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label"))

/* k3s */
cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server[0]\"")
_ = cfgViper.BindPFlag("cli.k3sargs", cmd.Flags().Lookup("k3s-arg"))

/******************
* "Normal" Flags *
******************
Expand Down Expand Up @@ -340,13 +364,6 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().String("registry-config", "", "Specify path to an extra registries.yaml file")
_ = cfgViper.BindPFlag("registries.config", cmd.Flags().Lookup("registry-config"))

/* k3s */
cmd.Flags().StringArray("k3s-server-arg", nil, "Additional args passed to the `k3s server` command on server nodes (new flag per arg)")
_ = cfgViper.BindPFlag("options.k3s.extraserverargs", cmd.Flags().Lookup("k3s-server-arg"))

cmd.Flags().StringArray("k3s-agent-arg", nil, "Additional args passed to the `k3s agent` command on agent nodes (new flag per arg)")
_ = cfgViper.BindPFlag("options.k3s.extraagentargs", cmd.Flags().Lookup("k3s-agent-arg"))

/* Subcommands */

// done
Expand Down Expand Up @@ -520,5 +537,30 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {

log.Tracef("EnvFilterMap: %+v", envFilterMap)

// --k3s-arg
argFilterMap := make(map[string][]string, 1)
for _, argFlag := range ppViper.GetStringSlice("cli.k3sargs") {

// split node filter from the specified arg
arg, filters, err := cliutil.SplitFiltersFromFlag(argFlag)
if err != nil {
log.Fatalln(err)
}

// create new entry or append filter to existing entry
if _, exists := argFilterMap[arg]; exists {
argFilterMap[arg] = append(argFilterMap[arg], filters...)
} else {
argFilterMap[arg] = filters
}
}

for arg, nodeFilters := range argFilterMap {
cfg.Options.K3sOptions.ExtraArgs = append(cfg.Options.K3sOptions.ExtraArgs, conf.K3sArgWithNodeFilters{
Arg: arg,
NodeFilters: nodeFilters,
})
}

return cfg, nil
}
1 change: 1 addition & 0 deletions cmd/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func NewCmdConfig() *cobra.Command {
}

cmd.AddCommand(NewCmdConfigInit())
cmd.AddCommand(NewCmdConfigMigrate())

return cmd
}
2 changes: 1 addition & 1 deletion cmd/config/configInit.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"fmt"
"os"

config "github.com/rancher/k3d/v4/pkg/config/v1alpha2"
config "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand Down
111 changes: 111 additions & 0 deletions cmd/config/configMigrate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
Copyright © 2020 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package config

import (
"os"
"strings"

"github.com/rancher/k3d/v4/pkg/config"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"
)

// NewCmdConfigMigrate returns a new cobra command
func NewCmdConfigMigrate() *cobra.Command {

cmd := &cobra.Command{
Use: "migrate INPUT OUTPUT",
Args: cobra.RangeArgs(1, 2),
Run: func(cmd *cobra.Command, args []string) {

configFile := args[0]

if _, err := os.Stat(configFile); err != nil {
log.Fatalf("Failed to stat config file %s: %+v", configFile, err)
}

cfgViper := viper.New()
cfgViper.SetConfigType("yaml")

cfgViper.SetConfigFile(configFile)

// try to read config into memory (viper map structure)
if err := cfgViper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
log.Fatalf("Config file %s not found: %+v", configFile, err)
}
// config file found but some other error happened
log.Fatalf("Failed to read config file %s: %+v", configFile, err)
}

schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
if err != nil {
log.Fatalf("Cannot validate config file %s: %+v", configFile, err)
}

if err := config.ValidateSchemaFile(configFile, schema); err != nil {
log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

log.Infof("Using config file %s (%s#%s)", cfgViper.ConfigFileUsed(), strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))

cfg, err := config.FromViper(cfgViper)
if err != nil {
log.Fatalln(err)
}

if cfg.GetAPIVersion() != config.DefaultConfigApiVersion {
cfg, err = config.Migrate(cfg, config.DefaultConfigApiVersion)
if err != nil {
log.Fatalln(err)
}
}

yamlout, err := yaml.Marshal(cfg)
if err != nil {
log.Fatalln(err)
}

output := "-"

if len(args) > 1 {
output = args[1]
}

if output == "-" {
if _, err := os.Stdout.Write(yamlout); err != nil {
log.Fatalln(err)
}
} else {
if err := os.WriteFile(output, yamlout, os.ModeAppend); err != nil {
log.Fatalln(err)
}
}

},
}

return cmd
}
15 changes: 8 additions & 7 deletions docs/usage/configfile.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ Using a config file is as easy as putting it in a well-known place in your file

As of the time of writing this documentation, the config file only **requires** you to define two fields:

- `apiVersion` to match the version of the config file that you want to use (at this time it would be `apiVersion: k3d.io/v1alpha2`)
- `apiVersion` to match the version of the config file that you want to use (at this time it would be `apiVersion: k3d.io/v1alpha3`)
- `kind` to define the kind of config file that you want to use (currently we only have the `Simple` config)

So this would be the minimal config file, which configures absolutely nothing:

```yaml
apiVersion: k3d.io/v1alpha2
apiVersion: k3d.io/v1alpha3
kind: Simple
```
Expand All @@ -43,15 +43,15 @@ Currently, the config file is still in an Alpha-State, meaning, that it is subje
!!! info "Validation via JSON-Schema"
k3d uses a [JSON-Schema](https://json-schema.org/) to describe the expected format and fields of the configuration file.
This schema is also used to [validate](https://github.com/xeipuuv/gojsonschema#validation) a user-given config file.
This JSON-Schema can be found in the specific config version sub-directory in the repository (e.g. [here for `v1alpha2`](https://github.com/rancher/k3d/blob/main/pkg/config/v1alpha2/schema.json)) and could be used to lookup supported fields or by linters to validate the config file, e.g. in your code editor.
This JSON-Schema can be found in the specific config version sub-directory in the repository (e.g. [here for `v1alpha3`](https://github.com/rancher/k3d/blob/main/pkg/config/v1alpha3/schema.json)) and could be used to lookup supported fields or by linters to validate the config file, e.g. in your code editor.

### All Options: Example

Since the config options and the config file are changing quite a bit, it's hard to keep track of all the supported config file settings, so here's an example showing all of them as of the time of writing:

```yaml
# k3d configuration file, saved as e.g. /home/me/myk3dcluster.yaml
apiVersion: k3d.io/v1alpha2 # this will change in the future as we make everything more stable
apiVersion: k3d.io/v1alpha3 # this will change in the future as we make everything more stable
kind: Simple # internally, we also have a Cluster config, which is not yet available externally
name: mycluster # name that you want to give to your cluster (will still be prefixed with `k3d-`)
servers: 1 # same as `--servers 1`
Expand Down Expand Up @@ -98,9 +98,10 @@ options:
disableRollback: false # same as `--no-Rollback`
disableHostIPInjection: false # same as `--no-hostip`
k3s: # options passed on to K3s itself
extraServerArgs: # additional arguments passed to the `k3s server` command; same as `--k3s-server-arg`
- --tls-san=my.host.domain
extraAgentArgs: [] # addditional arguments passed to the `k3s agent` command; same as `--k3s-agent-arg`
extraArgs: # additional arguments passed to the `k3s server|agent` command; same as `--k3s-arg`
- arg: --tls-san=my.host.domain
nodeFilters:
- server[*]
kubeconfig:
updateDefaultKubeconfig: true # add new cluster to your default Kubeconfig; same as `--kubeconfig-update-default` (default: true)
switchCurrentContext: true # also set current-context to the new cluster's context; same as `--kubeconfig-switch-context` (default: true)
Expand Down
2 changes: 1 addition & 1 deletion docs/usage/guides/registries.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ This file can also be used for providing additional information necessary for ac
If you're using a `SimpleConfig` file to configure your k3d cluster, you may as well embed the registries.yaml in there directly:

```yaml
apiVersion: k3d.io/v1alpha2
apiVersion: k3d.io/v1alpha3
kind: Simple
name: test
servers: 1
Expand Down
2 changes: 1 addition & 1 deletion pkg/client/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import (
"github.com/docker/go-connections/nat"
"github.com/imdario/mergo"
"github.com/rancher/k3d/v4/pkg/actions"
config "github.com/rancher/k3d/v4/pkg/config/v1alpha2"
config "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
k3drt "github.com/rancher/k3d/v4/pkg/runtimes"
"github.com/rancher/k3d/v4/pkg/runtimes/docker"
runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
Expand Down
Loading

0 comments on commit cef3b17

Please sign in to comment.