From 7f095d48aa91e73d35841d114b541c7afc5eb355 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 6 Dec 2018 14:08:11 +0100 Subject: [PATCH 1/7] multi node config --- cmd/kind/create/cluster/createcluster.go | 10 +- hack/update-generated.sh | 1 - pkg/cluster/config/encoding/scheme.go | 181 ++++++++++++--- pkg/cluster/config/encoding/scheme_test.go | 199 +++++++++++++--- pkg/cluster/config/fuzzer/fuzzer.go | 12 +- pkg/cluster/config/helpers.go | 213 ++++++++++++++++++ pkg/cluster/config/helpers_test.go | 189 ++++++++++++++++ pkg/cluster/config/register.go | 12 +- pkg/cluster/config/types.go | 96 +++++++- pkg/cluster/config/v1alpha1/conversion.go | 41 ++++ pkg/cluster/config/v1alpha1/register.go | 10 - .../v1alpha1/zz_generated.conversion.go | 176 --------------- pkg/cluster/config/v1alpha2/default.go | 8 +- pkg/cluster/config/v1alpha2/register.go | 12 +- pkg/cluster/config/v1alpha2/types.go | 33 ++- pkg/cluster/config/validate.go | 56 ++++- pkg/cluster/config/validate_test.go | 181 +++++++++++---- pkg/cluster/config/zz_generated.deepcopy.go | 78 ++++--- pkg/cluster/context.go | 30 +-- 19 files changed, 1163 insertions(+), 375 deletions(-) create mode 100644 pkg/cluster/config/helpers.go create mode 100644 pkg/cluster/config/helpers_test.go create mode 100644 pkg/cluster/config/v1alpha1/conversion.go delete mode 100644 pkg/cluster/config/v1alpha1/zz_generated.conversion.go diff --git a/cmd/kind/create/cluster/createcluster.go b/cmd/kind/create/cluster/createcluster.go index 8769595c48..8e27f1d27a 100644 --- a/cmd/kind/create/cluster/createcluster.go +++ b/cmd/kind/create/cluster/createcluster.go @@ -74,10 +74,18 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error { return fmt.Errorf("aborting due to invalid configuration") } + // TODO(fabrizio pandini): this check is temporary / WIP + // kind v1alpha config fully supports multi nodes, but the cluster creation logic implemented in + // pkg/cluster/contex.go does not (yet). + // As soon a multi node support is implemented in pkg/cluster/contex.go, this should go away + if len(cfg.Nodes()) > 1 { + return fmt.Errorf("multi node support is still a work in progress, currently only single node cluster are supported") + } + // create a cluster context and create the cluster ctx := cluster.NewContext(flags.Name) if flags.ImageName != "" { - cfg.Image = flags.ImageName + cfg.BootStrapControlPlane().Image = flags.ImageName err := cfg.Validate() if err != nil { log.Errorf("Invalid flags, configuration failed validation: %v", err) diff --git a/hack/update-generated.sh b/hack/update-generated.sh index 14fb4480fe..9b8933449e 100755 --- a/hack/update-generated.sh +++ b/hack/update-generated.sh @@ -36,7 +36,6 @@ deepcopy-gen -i ./pkg/cluster/config/ -O zz_generated.deepcopy --go-header-file deepcopy-gen -i ./pkg/cluster/config/v1alpha1 -O zz_generated.deepcopy --go-header-file hack/boilerplate.go.txt defaulter-gen -i ./pkg/cluster/config/v1alpha1 -O zz_generated.default --go-header-file hack/boilerplate.go.txt -conversion-gen -i ./pkg/cluster/config/v1alpha1 -O zz_generated.conversion --go-header-file hack/boilerplate.go.txt deepcopy-gen -i ./pkg/cluster/config/v1alpha2 -O zz_generated.deepcopy --go-header-file hack/boilerplate.go.txt defaulter-gen -i ./pkg/cluster/config/v1alpha2 -O zz_generated.default --go-header-file hack/boilerplate.go.txt diff --git a/pkg/cluster/config/encoding/scheme.go b/pkg/cluster/config/encoding/scheme.go index ec4f58fa03..70cffd2276 100644 --- a/pkg/cluster/config/encoding/scheme.go +++ b/pkg/cluster/config/encoding/scheme.go @@ -17,13 +17,20 @@ limitations under the License. package encoding import ( + "bufio" + "bytes" + "fmt" + "io" "io/ioutil" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" "sigs.k8s.io/kind/pkg/cluster/config" "sigs.k8s.io/kind/pkg/cluster/config/v1alpha1" @@ -48,56 +55,168 @@ func AddToScheme(scheme *runtime.Scheme) { utilruntime.Must(scheme.SetVersionPriority(v1alpha2.SchemeGroupVersion)) } +// Load reads the file at path and attempts to convert into a `kind` Config; the file +// can be one of the different API versions defined in scheme. +// If path == "" then the default config is returned +func Load(path string) (*config.Config, error) { + if path == "" { + return newDefaultedConfig(), nil + } + + // read in file + contents, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + // unmarshal the file content into a `kind` Config + return unmarshalConfig(contents) +} + // newDefaultedConfig creates a new, defaulted `kind` Config -// Defaulting uses Scheme registered defaulting functions +// with one control-plane node. func newDefaultedConfig() *config.Config { - var cfg = &v1alpha2.Config{} + var cfg = &v1alpha2.Node{Role: v1alpha2.ControlPlaneRole} // apply defaults Scheme.Default(cfg) - // converts to internal cfg + // converts to internal node + var internalNode = &config.Node{} + Scheme.Convert(cfg, internalNode, nil) + + // creates the internal cfg and add the node var internalCfg = &config.Config{} - Scheme.Convert(cfg, internalCfg, nil) + internalCfg.Add(internalNode) return internalCfg } -// unmarshalConfig attempt to decode data into a `kind` Config; data can be -// one of the different API versions defined in the Scheme. -func unmarshalConfig(data []byte) (*config.Config, error) { - var cfg = &v1alpha2.Config{} +// yamlDocument identifies a yaml document contained in a yaml file +// by its own GroupVersionKind +type yamlDocument struct { + GroupVersionKind schema.GroupVersionKind + Contents []byte +} - // decode data into a config object - _, _, err := Codecs.UniversalDecoder().Decode(data, nil, cfg) +// unmarshalConfig attempt to decode data into a `kind` Config; data can be +// one of the different API versions defined in the Scheme; for v1alpha2 +// multiple yaml documents within the same yaml file are supported +func unmarshalConfig(contents []byte) (*config.Config, error) { + // parses yamlDocuments separated by --- directives + yamlDocuments, err := splitYAMLDocuments(contents) if err != nil { - return nil, errors.Wrap(err, "decoding failure") + return nil, err } - // apply defaults - Scheme.Default(cfg) - - // converts to internal cfg - var internalCfg = &config.Config{} - Scheme.Convert(cfg, internalCfg, nil) + // checks if v1alpha1 config is present in the yamlDocuments + var v1alpha1Config = false + for _, doc := range yamlDocuments { + if doc.GroupVersionKind.GroupVersion() == v1alpha1.SchemeGroupVersion { + v1alpha1Config = true + } + } - return internalCfg, nil -} + // if using v1alpha1Config, + if v1alpha1Config { + // only one yaml document is supported + if len(yamlDocuments) > 1 { + return nil, fmt.Errorf("if using v1alpha1 config, only one yaml document should be provided") + } + + // decode the yaml into a config object + var v1alpha1Cfg = &v1alpha1.Config{} + _, _, err := Codecs.UniversalDecoder().Decode(yamlDocuments[0].Contents, nil, v1alpha1Cfg) + if err != nil { + return nil, errors.Wrap(err, "decoding failure") + } + + // apply defaults + Scheme.Default(v1alpha1Cfg) + + // converts to internal cfg. this will give Config with a single node with control-plane role + // NB. we are using custom conversion due to the complexity of conversion from v1alpha1 config + var internalCfg = &config.Config{} + v1alpha1Cfg.Convert(internalCfg) + + return internalCfg, nil + } -// Load reads the file at path and attempts to convert into a `kind` Config; the file -// can be one of the different API versions defined in scheme. -// If path == "" then the default config is returned -func Load(path string) (*config.Config, error) { - if path == "" { - return newDefaultedConfig(), nil + // Otherwise it is a multi node, v1alpha2 config + var cfg = &config.Config{} + + // Process all the yamlDocuments + for _, doc := range yamlDocuments { + // decode data into a Node object + var v1alpha2node = &v1alpha2.Node{} + _, _, err := Codecs.UniversalDecoder().Decode(doc.Contents, nil, v1alpha2node) + if err != nil { + return nil, errors.Wrap(err, "decoding failure") + } + + // apply defaults + Scheme.Default(v1alpha2node) + + // converts to internal cfg + var internalNode = &config.Node{} + Scheme.Convert(v1alpha2node, internalNode, nil) + + // adds the node to the internal config + // in case replicas shuold be generated for the nodes, Add will takes care. + if err := cfg.Add(internalNode); err != nil { + return nil, err + } } - // read in file - contents, err := ioutil.ReadFile(path) - if err != nil { - return nil, err + return cfg, nil +} + +// splitYAMLDocuments divides a yaml file into yamlDocuments, +// and identifies each document with its own GroupVersionKind +func splitYAMLDocuments(contents []byte) (yamlDocuments []yamlDocument, err error) { + yamlDocuments = []yamlDocument{} + + // parses yamlDocuments separated by --- directives + buf := bytes.NewBuffer(contents) + reader := utilyaml.NewYAMLReader(bufio.NewReader(buf)) + for { + // Read one YAML document at a time, until io.EOF is returned + documentContents, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + if len(documentContents) == 0 { + break + } + + // Unmarshal the TypeMeta information of this byte slice + typeMetaInfo := runtime.TypeMeta{} + if err := yaml.Unmarshal(documentContents, &typeMetaInfo); err != nil { + return nil, err + } + + // Require TypeMeta information to be present + if len(typeMetaInfo.APIVersion) == 0 || len(typeMetaInfo.Kind) == 0 { + return nil, errors.New("kind and apiVersion are mandatory information that needs to be specified in all YAML documents") + } + + // Build a GroupVersionKind object from the TypeMeta object + documentGroupVersion, err := schema.ParseGroupVersion(typeMetaInfo.APIVersion) + if err != nil { + return nil, errors.Wrap(err, "unable to parse apiVersion") + } + documentGroupVersionKind := documentGroupVersion.WithKind(typeMetaInfo.Kind) + + // checks that the Kind is a known type + if !Scheme.Recognizes(documentGroupVersionKind) { + return nil, errors.Errorf("unknown %q object type", documentGroupVersionKind) + } + + // add the document to the list of documents + yamlDocuments = append(yamlDocuments, yamlDocument{GroupVersionKind: documentGroupVersionKind, Contents: documentContents}) } - // unmarshal the file content into a `kind` Config - return unmarshalConfig(contents) + return yamlDocuments, nil } diff --git a/pkg/cluster/config/encoding/scheme_test.go b/pkg/cluster/config/encoding/scheme_test.go index c3c8f4d68d..742f71d5dd 100644 --- a/pkg/cluster/config/encoding/scheme_test.go +++ b/pkg/cluster/config/encoding/scheme_test.go @@ -17,76 +17,213 @@ limitations under the License. package encoding import ( + "io/ioutil" "reflect" "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" ) -// TODO(fabrizio pandini): once we have multiple config API versions we -// will need more tests +func TestSplitYAMLDocuments(t *testing.T) { + cases := []struct { + TestName string + Path string + ExpectDocuments []schema.GroupVersionKind + ExpectError bool + }{ + { + TestName: "One yaml document", + Path: "./testdata/v1alpha1/valid-minimal.yaml", + ExpectDocuments: []schema.GroupVersionKind{ + {Group: "kind.sigs.k8s.io", Version: "v1alpha1", Kind: "Config"}, + }, + ExpectError: false, + }, + { + TestName: "Two yaml documents", + Path: "./testdata/v1alpha2/valid-minimal-two-nodes.yaml", + ExpectDocuments: []schema.GroupVersionKind{ + {Group: "kind.sigs.k8s.io", Version: "v1alpha2", Kind: "Node"}, + {Group: "kind.sigs.k8s.io", Version: "v1alpha2", Kind: "Node"}, + }, + ExpectError: false, + }, + { + TestName: "No kind is specified", + Path: "./testdata/invalid-no-kind.yaml", + ExpectError: true, + }, + { + TestName: "No apiVersion is specified", + Path: "./testdata/invalid-yaml.yaml", + ExpectError: true, + }, + { + TestName: "Invalid apiversion", + Path: "./testdata/invalid-apiversion.yaml", + ExpectError: true, + }, + { + TestName: "Invalid kind", + Path: "./testdata/invalid-kind.yaml", + ExpectError: true, + }, + { + TestName: "Invalid yaml", + Path: "./testdata/invalid-yaml.yaml", + ExpectError: true, + }, + } + for _, c := range cases { + t.Run(c.TestName, func(t2 *testing.T) { + contents, err := ioutil.ReadFile(c.Path) + if err != nil { + t2.Fatalf("unexpected error while reading test file: %v", err) + } + + yamlDocuments, err := splitYAMLDocuments(contents) + // the error can be: + // - nil, in which case we should expect no errors or fail + if err != nil { + if !c.ExpectError { + t2.Fatalf("unexpected error while adding Nodes: %v", err) + } + } + // - not nil, in which case we should expect errors or fail + if err == nil { + if c.ExpectError { + t2.Fatalf("unexpected lack or error while adding Nodes") + } + } + + // checks that all the expected yamlDocuments are there + if len(c.ExpectDocuments) != len(yamlDocuments) { + t2.Errorf("expected %d documents, saw %d", len(c.ExpectDocuments), len(yamlDocuments)) + } + + // checks that GroupVersionKind for each yamlDocuments + for i, expectDocument := range c.ExpectDocuments { + if !reflect.DeepEqual(expectDocument, yamlDocuments[i].GroupVersionKind) { + t2.Errorf("Invalid document in position %d: expected %v, saw: %v", i, expectDocument, yamlDocuments[i].GroupVersionKind) + } + } + }) + } +} func TestLoadCurrent(t *testing.T) { cases := []struct { - Name string + TestName string Path string + ExpectNodes []string ExpectError bool }{ { - Name: "v1alpha1 valid minimal", + TestName: "no config", + Path: "", + ExpectNodes: []string{"control-plane"}, // no config (empty config path) should return a single node cluster + ExpectError: false, + }, + { + TestName: "v1alpha1 minimal", Path: "./testdata/v1alpha1/valid-minimal.yaml", + ExpectNodes: []string{"control-plane"}, ExpectError: false, }, { - Name: "v1alpha1 valid with lifecyclehooks", + TestName: "v1alpha1 with lifecyclehooks", Path: "./testdata/v1alpha1/valid-with-lifecyclehooks.yaml", + ExpectNodes: []string{"control-plane"}, ExpectError: false, }, { - Name: "v1alpha2 valid minimal", + TestName: "v1alpha1 with more than one doc", + Path: "./testdata/v1alpha1/invalid-minimal-two-nodes.yaml", + ExpectError: true, + }, + { + TestName: "v1alpha2 minimal", Path: "./testdata/v1alpha2/valid-minimal.yaml", + ExpectNodes: []string{"control-plane"}, ExpectError: false, }, { - Name: "v1alpha2 valid with lifecyclehooks", + TestName: "v1alpha2 lifecyclehooks", Path: "./testdata/v1alpha2/valid-with-lifecyclehooks.yaml", + ExpectNodes: []string{"control-plane"}, + ExpectError: false, + }, + { + TestName: "v1alpha2 config with 2 nodes", + Path: "./testdata/v1alpha2/valid-minimal-two-nodes.yaml", + ExpectNodes: []string{"control-plane", "worker"}, + ExpectError: false, + }, + { + TestName: "v1alpha2 full HA", + Path: "./testdata/v1alpha2/valid-full-ha.yaml", + ExpectNodes: []string{"etcd", "lb", "control-plane1", "control-plane2", "control-plane3", "worker1", "worker2"}, ExpectError: false, }, { - Name: "invalid path", + TestName: "invalid path", Path: "./testdata/not-a-file.bogus", ExpectError: true, }, { - Name: "invalid apiVersion", + TestName: "No kind is specified", + Path: "./testdata/invalid-no-kind.yaml", + ExpectError: true, + }, + { + TestName: "No apiVersion is specified", + Path: "./testdata/invalid-yaml.yaml", + ExpectError: true, + }, + { + TestName: "Invalid apiversion", Path: "./testdata/invalid-apiversion.yaml", ExpectError: true, }, { - Name: "invalid yaml", + TestName: "Invalid kind", + Path: "./testdata/invalid-kind.yaml", + ExpectError: true, + }, + { + TestName: "Invalid yaml", Path: "./testdata/invalid-yaml.yaml", ExpectError: true, }, } - for _, tc := range cases { - _, err := Load(tc.Path) - if err != nil && !tc.ExpectError { - t.Errorf("case: '%s' got error loading and expected none: %v", tc.Name, err) - } else if err == nil && tc.ExpectError { - t.Errorf("case: '%s' got no error loading but expected one", tc.Name) - } - } -} + for _, c := range cases { + t.Run(c.TestName, func(t2 *testing.T) { + cfg, err := Load(c.Path) -func TestLoadDefault(t *testing.T) { - cfg, err := Load("") - if err != nil { - t.Errorf("got error loading default config but expected none: %v", err) - t.FailNow() - } - defaultConfig := newDefaultedConfig() - if !reflect.DeepEqual(cfg, defaultConfig) { - t.Errorf( - "Load(\"\") should match config.New() but does not: %v != %v", - cfg, defaultConfig, - ) + // the error can be: + // - nil, in which case we should expect no errors or fail + if err != nil { + if !c.ExpectError { + t2.Fatalf("unexpected error while unmarshalConfig: %v", err) + } + return + } + // - not nil, in which case we should expect errors or fail + if err == nil { + if c.ExpectError { + t2.Fatalf("unexpected lack or error while unmarshalConfig") + } + } + + if len(cfg.Nodes()) != len(c.ExpectNodes) { + t2.Errorf("expected %d nodes, saw %d", len(c.ExpectNodes), len(cfg.Nodes())) + } + + for i, name := range c.ExpectNodes { + if cfg.Nodes()[i].Name != name { + t2.Errorf("expected %q node at position %d, saw %q", name, i, cfg.Nodes()[i].Name) + } + } + }) } } diff --git a/pkg/cluster/config/fuzzer/fuzzer.go b/pkg/cluster/config/fuzzer/fuzzer.go index 4eae9a09f8..e1291e8ecb 100644 --- a/pkg/cluster/config/fuzzer/fuzzer.go +++ b/pkg/cluster/config/fuzzer/fuzzer.go @@ -27,13 +27,17 @@ import ( // Funcs returns custom fuzzer functions for the `kind` Config. func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { return []interface{}{ - fuzzConfig, + fuzzNode, } } -func fuzzConfig(obj *config.Config, c fuzz.Continue) { +func fuzzNode(obj *config.Node, c fuzz.Continue) { c.FuzzNoCustom(obj) - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.Image = "fuzzimage:latest" + // Pinning values for fields that get defaults if fuzz value is empty string or nil + obj.Image = "foo:bar" + obj.Role = "baz" + + // Pinning default values for `kind` internal state fields + obj.Name = "" } diff --git a/pkg/cluster/config/helpers.go b/pkg/cluster/config/helpers.go new file mode 100644 index 0000000000..9d647d5545 --- /dev/null +++ b/pkg/cluster/config/helpers.go @@ -0,0 +1,213 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "sort" + + "github.com/pkg/errors" +) + +// IsControlPlane returns true if the node hosts a control plane instance +func (n *Node) IsControlPlane() bool { + return n.Role == ControlPlaneRole +} + +// IsWorker returns true if the node hosts a worker instance +func (n *Node) IsWorker() bool { + return n.Role == WorkerRole +} + +// IsExternalEtcd returns true if the node hosts an external etcd member +func (n *Node) IsExternalEtcd() bool { + return n.Role == ExternalEtcdRole +} + +// IsExternalLoadBalancer returns true if the node hosts an external load balancer +func (n *Node) IsExternalLoadBalancer() bool { + return n.Role == ExternalLoadBalancerRole +} + +// ProvisioningOrder returns the provisioning order for nodes, that +// should be defined according to the assigned NodeRole +func (n *Node) ProvisioningOrder() int { + switch n.Role { + // External dependencies should be provisioned first; we are defining an arbitrary + // precedence between etcd and load balancer in order to get predictable/repeatable results + case ExternalEtcdRole: + return 1 + case ExternalLoadBalancerRole: + return 2 + // Then control plane nodes + case ControlPlaneRole: + return 3 + // Finally workers + case WorkerRole: + return 4 + default: + return 99 + } +} + +// Len of the NodeList. +// It is required for making NodeList sortable. +func (t NodeList) Len() int { + return len(t) +} + +// Less return the lower between two elements of the NodeList, where the +// lower element should be provisioned before the other. +// It is required for making NodeList sortable. +func (t NodeList) Less(i, j int) bool { + return t[i].ProvisioningOrder() < t[j].ProvisioningOrder() || + // In case of same provisioning order, the name is used to get predictable/repeatable results + (t[i].ProvisioningOrder() == t[j].ProvisioningOrder() && t[i].Name < t[j].Name) +} + +// Swap two elements of the NodeList. +// It is required for making NodeList sortable. +func (t NodeList) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +// Add a Node to the `kind` cluster and assignes a unique node name. +// If the node should have replicas, more instances of the node are created as well +func (c *Config) Add(node *Node) error { + // defines the list of expected replica of the node that by default are one (the node itself) + replicas := NodeList{node} + + // in case the node should have replicas + if node.Replicas != nil { + // generate expected replicas + replicas = NodeList{} + for i := 1; i <= int(*node.Replicas); i++ { + replicas = append(replicas, node.DeepCopy()) + } + } + + // adds replica of the node to the config + for _, replica := range replicas { + + // adds the replica to the list of nodes + c.nodes = append(c.nodes, replica) + + // updates derivedConfigData + + // list of nodes with control plane role + if replica.IsControlPlane() { + // assign selected name for control plane node + replica.Name = "control-plane" + // stores the node in derivedConfigData + c.controlPlanes = append(c.controlPlanes, replica) + } + + // list of nodes with worker role + if replica.IsWorker() { + // assign selected name for worker node + replica.Name = "worker" + // stores the node in derivedConfigData + c.workers = append(c.workers, replica) + } + + // node with external etcd role + if replica.IsExternalEtcd() { + if c.externalEtcd != nil { + return errors.Errorf("invalid config. there are two nodes with role %q", ExternalEtcdRole) + } + // assign selected name for etcd node + replica.Name = "etcd" + // stores the node in derivedConfigData + c.externalEtcd = replica + } + + // node with external load balancer role + if replica.IsExternalLoadBalancer() { + if c.externalLoadBalancer != nil { + return errors.Errorf("invalid config. there are two nodes with role %q", ExternalLoadBalancerRole) + } + // assign selected name for load balancer node + replica.Name = "lb" + // stores the node in derivedConfigData + c.externalLoadBalancer = replica + } + + } + + // if more than one control plane node exists, fixes names to get a progressive index + if len(c.controlPlanes) > 1 { + for i, n := range c.controlPlanes { + n.Name = fmt.Sprintf("%s%d", "control-plane", i+1) + } + } + + // if more than one worker node exists, fixes names to get a progressive index + if len(c.workers) > 1 { + for i, n := range c.workers { + n.Name = fmt.Sprintf("%s%d", "worker", i+1) + } + } + + // ensure the list of nodes is ordered + sort.Sort(c.nodes) + + return nil +} + +// Nodes returns all the nodes defined in the `kind` Config. +// Always use the Add method to add nodes. +func (c *Config) Nodes() NodeList { + return c.nodes +} + +// ControlPlanes returns all the nodes with control-plane role +func (c *Config) ControlPlanes() NodeList { + return c.controlPlanes +} + +// BootStrapControlPlane returns the first node with control-plane role +// This is the node where kubeadm init will be executed. +func (c *Config) BootStrapControlPlane() *Node { + if len(c.controlPlanes) == 0 { + return nil + } + return c.controlPlanes[0] +} + +// SecondaryControlPlanes returns all the nodes with control-plane role +// except the BootStrapControlPlane node, if any, +func (c *Config) SecondaryControlPlanes() NodeList { + if len(c.controlPlanes) <= 1 { + return nil + } + return c.controlPlanes[1:] +} + +// Workers returns all the nodes with Worker role, if any +func (c *Config) Workers() NodeList { + return c.workers +} + +// ExternalEtcd returns the node with external-etcd role, if defined +func (c *Config) ExternalEtcd() *Node { + return c.externalEtcd +} + +// ExternalLoadBalancer returns the node with external-load-balancer role, if defined +func (c *Config) ExternalLoadBalancer() *Node { + return c.externalLoadBalancer +} diff --git a/pkg/cluster/config/helpers_test.go b/pkg/cluster/config/helpers_test.go new file mode 100644 index 0000000000..651ed46cb7 --- /dev/null +++ b/pkg/cluster/config/helpers_test.go @@ -0,0 +1,189 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "testing" + + utilpointer "k8s.io/utils/pointer" +) + +func TestAdd(t *testing.T) { + cases := []struct { + TestName string + Nodes []*Node + ExpectNodes []string + ExpectControlPlanes []string + ExpectBootStrapControlPlane *string + ExpectSecondaryControlPlanes []string + ExpectWorkers []string + ExpectEtcd *string + ExpectLoadBalancer *string + ExpectError bool + }{ + { + TestName: "Defaults/Empty config should give empty Nodes", + ExpectNodes: nil, + ExpectControlPlanes: nil, + ExpectBootStrapControlPlane: nil, + ExpectSecondaryControlPlanes: nil, + ExpectWorkers: nil, + ExpectEtcd: nil, + ExpectLoadBalancer: nil, + ExpectError: false, + }, + { + TestName: "Single control plane get properly assigned to bootstrap control-plane", + Nodes: []*Node{ + {Role: ControlPlaneRole}, + }, + ExpectNodes: []string{"control-plane"}, + ExpectControlPlanes: []string{"control-plane"}, + ExpectBootStrapControlPlane: utilpointer.StringPtr("control-plane"), + ExpectSecondaryControlPlanes: nil, + ExpectError: false, + }, + { + TestName: "Control planes get properly splitted between bootstrap and secondary control-planes", + Nodes: []*Node{ + {Role: ControlPlaneRole}, + {Role: ControlPlaneRole}, + {Role: ControlPlaneRole}, + }, + ExpectNodes: []string{"control-plane1", "control-plane2", "control-plane3"}, + ExpectControlPlanes: []string{"control-plane1", "control-plane2", "control-plane3"}, + ExpectBootStrapControlPlane: utilpointer.StringPtr("control-plane1"), + ExpectSecondaryControlPlanes: []string{"control-plane2", "control-plane3"}, + ExpectError: false, + }, + { + TestName: "Single control plane get properly named if more than one node exists", + Nodes: []*Node{ + {Role: ControlPlaneRole}, + {Role: WorkerRole}, + }, + ExpectNodes: []string{"control-plane", "worker"}, + ExpectControlPlanes: []string{"control-plane"}, + ExpectBootStrapControlPlane: utilpointer.StringPtr("control-plane"), + ExpectSecondaryControlPlanes: nil, + ExpectWorkers: []string{"worker"}, + ExpectError: false, + }, + { + TestName: "Full HA cluster", // NB. This test case test that provisioning order is applied to all the node lists as well + Nodes: []*Node{ + {Role: WorkerRole}, + {Role: ControlPlaneRole}, + {Role: ExternalEtcdRole}, + {Role: ControlPlaneRole}, + {Role: WorkerRole}, + {Role: ControlPlaneRole}, + {Role: ExternalLoadBalancerRole}, + }, + ExpectNodes: []string{"etcd", "lb", "control-plane1", "control-plane2", "control-plane3", "worker1", "worker2"}, + ExpectControlPlanes: []string{"control-plane1", "control-plane2", "control-plane3"}, + ExpectBootStrapControlPlane: utilpointer.StringPtr("control-plane1"), + ExpectSecondaryControlPlanes: []string{"control-plane2", "control-plane3"}, + ExpectWorkers: []string{"worker1", "worker2"}, + ExpectEtcd: utilpointer.StringPtr("etcd"), + ExpectLoadBalancer: utilpointer.StringPtr("lb"), + ExpectError: false, + }, + { + TestName: "Fails because two etcds Nodes are added", + Nodes: []*Node{ + {Role: ExternalEtcdRole}, + {Role: ExternalEtcdRole}, + }, + ExpectError: true, + }, + { + TestName: "Fails because two load balancer Nodes are added", + Nodes: []*Node{ + {Role: ExternalLoadBalancerRole}, + {Role: ExternalLoadBalancerRole}, + }, + ExpectError: true, + }, + } + + for _, c := range cases { + t.Run(c.TestName, func(t2 *testing.T) { + // Adding Nodes to the config until first error or completing all Nodes + var cfg = Config{} + var err error + for _, n := range c.Nodes { + if e := cfg.Add(n); e != nil { + err = e + break + } + } + // the error can be: + // - nil, in which case we should expect no errors or fail + if err != nil { + if !c.ExpectError { + t2.Fatalf("unexpected error while adding Nodes: %v", err) + } + return + } + // - not nil, in which case we should expect errors or fail + if err == nil { + if c.ExpectError { + t2.Fatalf("unexpected lack or error while adding Nodes") + } + } + + // Fail if Nodes does not match + checkNodeList(t2, cfg.Nodes(), c.ExpectNodes) + + // Fail if fields derived from Nodes does not match + checkNodeList(t2, cfg.ControlPlanes(), c.ExpectControlPlanes) + checkNode(t2, cfg.BootStrapControlPlane(), c.ExpectBootStrapControlPlane) + checkNodeList(t2, cfg.SecondaryControlPlanes(), c.ExpectSecondaryControlPlanes) + checkNodeList(t2, cfg.Workers(), c.ExpectWorkers) + checkNode(t2, cfg.ExternalEtcd(), c.ExpectEtcd) + checkNode(t2, cfg.ExternalLoadBalancer(), c.ExpectLoadBalancer) + }) + } +} + +func checkNode(t *testing.T, n *Node, name *string) { + if (n == nil) != (name == nil) { + t.Errorf("expected %v node, saw %v", name, n) + } + + if n == nil { + return + } + + if n.Name != *name { + t.Errorf("expected %v node, saw %v", name, n.Name) + } +} + +func checkNodeList(t *testing.T, list NodeList, names []string) { + if len(list) != len(names) { + t.Errorf("expected %d nodes, saw %d", len(names), len(list)) + return + } + + for i, name := range names { + if list[i].Name != name { + t.Errorf("expected %q node at position %d, saw %q", name, i, list[i].Name) + } + } +} diff --git a/pkg/cluster/config/register.go b/pkg/cluster/config/register.go index bb1cb78497..2ef970fd99 100644 --- a/pkg/cluster/config/register.go +++ b/pkg/cluster/config/register.go @@ -36,16 +36,6 @@ var ( AddToScheme = localSchemeBuilder.AddToScheme ) -// Kind takes an unqualified kind and returns a Group qualified GroupKind. -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation @@ -55,7 +45,7 @@ func init() { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, - &Config{}, + &Node{}, ) return nil diff --git a/pkg/cluster/config/types.go b/pkg/cluster/config/types.go index f0a7113c53..6238ab30ac 100644 --- a/pkg/cluster/config/types.go +++ b/pkg/cluster/config/types.go @@ -21,14 +21,47 @@ import ( "sigs.k8s.io/kind/pkg/kustomize" ) -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:deepcopy-gen=false -// Config contains cluster creation config -// This is the current internal config type used by cluster -// Other API versions can be converted to this struct with Convert() +// Config groups all nodes in the `kind` Config. +// This struct is used internally by `kind` and it is NOT EXPOSED as a object of the public API. +// All the field of this type are intentionally defined a private fields, thus ensuring +// that nodes and all the derivedConfigData respect a set of assumptions that will simplify +// the rest of the code e.g. nodes are ordered by provisioning order, node names are +// unique, derivedConfigData are properly set etc. +// Config field can be modified or accessed only using provided helper func. type Config struct { + // nodes constains the list of nodes defined in the `kind` Config + // Such list is not meant to be set by hand, but the Add method + // should be used instead + nodes NodeList + + // derivedConfigData is struct populated starting from the node list + // that provides a set of convenience func for accessing nodes + // with different role in the kind cluster. + derivedConfigData +} + +// +k8s:deepcopy-gen=false + +// NodeList defines a list of Node in the `kind` Config +type NodeList []*Node + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Node contains settings for a node in the `kind` Config. +// A node in kind config represent a container that will be provisioned with all the components +// required for the assigned role in the Kubernetes cluster +type Node struct { + // TypeMeta representing the type of the object and its API schema version. metav1.TypeMeta + // Replicas is the number of desired node replicas. + // Defaults to 1 + Replicas *int32 + // Role defines the role of the nodw in the in the Kubernetes cluster managed by `kind` + // Defaults to "control-plane" + Role NodeRole // Image is the node image to use when running the cluster // TODO(bentheelder): split this into image and tag? Image string @@ -42,8 +75,34 @@ type Config struct { KubeadmConfigPatchesJSON6902 []kustomize.PatchJSON6902 // ControlPlane holds config for the control plane node ControlPlane *ControlPlane + + // The unique name assigned to the node + // This information is internal to `kind`. + // +k8s:conversion-gen=false + Name string + // ContainerHandle provides an handle to the container implementing the node + // This information is internal to `kind`. + // +k8s:conversion-gen=false + ContainerHandle } +// NodeRole defines possible role for nodes in a Kubernetes cluster managed by `kind` +type NodeRole string + +const ( + // ControlPlaneRole identifies a node that hosts a Kubernetes control-plane + ControlPlaneRole NodeRole = "control-plane" + // WorkerRole identifies a node that hosts a Kubernetes worker + WorkerRole NodeRole = "worker" + // ExternalEtcdRole identifies a node that hosts an external-etcd instance. + // Please note that `kind` nodes hosting external etcd are not kubernetes nodes + ExternalEtcdRole NodeRole = "external-etcd" + // ExternalLoadBalancerRole identifies a node that hosts an external load balancer for API server + // in HA configurations. + // Please note that `kind` nodes hosting external load balancer are not kubernetes nodes + ExternalLoadBalancerRole NodeRole = "external-load-balancer" +) + // ControlPlane holds configurations specific to the control plane nodes // (currently the only node). type ControlPlane struct { @@ -75,3 +134,32 @@ type LifecycleHook struct { // the boot process will continue MustSucceed bool } + +// +k8s:deepcopy-gen=false + +// derivedConfigData is a struct populated starting from the node list. +// This struct is used internally by `kind` and it is NOT EXPOSED as a object of the public API. +// All the field of this type are intentionally defined a private fields, thus ensuring +// that derivedConfigData respect a set of assumptions that will simplify the rest of the code. +// derivedConfigData fields can be modified or accessed only using provided helper func. +type derivedConfigData struct { + // controlPlanes contains the subset of nodes with control-plane role + controlPlanes NodeList + // workers contains the subset of nodes with worker role, if any + workers NodeList + // externalEtcd contains the node with external-etcd role, if defined + // TODO(fabriziopandini): eventually in future we would like to support + // external etcd clusters with more than one member + externalEtcd *Node + // externalLoadBalancer contains the node with external-load-balancer role, if defined + externalLoadBalancer *Node +} + +// +k8s:conversion-gen=false + +// ContainerHandle defines info used by `kind` for transforming Nodes into containers. +// This struct is used internally by `kind` and it is NOT EXPOSED as a object of the public API. +// TODO(fabriziopandini): this is a place holder for an object that will replace current container handle +// when pkg/cluster/context.go will support multi master +type ContainerHandle struct { +} diff --git a/pkg/cluster/config/v1alpha1/conversion.go b/pkg/cluster/config/v1alpha1/conversion.go new file mode 100644 index 0000000000..c287f4a07c --- /dev/null +++ b/pkg/cluster/config/v1alpha1/conversion.go @@ -0,0 +1,41 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + unsafe "unsafe" + + "sigs.k8s.io/kind/pkg/cluster/config" + kustomize "sigs.k8s.io/kind/pkg/kustomize" +) + +// Convert implement a custom conversion func (not using the api machinery) +// that transform v1alpha1 Config into a v1alpha2 Config. +// Using the api machinery for this conversion could be done, but it add several +// constraints to the internal Config object (e.g. add TypeMeta). +// Instead it was preferred to keep the desing of the internal Config clean and simple. +func (in *Config) Convert(out *config.Config) { + // Internal configuration now supports multinode, so it is necessary to transform + // v1alpha1 Config into one Node with role control plane and then add it to the list of nodes. + var node = &config.Node{} + node.Role = config.ControlPlaneRole + node.Image = in.Image + node.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) + node.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) + node.ControlPlane = (*config.ControlPlane)(unsafe.Pointer(in.ControlPlane)) + out.Add(node) +} diff --git a/pkg/cluster/config/v1alpha1/register.go b/pkg/cluster/config/v1alpha1/register.go index caabb403ab..01b71cf72f 100644 --- a/pkg/cluster/config/v1alpha1/register.go +++ b/pkg/cluster/config/v1alpha1/register.go @@ -36,16 +36,6 @@ var ( AddToScheme = localSchemeBuilder.AddToScheme ) -// Kind takes an unqualified kind and returns a Group qualified GroupKind. -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation diff --git a/pkg/cluster/config/v1alpha1/zz_generated.conversion.go b/pkg/cluster/config/v1alpha1/zz_generated.conversion.go deleted file mode 100644 index f2b35f698a..0000000000 --- a/pkg/cluster/config/v1alpha1/zz_generated.conversion.go +++ /dev/null @@ -1,176 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by conversion-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - unsafe "unsafe" - - conversion "k8s.io/apimachinery/pkg/conversion" - runtime "k8s.io/apimachinery/pkg/runtime" - config "sigs.k8s.io/kind/pkg/cluster/config" - kustomize "sigs.k8s.io/kind/pkg/kustomize" -) - -func init() { - localSchemeBuilder.Register(RegisterConversions) -} - -// RegisterConversions adds conversion functions to the given scheme. -// Public to allow building arbitrary schemes. -func RegisterConversions(s *runtime.Scheme) error { - if err := s.AddGeneratedConversionFunc((*Config)(nil), (*config.Config)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_Config_To_config_Config(a.(*Config), b.(*config.Config), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.Config)(nil), (*Config)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_Config_To_v1alpha1_Config(a.(*config.Config), b.(*Config), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ControlPlane)(nil), (*config.ControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_ControlPlane_To_config_ControlPlane(a.(*ControlPlane), b.(*config.ControlPlane), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.ControlPlane)(nil), (*ControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_ControlPlane_To_v1alpha1_ControlPlane(a.(*config.ControlPlane), b.(*ControlPlane), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*LifecycleHook)(nil), (*config.LifecycleHook)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_LifecycleHook_To_config_LifecycleHook(a.(*LifecycleHook), b.(*config.LifecycleHook), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.LifecycleHook)(nil), (*LifecycleHook)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_LifecycleHook_To_v1alpha1_LifecycleHook(a.(*config.LifecycleHook), b.(*LifecycleHook), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*NodeLifecycle)(nil), (*config.NodeLifecycle)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_NodeLifecycle_To_config_NodeLifecycle(a.(*NodeLifecycle), b.(*config.NodeLifecycle), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.NodeLifecycle)(nil), (*NodeLifecycle)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_NodeLifecycle_To_v1alpha1_NodeLifecycle(a.(*config.NodeLifecycle), b.(*NodeLifecycle), scope) - }); err != nil { - return err - } - return nil -} - -func autoConvert_v1alpha1_Config_To_config_Config(in *Config, out *config.Config, s conversion.Scope) error { - out.Image = in.Image - out.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) - out.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) - out.ControlPlane = (*config.ControlPlane)(unsafe.Pointer(in.ControlPlane)) - return nil -} - -// Convert_v1alpha1_Config_To_config_Config is an autogenerated conversion function. -func Convert_v1alpha1_Config_To_config_Config(in *Config, out *config.Config, s conversion.Scope) error { - return autoConvert_v1alpha1_Config_To_config_Config(in, out, s) -} - -func autoConvert_config_Config_To_v1alpha1_Config(in *config.Config, out *Config, s conversion.Scope) error { - out.Image = in.Image - out.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) - out.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) - out.ControlPlane = (*ControlPlane)(unsafe.Pointer(in.ControlPlane)) - return nil -} - -// Convert_config_Config_To_v1alpha1_Config is an autogenerated conversion function. -func Convert_config_Config_To_v1alpha1_Config(in *config.Config, out *Config, s conversion.Scope) error { - return autoConvert_config_Config_To_v1alpha1_Config(in, out, s) -} - -func autoConvert_v1alpha1_ControlPlane_To_config_ControlPlane(in *ControlPlane, out *config.ControlPlane, s conversion.Scope) error { - out.NodeLifecycle = (*config.NodeLifecycle)(unsafe.Pointer(in.NodeLifecycle)) - return nil -} - -// Convert_v1alpha1_ControlPlane_To_config_ControlPlane is an autogenerated conversion function. -func Convert_v1alpha1_ControlPlane_To_config_ControlPlane(in *ControlPlane, out *config.ControlPlane, s conversion.Scope) error { - return autoConvert_v1alpha1_ControlPlane_To_config_ControlPlane(in, out, s) -} - -func autoConvert_config_ControlPlane_To_v1alpha1_ControlPlane(in *config.ControlPlane, out *ControlPlane, s conversion.Scope) error { - out.NodeLifecycle = (*NodeLifecycle)(unsafe.Pointer(in.NodeLifecycle)) - return nil -} - -// Convert_config_ControlPlane_To_v1alpha1_ControlPlane is an autogenerated conversion function. -func Convert_config_ControlPlane_To_v1alpha1_ControlPlane(in *config.ControlPlane, out *ControlPlane, s conversion.Scope) error { - return autoConvert_config_ControlPlane_To_v1alpha1_ControlPlane(in, out, s) -} - -func autoConvert_v1alpha1_LifecycleHook_To_config_LifecycleHook(in *LifecycleHook, out *config.LifecycleHook, s conversion.Scope) error { - out.Name = in.Name - out.Command = *(*[]string)(unsafe.Pointer(&in.Command)) - out.MustSucceed = in.MustSucceed - return nil -} - -// Convert_v1alpha1_LifecycleHook_To_config_LifecycleHook is an autogenerated conversion function. -func Convert_v1alpha1_LifecycleHook_To_config_LifecycleHook(in *LifecycleHook, out *config.LifecycleHook, s conversion.Scope) error { - return autoConvert_v1alpha1_LifecycleHook_To_config_LifecycleHook(in, out, s) -} - -func autoConvert_config_LifecycleHook_To_v1alpha1_LifecycleHook(in *config.LifecycleHook, out *LifecycleHook, s conversion.Scope) error { - out.Name = in.Name - out.Command = *(*[]string)(unsafe.Pointer(&in.Command)) - out.MustSucceed = in.MustSucceed - return nil -} - -// Convert_config_LifecycleHook_To_v1alpha1_LifecycleHook is an autogenerated conversion function. -func Convert_config_LifecycleHook_To_v1alpha1_LifecycleHook(in *config.LifecycleHook, out *LifecycleHook, s conversion.Scope) error { - return autoConvert_config_LifecycleHook_To_v1alpha1_LifecycleHook(in, out, s) -} - -func autoConvert_v1alpha1_NodeLifecycle_To_config_NodeLifecycle(in *NodeLifecycle, out *config.NodeLifecycle, s conversion.Scope) error { - out.PreBoot = *(*[]config.LifecycleHook)(unsafe.Pointer(&in.PreBoot)) - out.PreKubeadm = *(*[]config.LifecycleHook)(unsafe.Pointer(&in.PreKubeadm)) - out.PostKubeadm = *(*[]config.LifecycleHook)(unsafe.Pointer(&in.PostKubeadm)) - out.PostSetup = *(*[]config.LifecycleHook)(unsafe.Pointer(&in.PostSetup)) - return nil -} - -// Convert_v1alpha1_NodeLifecycle_To_config_NodeLifecycle is an autogenerated conversion function. -func Convert_v1alpha1_NodeLifecycle_To_config_NodeLifecycle(in *NodeLifecycle, out *config.NodeLifecycle, s conversion.Scope) error { - return autoConvert_v1alpha1_NodeLifecycle_To_config_NodeLifecycle(in, out, s) -} - -func autoConvert_config_NodeLifecycle_To_v1alpha1_NodeLifecycle(in *config.NodeLifecycle, out *NodeLifecycle, s conversion.Scope) error { - out.PreBoot = *(*[]LifecycleHook)(unsafe.Pointer(&in.PreBoot)) - out.PreKubeadm = *(*[]LifecycleHook)(unsafe.Pointer(&in.PreKubeadm)) - out.PostKubeadm = *(*[]LifecycleHook)(unsafe.Pointer(&in.PostKubeadm)) - out.PostSetup = *(*[]LifecycleHook)(unsafe.Pointer(&in.PostSetup)) - return nil -} - -// Convert_config_NodeLifecycle_To_v1alpha1_NodeLifecycle is an autogenerated conversion function. -func Convert_config_NodeLifecycle_To_v1alpha1_NodeLifecycle(in *config.NodeLifecycle, out *NodeLifecycle, s conversion.Scope) error { - return autoConvert_config_NodeLifecycle_To_v1alpha1_NodeLifecycle(in, out, s) -} diff --git a/pkg/cluster/config/v1alpha2/default.go b/pkg/cluster/config/v1alpha2/default.go index d41d7c0a00..91a4112eef 100644 --- a/pkg/cluster/config/v1alpha2/default.go +++ b/pkg/cluster/config/v1alpha2/default.go @@ -28,9 +28,13 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error { return RegisterDefaults(scheme) } -// SetDefaults_Config sets uninitialized fields to their default value. -func SetDefaults_Config(obj *Config) { +// SetDefaults_Node sets uninitialized fields to their default value. +func SetDefaults_Node(obj *Node) { if obj.Image == "" { obj.Image = DefaultImage } + + if obj.Role == "" { + obj.Role = ControlPlaneRole + } } diff --git a/pkg/cluster/config/v1alpha2/register.go b/pkg/cluster/config/v1alpha2/register.go index b985435d5d..da7b0ace33 100644 --- a/pkg/cluster/config/v1alpha2/register.go +++ b/pkg/cluster/config/v1alpha2/register.go @@ -36,16 +36,6 @@ var ( AddToScheme = localSchemeBuilder.AddToScheme ) -// Kind takes an unqualified kind and returns a Group qualified GroupKind. -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - func init() { // We only register manually written functions here. The registration of the // generated functions takes place in the generated files. The separation @@ -55,7 +45,7 @@ func init() { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, - &Config{}, + &Node{}, ) return nil diff --git a/pkg/cluster/config/v1alpha2/types.go b/pkg/cluster/config/v1alpha2/types.go index be5937f60d..af8e56d48f 100644 --- a/pkg/cluster/config/v1alpha2/types.go +++ b/pkg/cluster/config/v1alpha2/types.go @@ -23,11 +23,19 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Config contains cluster creation config -// This is the current internal config type used by cluster -type Config struct { - metav1.TypeMeta +// Node contains settings for a node in the `kind` Config. +// A node in kind config represent a container that will be provisioned with all the components +// required for the assigned role in the Kubernetes cluster +type Node struct { + // TypeMeta representing the type of the object and its API schema version. + metav1.TypeMeta `json:",inline"` + // Replicas is the number of desired node replicas. + // Defaults to 1 + Replicas *int32 `json:"replicas,omitempty"` + // Role defines the role of the nodw in the in the Kubernetes cluster managed by `kind` + // Defaults to "control-plane" + Role NodeRole `json:"role,omitempty"` // Image is the node image to use when running the cluster // TODO(bentheelder): split this into image and tag? Image string `json:"image,omitempty"` @@ -43,6 +51,23 @@ type Config struct { ControlPlane *ControlPlane `json:"ControlPlane,omitempty"` } +// NodeRole defines possible role for nodes in a Kubernetes cluster managed by `kind` +type NodeRole string + +const ( + // ControlPlaneRole identifies a node that hosts a Kubernetes control-plane + ControlPlaneRole NodeRole = "control-plane" + // WorkerRole identifies a node that hosts a Kubernetes worker + WorkerRole NodeRole = "worker" + // ExternalEtcdRole identifies a node that hosts an external-etcd instance. + // Please note that `kind` nodes hosting external etcd are not kubernetes nodes + ExternalEtcdRole NodeRole = "external-etcd" + // ExternalLoadBalancerRole identifies a node that hosts an external load balancer for API server + // in HA configurations. + // Please note that `kind` nodes hosting external load balancer are not kubernetes nodes + ExternalLoadBalancerRole NodeRole = "external-load-balancer" +) + // ControlPlane holds configurations specific to the control plane nodes // (currently the only node). type ControlPlane struct { diff --git a/pkg/cluster/config/validate.go b/pkg/cluster/config/validate.go index 87773a5e5d..bf81b16040 100644 --- a/pkg/cluster/config/validate.go +++ b/pkg/cluster/config/validate.go @@ -26,12 +26,53 @@ import ( // with the config, or nil if there are none func (c *Config) Validate() error { errs := []error{} - if c.Image == "" { + + // All nodes in the config should be valid + for _, n := range c.nodes { + if err := n.Validate(); err != nil { + errs = append(errs, fmt.Errorf("please fix invalid configuration for node %q: \n%v", n.Name, err)) + } + } + + // There should be at least one control plane + if c.BootStrapControlPlane() == nil { + errs = append(errs, fmt.Errorf("please add at least one node with role %q", ControlPlaneRole)) + } + // There should be one load balancer if more than one control plane exists in the cluster + if c.ControlPlanes() != nil && len(c.ControlPlanes()) > 1 && c.ExternalLoadBalancer() == nil { + errs = append(errs, fmt.Errorf("please add a node with role %s because in the cluster there are more than one node with role %s", ExternalLoadBalancerRole, ControlPlaneRole)) + } + + if len(errs) > 0 { + return util.NewErrors(errs) + } + return nil +} + +// Validate returns a ConfigErrors with an entry for each problem +// with the Node, or nil if there are none +func (n *Node) Validate() error { + errs := []error{} + + // validate node role should be one of the expected values + switch n.Role { + case ControlPlaneRole, + WorkerRole, + ExternalEtcdRole, + ExternalLoadBalancerRole: + default: + errs = append(errs, fmt.Errorf("role is a required field")) + } + + // image should be defined + if n.Image == "" { errs = append(errs, fmt.Errorf("image is a required field")) } - if c.ControlPlane != nil { - if c.ControlPlane.NodeLifecycle != nil { - for _, hook := range c.ControlPlane.NodeLifecycle.PreBoot { + + // validate NodeLifecycle + if n.ControlPlane != nil { + if n.ControlPlane.NodeLifecycle != nil { + for _, hook := range n.ControlPlane.NodeLifecycle.PreBoot { if len(hook.Command) == 0 { errs = append(errs, fmt.Errorf( "preBoot hooks must set command to a non-empty value", @@ -41,7 +82,7 @@ func (c *Config) Validate() error { break } } - for _, hook := range c.ControlPlane.NodeLifecycle.PreKubeadm { + for _, hook := range n.ControlPlane.NodeLifecycle.PreKubeadm { if len(hook.Command) == 0 { errs = append(errs, fmt.Errorf( "preKubeadm hooks must set command to a non-empty value", @@ -51,7 +92,7 @@ func (c *Config) Validate() error { break } } - for _, hook := range c.ControlPlane.NodeLifecycle.PostKubeadm { + for _, hook := range n.ControlPlane.NodeLifecycle.PostKubeadm { if len(hook.Command) == 0 { errs = append(errs, fmt.Errorf( "postKubeadm hooks must set command to a non-empty value", @@ -61,7 +102,7 @@ func (c *Config) Validate() error { break } } - for _, hook := range c.ControlPlane.NodeLifecycle.PostSetup { + for _, hook := range n.ControlPlane.NodeLifecycle.PostSetup { if len(hook.Command) == 0 { errs = append(errs, fmt.Errorf( "postKubeadm hooks must set command to a non-empty value", @@ -76,5 +117,6 @@ func (c *Config) Validate() error { if len(errs) > 0 { return util.NewErrors(errs) } + return nil } diff --git a/pkg/cluster/config/validate_test.go b/pkg/cluster/config/validate_test.go index 5d359fd7d0..4efc947188 100644 --- a/pkg/cluster/config/validate_test.go +++ b/pkg/cluster/config/validate_test.go @@ -23,29 +23,30 @@ import ( ) // TODO(fabriziopandini): ideally this should use scheme.Default, but this creates a circular dependency -// So the current solution is to mimic defaulting for the validation test, but probably there should be a better solution here -func newDefaultedConfig() *Config { - cfg := &Config{ +// So the current solution is to mimic defaulting for the validation test +func newDefaultedNode(role NodeRole) *Node { + cfg := &Node{ + Role: role, Image: "myImage:latest", } return cfg } -func TestConfigValidate(t *testing.T) { +func TestNodeValidate(t *testing.T) { cases := []struct { - TestName string - Config *Config - ExpectedErrors int + TestName string + Node *Node + ExpectErrors int }{ { - TestName: "Canonical config", - Config: newDefaultedConfig(), - ExpectedErrors: 0, + TestName: "Canonical node", + Node: newDefaultedNode(ControlPlaneRole), + ExpectErrors: 0, }, { TestName: "Invalid PreBoot hook", - Config: func() *Config { - cfg := newDefaultedConfig() + Node: func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) cfg.ControlPlane = &ControlPlane{ NodeLifecycle: &NodeLifecycle{ PreBoot: []LifecycleHook{ @@ -57,12 +58,12 @@ func TestConfigValidate(t *testing.T) { } return cfg }(), - ExpectedErrors: 1, + ExpectErrors: 1, }, { TestName: "Invalid PreKubeadm hook", - Config: func() *Config { - cfg := newDefaultedConfig() + Node: func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) cfg.ControlPlane = &ControlPlane{ NodeLifecycle: &NodeLifecycle{ PreKubeadm: []LifecycleHook{ @@ -75,12 +76,12 @@ func TestConfigValidate(t *testing.T) { } return cfg }(), - ExpectedErrors: 1, + ExpectErrors: 1, }, { TestName: "Invalid PostKubeadm hook", - Config: func() *Config { - cfg := newDefaultedConfig() + Node: func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) cfg.ControlPlane = &ControlPlane{ NodeLifecycle: &NodeLifecycle{ PostKubeadm: []LifecycleHook{ @@ -93,39 +94,137 @@ func TestConfigValidate(t *testing.T) { } return cfg }(), - ExpectedErrors: 1, + ExpectErrors: 1, }, { TestName: "Empty image field", - Config: func() *Config { - cfg := newDefaultedConfig() + Node: func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) cfg.Image = "" return cfg }(), - ExpectedErrors: 1, + ExpectErrors: 1, + }, + { + TestName: "Empty role field", + Node: func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) + cfg.Role = "" + return cfg + }(), + ExpectErrors: 1, + }, + { + TestName: "Unknows role field", + Node: func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) + cfg.Role = "ssss" + return cfg + }(), + ExpectErrors: 1, + }, + } + + for _, tc := range cases { + t.Run(tc.TestName, func(t2 *testing.T) { + err := tc.Node.Validate() + // the error can be: + // - nil, in which case we should expect no errors or fail + if err == nil { + if tc.ExpectErrors != 0 { + t2.Error("received no errors but expected errors for case") + } + return + } + // - not castable to *Errors, in which case we have the wrong error type ... + configErrors, ok := err.(util.Errors) + if !ok { + t2.Errorf("config.Validate should only return nil or ConfigErrors{...}, got: %v", err) + return + } + // - ConfigErrors, in which case expect a certain number of errors + errors := configErrors.Errors() + if len(errors) != tc.ExpectErrors { + t2.Errorf("expected %d errors but got len(%v) = %d", tc.ExpectErrors, errors, len(errors)) + } + }) + } +} + +func TestConfigValidate(t *testing.T) { + cases := []struct { + TestName string + Nodes []*Node + ExpectErrors int + }{ + { + TestName: "Canonical config", + Nodes: []*Node{ + newDefaultedNode(ControlPlaneRole), + }, + }, + { + TestName: "Fail without at least one control plane", + ExpectErrors: 1, + }, + { + TestName: "Fail without at load balancer and more than one control plane", + Nodes: []*Node{ + newDefaultedNode(ControlPlaneRole), + newDefaultedNode(ControlPlaneRole), + }, + ExpectErrors: 1, + }, + { + TestName: "Fail with not valid nodes", + Nodes: []*Node{ + func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) + cfg.Image = "" + return cfg + }(), + func() *Node { + cfg := newDefaultedNode(ControlPlaneRole) + cfg.Role = "" + return cfg + }(), + }, + ExpectErrors: 2, }, } for _, tc := range cases { - err := tc.Config.Validate() - // the error can be: - // - nil, in which case we should expect no errors or fail - if err == nil { - if tc.ExpectedErrors != 0 { - t.Errorf("received no errors but expected errors for case %s", tc.TestName) + t.Run(tc.TestName, func(t2 *testing.T) { + var c = Config{} + // Adding nodes to the config + for _, n := range tc.Nodes { + if err := c.Add(n); err != nil { + t.Fatalf("unexpected error while adding nodes: %v", err) + break + } + } + // validating config + err := c.Validate() + + // the error can be: + // - nil, in which case we should expect no errors or fail + if err == nil { + if tc.ExpectErrors != 0 { + t2.Error("received no errors but expected errors") + } + return + } + // - not castable to *Errors, in which case we have the wrong error type ... + configErrors, ok := err.(util.Errors) + if !ok { + t2.Errorf("config.Validate should only return nil or ConfigErrors{...}, got: %v", err) + return + } + // - ConfigErrors, in which case expect a certain number of errors + errors := configErrors.Errors() + if len(errors) != tc.ExpectErrors { + t2.Errorf("expected %d errors but got len(%v) = %d", tc.ExpectErrors, errors, len(errors)) } - continue - } - // - not castable to *Errors, in which case we have the wrong error type ... - configErrors, ok := err.(*util.Errors) - if !ok { - t.Errorf("config.Validate should only return nil or ConfigErrors{...}, got: %v for case: %s", err, tc.TestName) - continue - } - // - ConfigErrors, in which case expect a certain number of errors - errors := configErrors.Errors() - if len(errors) != tc.ExpectedErrors { - t.Errorf("expected %d errors but got len(%v) = %d for case: %s", tc.ExpectedErrors, errors, len(errors), tc.TestName) - } + }) } } diff --git a/pkg/cluster/config/zz_generated.deepcopy.go b/pkg/cluster/config/zz_generated.deepcopy.go index 17bec49089..b103d660c3 100644 --- a/pkg/cluster/config/zz_generated.deepcopy.go +++ b/pkg/cluster/config/zz_generated.deepcopy.go @@ -26,45 +26,21 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Config) DeepCopyInto(out *Config) { +func (in *ContainerHandle) DeepCopyInto(out *ContainerHandle) { *out = *in - out.TypeMeta = in.TypeMeta - if in.KubeadmConfigPatches != nil { - in, out := &in.KubeadmConfigPatches, &out.KubeadmConfigPatches - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeadmConfigPatchesJSON6902 != nil { - in, out := &in.KubeadmConfigPatchesJSON6902, &out.KubeadmConfigPatchesJSON6902 - *out = make([]kustomize.PatchJSON6902, len(*in)) - copy(*out, *in) - } - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(ControlPlane) - (*in).DeepCopyInto(*out) - } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. -func (in *Config) DeepCopy() *Config { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerHandle. +func (in *ContainerHandle) DeepCopy() *ContainerHandle { if in == nil { return nil } - out := new(Config) + out := new(ContainerHandle) in.DeepCopyInto(out) return out } -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Config) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlPlane) DeepCopyInto(out *ControlPlane) { *out = *in @@ -107,6 +83,52 @@ func (in *LifecycleHook) DeepCopy() *LifecycleHook { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Node) DeepCopyInto(out *Node) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.KubeadmConfigPatches != nil { + in, out := &in.KubeadmConfigPatches, &out.KubeadmConfigPatches + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.KubeadmConfigPatchesJSON6902 != nil { + in, out := &in.KubeadmConfigPatchesJSON6902, &out.KubeadmConfigPatchesJSON6902 + *out = make([]kustomize.PatchJSON6902, len(*in)) + copy(*out, *in) + } + if in.ControlPlane != nil { + in, out := &in.ControlPlane, &out.ControlPlane + *out = new(ControlPlane) + (*in).DeepCopyInto(*out) + } + out.ContainerHandle = in.ContainerHandle + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Node. +func (in *Node) DeepCopy() *Node { + if in == nil { + return nil + } + out := new(Node) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Node) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeLifecycle) DeepCopyInto(out *NodeLifecycle) { *out = *in diff --git a/pkg/cluster/context.go b/pkg/cluster/context.go index e92f53a3ee..de107d5c44 100644 --- a/pkg/cluster/context.go +++ b/pkg/cluster/context.go @@ -142,7 +142,11 @@ func (c *Context) Create(cfg *config.Config, retain bool, wait time.Duration) er cc.status.MaybeWrapLogrus(log.StandardLogger()) defer cc.status.End(false) - image := cfg.Image + + // TODO(fabrizio pandini): usage of BootStrapControlPlane() is temporary / WIP + // kind v1alpha2 config fully supports multi nodes, but the cluster creation logic implemented in + // in this file does not (yet). + image := cfg.BootStrapControlPlane().Image if strings.Contains(image, "@sha256:") { image = strings.Split(image, "@sha256:")[0] } @@ -150,7 +154,7 @@ func (c *Context) Create(cfg *config.Config, retain bool, wait time.Duration) er // attempt to explicitly pull the image if it doesn't exist locally // we don't care if this errors, we'll still try to run which also pulls - _, _ = docker.PullIfNotPresent(cfg.Image, 4) + _, _ = docker.PullIfNotPresent(cfg.BootStrapControlPlane().Image, 4) // TODO(bentheelder): multiple nodes ... kubeadmConfig, err := cc.provisionControlPlane( @@ -211,7 +215,7 @@ func (cc *createContext) provisionControlPlane( ) (kubeadmConfigPath string, err error) { cc.status.Start(fmt.Sprintf("[%s] Creating node container πŸ“¦", nodeName)) // create the "node" container (docker run, but it is paused, see createNode) - node, port, err := nodes.CreateControlPlaneNode(nodeName, cc.config.Image, cc.ClusterLabel()) + node, port, err := nodes.CreateControlPlaneNode(nodeName, cc.config.BootStrapControlPlane().Image, cc.ClusterLabel()) if err != nil { return "", err } @@ -234,8 +238,8 @@ func (cc *createContext) provisionControlPlane( } // run any pre-boot hooks - if cc.config.ControlPlane != nil && cc.config.ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.ControlPlane.NodeLifecycle.PreBoot { + if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { + for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PreBoot { if err := runHook(node, &hook, "preBoot"); err != nil { return "", err } @@ -301,8 +305,8 @@ func (cc *createContext) provisionControlPlane( } // run any pre-kubeadm hooks - if cc.config.ControlPlane != nil && cc.config.ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.ControlPlane.NodeLifecycle.PreKubeadm { + if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { + for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PreKubeadm { if err := runHook(node, &hook, "preKubeadm"); err != nil { return kubeadmConfig, err } @@ -330,8 +334,8 @@ func (cc *createContext) provisionControlPlane( } // run any post-kubeadm hooks - if cc.config.ControlPlane != nil && cc.config.ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.ControlPlane.NodeLifecycle.PostKubeadm { + if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { + for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PostKubeadm { if err := runHook(node, &hook, "postKubeadm"); err != nil { return kubeadmConfig, err } @@ -372,8 +376,8 @@ func (cc *createContext) provisionControlPlane( } // run any post-overlay hooks - if cc.config.ControlPlane != nil && cc.config.ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.ControlPlane.NodeLifecycle.PostSetup { + if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { + for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PostSetup { if err := runHook(node, &hook, "postSetup"); err != nil { return kubeadmConfig, err } @@ -442,8 +446,8 @@ func (c *Context) createKubeadmConfig(cfg *config.Config, data kubeadm.ConfigDat // apply patches patchedConfig, err := kustomize.Build( []string{config}, - cfg.KubeadmConfigPatches, - cfg.KubeadmConfigPatchesJSON6902, + cfg.BootStrapControlPlane().KubeadmConfigPatches, + cfg.BootStrapControlPlane().KubeadmConfigPatchesJSON6902, ) if err != nil { os.Remove(path) From 4d6abfe9ad708dcc31e60480fbc28bed1e4993bf Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 6 Dec 2018 14:08:24 +0100 Subject: [PATCH 2/7] test cases --- .../encoding/testdata/invalid-apiversion.yaml | 6 +++--- .../config/encoding/testdata/invalid-kind.yaml | 3 +++ .../testdata/invalid-no-apiversion.yaml | 2 ++ .../encoding/testdata/invalid-no-kind.yaml | 2 ++ .../v1alpha1/invalid-minimal-two-nodes.yaml | 6 ++++++ .../invalid-minimal-duplicated-names.yaml | 6 ++++++ .../testdata/v1alpha2/valid-full-ha.yaml | 18 ++++++++++++++++++ .../v1alpha2/valid-minimal-two-nodes.yaml | 7 +++++++ .../testdata/v1alpha2/valid-minimal.yaml | 2 +- .../v1alpha2/valid-with-lifecyclehooks.yaml | 2 +- 10 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 pkg/cluster/config/encoding/testdata/invalid-kind.yaml create mode 100644 pkg/cluster/config/encoding/testdata/invalid-no-apiversion.yaml create mode 100644 pkg/cluster/config/encoding/testdata/invalid-no-kind.yaml create mode 100644 pkg/cluster/config/encoding/testdata/v1alpha1/invalid-minimal-two-nodes.yaml create mode 100644 pkg/cluster/config/encoding/testdata/v1alpha2/invalid-minimal-duplicated-names.yaml create mode 100644 pkg/cluster/config/encoding/testdata/v1alpha2/valid-full-ha.yaml create mode 100644 pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal-two-nodes.yaml diff --git a/pkg/cluster/config/encoding/testdata/invalid-apiversion.yaml b/pkg/cluster/config/encoding/testdata/invalid-apiversion.yaml index 9f08271ed8..22771e0ae4 100644 --- a/pkg/cluster/config/encoding/testdata/invalid-apiversion.yaml +++ b/pkg/cluster/config/encoding/testdata/invalid-apiversion.yaml @@ -1,3 +1,3 @@ -# this file contains an invalid config api version for testing -kind: Config -apiVersion: not-valid +# this file contains an invalid config kind for testing +kind: not-valid +apiVersion: kind.sigs.k8s.io/v1alpha2 diff --git a/pkg/cluster/config/encoding/testdata/invalid-kind.yaml b/pkg/cluster/config/encoding/testdata/invalid-kind.yaml new file mode 100644 index 0000000000..b5f797a633 --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/invalid-kind.yaml @@ -0,0 +1,3 @@ +# this file contains an invalid config api version for testing +kind: Node +apiVersion: not-valid diff --git a/pkg/cluster/config/encoding/testdata/invalid-no-apiversion.yaml b/pkg/cluster/config/encoding/testdata/invalid-no-apiversion.yaml new file mode 100644 index 0000000000..a5ffac0cc8 --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/invalid-no-apiversion.yaml @@ -0,0 +1,2 @@ +# this file contains an invalid config without apiVersion for testing +kind: Node diff --git a/pkg/cluster/config/encoding/testdata/invalid-no-kind.yaml b/pkg/cluster/config/encoding/testdata/invalid-no-kind.yaml new file mode 100644 index 0000000000..eafbaab066 --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/invalid-no-kind.yaml @@ -0,0 +1,2 @@ +# this file contains an invalid config without kind for testing +apiVersion: kind.sigs.k8s.io/v1alpha2 diff --git a/pkg/cluster/config/encoding/testdata/v1alpha1/invalid-minimal-two-nodes.yaml b/pkg/cluster/config/encoding/testdata/v1alpha1/invalid-minimal-two-nodes.yaml new file mode 100644 index 0000000000..21ec42116b --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/v1alpha1/invalid-minimal-two-nodes.yaml @@ -0,0 +1,6 @@ +# invalid v1alpha1 config file with two documents +kind: Config +apiVersion: kind.sigs.k8s.io/v1alpha1 +--- +kind: Config +apiVersion: kind.sigs.k8s.io/v1alpha1 \ No newline at end of file diff --git a/pkg/cluster/config/encoding/testdata/v1alpha2/invalid-minimal-duplicated-names.yaml b/pkg/cluster/config/encoding/testdata/v1alpha2/invalid-minimal-duplicated-names.yaml new file mode 100644 index 0000000000..ee22fff54e --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/v1alpha2/invalid-minimal-duplicated-names.yaml @@ -0,0 +1,6 @@ +# invalid config file with nodes with the same name +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +--- +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 \ No newline at end of file diff --git a/pkg/cluster/config/encoding/testdata/v1alpha2/valid-full-ha.yaml b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-full-ha.yaml new file mode 100644 index 0000000000..7b04c7cad9 --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-full-ha.yaml @@ -0,0 +1,18 @@ +# technically valid, config file with a full ha cluster +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +role: control-plane +replicas: 3 +--- +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +role: worker +replicas: 2 +--- +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +role: external-etcd +--- +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +role: external-load-balancer diff --git a/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal-two-nodes.yaml b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal-two-nodes.yaml new file mode 100644 index 0000000000..1404d6d8b6 --- /dev/null +++ b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal-two-nodes.yaml @@ -0,0 +1,7 @@ +# technically valid, minimal config file with two nodes +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +--- +kind: Node +apiVersion: kind.sigs.k8s.io/v1alpha2 +role: worker \ No newline at end of file diff --git a/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal.yaml b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal.yaml index bffca0fd7d..f960aa0653 100644 --- a/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal.yaml +++ b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-minimal.yaml @@ -1,3 +1,3 @@ # technically valid, minimal config file -kind: Config +kind: Node apiVersion: kind.sigs.k8s.io/v1alpha2 diff --git a/pkg/cluster/config/encoding/testdata/v1alpha2/valid-with-lifecyclehooks.yaml b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-with-lifecyclehooks.yaml index 5b1521ddbf..29ad5ae50d 100644 --- a/pkg/cluster/config/encoding/testdata/v1alpha2/valid-with-lifecyclehooks.yaml +++ b/pkg/cluster/config/encoding/testdata/v1alpha2/valid-with-lifecyclehooks.yaml @@ -1,4 +1,4 @@ -kind: Config +kind: Node apiVersion: kind.sigs.k8s.io/v1alpha2 nodeLifecycle: preKubeadm: From f467af09d43d1bf6e4d7018a0d3cd9a489622c47 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 6 Dec 2018 14:08:42 +0100 Subject: [PATCH 3/7] update deps --- Gopkg.lock | 11 + Gopkg.toml | 4 + .../v1alpha2/zz_generated.conversion.go | 78 +++---- .../config/v1alpha2/zz_generated.deepcopy.go | 85 ++++---- .../config/v1alpha2/zz_generated.default.go | 6 +- vendor/k8s.io/utils/LICENSE | 202 ++++++++++++++++++ vendor/k8s.io/utils/pointer/pointer.go | 86 ++++++++ 7 files changed, 393 insertions(+), 79 deletions(-) create mode 100644 vendor/k8s.io/utils/LICENSE create mode 100644 vendor/k8s.io/utils/pointer/pointer.go diff --git a/Gopkg.lock b/Gopkg.lock index 57e0fe4ddb..1034374d57 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -697,6 +697,14 @@ pruneopts = "NUT" revision = "3a9b63ab1e397dc12a9764df998f99bc59dfd9ae" +[[projects]] + branch = "master" + digest = "1:ff54706d46de40c865b5fcfc4bde1087c02510cd12e0150de8e405ab427d9907" + name = "k8s.io/utils" + packages = ["pointer"] + pruneopts = "NUT" + revision = "0d26856f57b32ec3398579285e5c8a2bfe8c5243" + [[projects]] branch = "master" digest = "1:9dbda414956b2af5c2e24cbc54074ed7158902912bde8c0a0ec7f7344a309720" @@ -762,12 +770,15 @@ "k8s.io/apimachinery/pkg/util/runtime", "k8s.io/apimachinery/pkg/util/sets", "k8s.io/apimachinery/pkg/util/version", + "k8s.io/apimachinery/pkg/util/yaml", "k8s.io/code-generator/cmd/conversion-gen", "k8s.io/code-generator/cmd/deepcopy-gen", "k8s.io/code-generator/cmd/defaulter-gen", + "k8s.io/utils/pointer", "sigs.k8s.io/kustomize/k8sdeps", "sigs.k8s.io/kustomize/pkg/commands/build", "sigs.k8s.io/kustomize/pkg/fs", + "sigs.k8s.io/yaml", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index c1d3fb5374..319d882178 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -52,3 +52,7 @@ required = [ name = "k8s.io/code-generator" branch = "master" + +[[constraint]] + branch = "master" + name = "k8s.io/utils" diff --git a/pkg/cluster/config/v1alpha2/zz_generated.conversion.go b/pkg/cluster/config/v1alpha2/zz_generated.conversion.go index ab004cf311..803d54d01e 100644 --- a/pkg/cluster/config/v1alpha2/zz_generated.conversion.go +++ b/pkg/cluster/config/v1alpha2/zz_generated.conversion.go @@ -36,16 +36,6 @@ func init() { // RegisterConversions adds conversion functions to the given scheme. // Public to allow building arbitrary schemes. func RegisterConversions(s *runtime.Scheme) error { - if err := s.AddGeneratedConversionFunc((*Config)(nil), (*config.Config)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_Config_To_config_Config(a.(*Config), b.(*config.Config), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*config.Config)(nil), (*Config)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_config_Config_To_v1alpha2_Config(a.(*config.Config), b.(*Config), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*ControlPlane)(nil), (*config.ControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_ControlPlane_To_config_ControlPlane(a.(*ControlPlane), b.(*config.ControlPlane), scope) }); err != nil { @@ -66,6 +56,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*Node)(nil), (*config.Node)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_Node_To_config_Node(a.(*Node), b.(*config.Node), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*config.Node)(nil), (*Node)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_config_Node_To_v1alpha2_Node(a.(*config.Node), b.(*Node), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*NodeLifecycle)(nil), (*config.NodeLifecycle)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_NodeLifecycle_To_config_NodeLifecycle(a.(*NodeLifecycle), b.(*config.NodeLifecycle), scope) }); err != nil { @@ -79,32 +79,6 @@ func RegisterConversions(s *runtime.Scheme) error { return nil } -func autoConvert_v1alpha2_Config_To_config_Config(in *Config, out *config.Config, s conversion.Scope) error { - out.Image = in.Image - out.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) - out.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) - out.ControlPlane = (*config.ControlPlane)(unsafe.Pointer(in.ControlPlane)) - return nil -} - -// Convert_v1alpha2_Config_To_config_Config is an autogenerated conversion function. -func Convert_v1alpha2_Config_To_config_Config(in *Config, out *config.Config, s conversion.Scope) error { - return autoConvert_v1alpha2_Config_To_config_Config(in, out, s) -} - -func autoConvert_config_Config_To_v1alpha2_Config(in *config.Config, out *Config, s conversion.Scope) error { - out.Image = in.Image - out.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) - out.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) - out.ControlPlane = (*ControlPlane)(unsafe.Pointer(in.ControlPlane)) - return nil -} - -// Convert_config_Config_To_v1alpha2_Config is an autogenerated conversion function. -func Convert_config_Config_To_v1alpha2_Config(in *config.Config, out *Config, s conversion.Scope) error { - return autoConvert_config_Config_To_v1alpha2_Config(in, out, s) -} - func autoConvert_v1alpha2_ControlPlane_To_config_ControlPlane(in *ControlPlane, out *config.ControlPlane, s conversion.Scope) error { out.NodeLifecycle = (*config.NodeLifecycle)(unsafe.Pointer(in.NodeLifecycle)) return nil @@ -149,6 +123,38 @@ func Convert_config_LifecycleHook_To_v1alpha2_LifecycleHook(in *config.Lifecycle return autoConvert_config_LifecycleHook_To_v1alpha2_LifecycleHook(in, out, s) } +func autoConvert_v1alpha2_Node_To_config_Node(in *Node, out *config.Node, s conversion.Scope) error { + out.Replicas = (*int32)(unsafe.Pointer(in.Replicas)) + out.Role = config.NodeRole(in.Role) + out.Image = in.Image + out.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) + out.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) + out.ControlPlane = (*config.ControlPlane)(unsafe.Pointer(in.ControlPlane)) + return nil +} + +// Convert_v1alpha2_Node_To_config_Node is an autogenerated conversion function. +func Convert_v1alpha2_Node_To_config_Node(in *Node, out *config.Node, s conversion.Scope) error { + return autoConvert_v1alpha2_Node_To_config_Node(in, out, s) +} + +func autoConvert_config_Node_To_v1alpha2_Node(in *config.Node, out *Node, s conversion.Scope) error { + out.Replicas = (*int32)(unsafe.Pointer(in.Replicas)) + out.Role = NodeRole(in.Role) + out.Image = in.Image + out.KubeadmConfigPatches = *(*[]string)(unsafe.Pointer(&in.KubeadmConfigPatches)) + out.KubeadmConfigPatchesJSON6902 = *(*[]kustomize.PatchJSON6902)(unsafe.Pointer(&in.KubeadmConfigPatchesJSON6902)) + out.ControlPlane = (*ControlPlane)(unsafe.Pointer(in.ControlPlane)) + // INFO: in.Name opted out of conversion generation + // INFO: in.ContainerHandle opted out of conversion generation + return nil +} + +// Convert_config_Node_To_v1alpha2_Node is an autogenerated conversion function. +func Convert_config_Node_To_v1alpha2_Node(in *config.Node, out *Node, s conversion.Scope) error { + return autoConvert_config_Node_To_v1alpha2_Node(in, out, s) +} + func autoConvert_v1alpha2_NodeLifecycle_To_config_NodeLifecycle(in *NodeLifecycle, out *config.NodeLifecycle, s conversion.Scope) error { out.PreBoot = *(*[]config.LifecycleHook)(unsafe.Pointer(&in.PreBoot)) out.PreKubeadm = *(*[]config.LifecycleHook)(unsafe.Pointer(&in.PreKubeadm)) diff --git a/pkg/cluster/config/v1alpha2/zz_generated.deepcopy.go b/pkg/cluster/config/v1alpha2/zz_generated.deepcopy.go index 08cc10d471..3961c0abc7 100644 --- a/pkg/cluster/config/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/cluster/config/v1alpha2/zz_generated.deepcopy.go @@ -25,46 +25,6 @@ import ( kustomize "sigs.k8s.io/kind/pkg/kustomize" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Config) DeepCopyInto(out *Config) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.KubeadmConfigPatches != nil { - in, out := &in.KubeadmConfigPatches, &out.KubeadmConfigPatches - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.KubeadmConfigPatchesJSON6902 != nil { - in, out := &in.KubeadmConfigPatchesJSON6902, &out.KubeadmConfigPatchesJSON6902 - *out = make([]kustomize.PatchJSON6902, len(*in)) - copy(*out, *in) - } - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(ControlPlane) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. -func (in *Config) DeepCopy() *Config { - if in == nil { - return nil - } - out := new(Config) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Config) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlPlane) DeepCopyInto(out *ControlPlane) { *out = *in @@ -107,6 +67,51 @@ func (in *LifecycleHook) DeepCopy() *LifecycleHook { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Node) DeepCopyInto(out *Node) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.KubeadmConfigPatches != nil { + in, out := &in.KubeadmConfigPatches, &out.KubeadmConfigPatches + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.KubeadmConfigPatchesJSON6902 != nil { + in, out := &in.KubeadmConfigPatchesJSON6902, &out.KubeadmConfigPatchesJSON6902 + *out = make([]kustomize.PatchJSON6902, len(*in)) + copy(*out, *in) + } + if in.ControlPlane != nil { + in, out := &in.ControlPlane, &out.ControlPlane + *out = new(ControlPlane) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Node. +func (in *Node) DeepCopy() *Node { + if in == nil { + return nil + } + out := new(Node) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Node) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeLifecycle) DeepCopyInto(out *NodeLifecycle) { *out = *in diff --git a/pkg/cluster/config/v1alpha2/zz_generated.default.go b/pkg/cluster/config/v1alpha2/zz_generated.default.go index 54567365ae..5a6370a9d9 100644 --- a/pkg/cluster/config/v1alpha2/zz_generated.default.go +++ b/pkg/cluster/config/v1alpha2/zz_generated.default.go @@ -28,10 +28,10 @@ import ( // Public to allow building arbitrary schemes. // All generated defaulters are covering - they call all nested defaulters. func RegisterDefaults(scheme *runtime.Scheme) error { - scheme.AddTypeDefaultingFunc(&Config{}, func(obj interface{}) { SetObjectDefaults_Config(obj.(*Config)) }) + scheme.AddTypeDefaultingFunc(&Node{}, func(obj interface{}) { SetObjectDefaults_Node(obj.(*Node)) }) return nil } -func SetObjectDefaults_Config(in *Config) { - SetDefaults_Config(in) +func SetObjectDefaults_Node(in *Node) { + SetDefaults_Node(in) } diff --git a/vendor/k8s.io/utils/LICENSE b/vendor/k8s.io/utils/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/k8s.io/utils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/utils/pointer/pointer.go b/vendor/k8s.io/utils/pointer/pointer.go new file mode 100644 index 0000000000..a11a540f46 --- /dev/null +++ b/vendor/k8s.io/utils/pointer/pointer.go @@ -0,0 +1,86 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pointer + +import ( + "fmt" + "reflect" +) + +// AllPtrFieldsNil tests whether all pointer fields in a struct are nil. This is useful when, +// for example, an API struct is handled by plugins which need to distinguish +// "no plugin accepted this spec" from "this spec is empty". +// +// This function is only valid for structs and pointers to structs. Any other +// type will cause a panic. Passing a typed nil pointer will return true. +func AllPtrFieldsNil(obj interface{}) bool { + v := reflect.ValueOf(obj) + if !v.IsValid() { + panic(fmt.Sprintf("reflect.ValueOf() produced a non-valid Value for %#v", obj)) + } + if v.Kind() == reflect.Ptr { + if v.IsNil() { + return true + } + v = v.Elem() + } + for i := 0; i < v.NumField(); i++ { + if v.Field(i).Kind() == reflect.Ptr && !v.Field(i).IsNil() { + return false + } + } + return true +} + +// Int32Ptr returns a pointer to an int32 +func Int32Ptr(i int32) *int32 { + return &i +} + +// Int64Ptr returns a pointer to an int64 +func Int64Ptr(i int64) *int64 { + return &i +} + +// Int32PtrDerefOr dereference the int32 ptr and returns it i not nil, +// else returns def. +func Int32PtrDerefOr(ptr *int32, def int32) int32 { + if ptr != nil { + return *ptr + } + return def +} + +// BoolPtr returns a pointer to a bool +func BoolPtr(b bool) *bool { + return &b +} + +// StringPtr returns a pointer to the passed string. +func StringPtr(s string) *string { + return &s +} + +// Float32Ptr returns a pointer to the passed float32. +func Float32Ptr(i float32) *float32 { + return &i +} + +// Float64Ptr returns a pointer to the passed float64. +func Float64Ptr(i float64) *float64 { + return &i +} From 95327743bea76c09268de0eb5713806c7c976d5a Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Mon, 10 Dec 2018 16:13:56 +0100 Subject: [PATCH 4/7] introduce actions --- cmd/kind/create/cluster/createcluster.go | 25 +- pkg/cluster/actions.go | 231 +++++++++++++ pkg/cluster/actions_test.go | 252 ++++++++++++++ pkg/cluster/consts/consts.go | 3 + pkg/cluster/context.go | 412 +++++++++-------------- pkg/cluster/kubeadm-config.go | 136 ++++++++ pkg/cluster/kubeadm-init.go | 171 ++++++++++ pkg/cluster/kubeadm-join.go | 102 ++++++ pkg/cluster/kubeadm/config.go | 18 +- pkg/cluster/kubeadm/const.go | 3 + pkg/cluster/nodes/create.go | 65 +++- pkg/cluster/nodes/node.go | 59 +++- pkg/docker/inspect.go | 34 ++ 13 files changed, 1234 insertions(+), 277 deletions(-) create mode 100644 pkg/cluster/actions.go create mode 100644 pkg/cluster/actions_test.go create mode 100644 pkg/cluster/kubeadm-config.go create mode 100644 pkg/cluster/kubeadm-init.go create mode 100644 pkg/cluster/kubeadm-join.go create mode 100644 pkg/docker/inspect.go diff --git a/cmd/kind/create/cluster/createcluster.go b/cmd/kind/create/cluster/createcluster.go index 8e27f1d27a..fd1a39f8e6 100644 --- a/cmd/kind/create/cluster/createcluster.go +++ b/cmd/kind/create/cluster/createcluster.go @@ -76,16 +76,31 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error { // TODO(fabrizio pandini): this check is temporary / WIP // kind v1alpha config fully supports multi nodes, but the cluster creation logic implemented in - // pkg/cluster/contex.go does not (yet). - // As soon a multi node support is implemented in pkg/cluster/contex.go, this should go away - if len(cfg.Nodes()) > 1 { - return fmt.Errorf("multi node support is still a work in progress, currently only single node cluster are supported") + // pkg/cluster/contex.go does it only partially (yet). + // As soon a external load-balancer and external etcd is implemented in pkg/cluster, this should go away + + if cfg.ExternalLoadBalancer() != nil { + return fmt.Errorf("multi node support is still a work in progress, currently external load balancer node is not supported") + } + + if cfg.SecondaryControlPlanes() != nil { + return fmt.Errorf("multi node support is still a work in progress, currently only single control-plane node are supported") + } + + if cfg.ExternalEtcd() != nil { + return fmt.Errorf("multi node support is still a work in progress, currently external etcd node is not supported") } // create a cluster context and create the cluster ctx := cluster.NewContext(flags.Name) if flags.ImageName != "" { - cfg.BootStrapControlPlane().Image = flags.ImageName + // Apply image override to all the Nodes defined in Config + // TODO(fabrizio pandini): this should be reconsidered when implementing + // https://github.com/kubernetes-sigs/kind/issues/133 + for _, n := range cfg.Nodes() { + n.Image = flags.ImageName + } + err := cfg.Validate() if err != nil { log.Errorf("Invalid flags, configuration failed validation: %v", err) diff --git a/pkg/cluster/actions.go b/pkg/cluster/actions.go new file mode 100644 index 0000000000..9acfd9a300 --- /dev/null +++ b/pkg/cluster/actions.go @@ -0,0 +1,231 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "sort" + "sync" + + "sigs.k8s.io/kind/pkg/cluster/config" +) + +// Action define a set of tasks to be executed on a `kind` cluster. +// Usage of actions allows to define repetitive, high level abstractions/workflows +// by composing lower level tasks +type Action interface { + // Tasks returns the list of task that are identified by this action + // Please note that the order of task is important, and it will be + // respected during execution + Tasks() []Task +} + +// Task define a logical step of an action to be executed on a `kind` cluster. +// At exec time the logical step will then apply to the current cluster +// topology, and be planned for execution zero, one or many times accordingly. +type Task struct { + // Description of the task + Description string + // TargetNodes define a function that identifies the nodes where this + // task should be executed + TargetNodes NodeSelector + // Run the func that implements the task action + Run func(ec *execContext, configNode *config.Node) error +} + +// NodeSelector defines a function returning a subset of nodes where tasks +// should be planned. +type NodeSelector func(*config.Config) config.NodeList + +// PlannedTask defines a Task planned for execution on a given node. +type PlannedTask struct { + // task to be executed + Task Task + // node where the task should be executed + Node *config.Node + + // PlannedTask should respects the given order of actions and tasks + actionIndex int + taskIndex int +} + +// ExecutionPlan contain an ordered list of Planned Tasks +// Please note that the planning order is critical for providing a +// predictable, "kubeadm friendly" and consistent execution order. +type ExecutionPlan []*PlannedTask + +// internal registry of named Action implementations +var actionImpls = struct { + impls map[string]func() Action + sync.Mutex +}{ + impls: map[string]func() Action{}, +} + +// RegisterAction registers a new named actionBuilder function for use +func RegisterAction(name string, actionBuilderFunc func() Action) { + actionImpls.Lock() + actionImpls.impls[name] = actionBuilderFunc + actionImpls.Unlock() +} + +// GetAction returns one instance of a registered action +func GetAction(name string) (Action, error) { + actionImpls.Lock() + actionBuilderFunc, ok := actionImpls.impls[name] + actionImpls.Unlock() + if !ok { + return nil, fmt.Errorf("no Action implementation with name: %s", name) + } + return actionBuilderFunc(), nil +} + +// NewExecutionPlan creates an execution plan by applying logical step/task +// defined for each action to the actual cluster topology. As a result task +// could be executed zero, one or more times according with the target nodes +// selector defined for each task. +// The execution plan is ordered, providing a predictable, "kubeadm friendly" +// and consistent execution order; with this regard please note that the order +// of actions is important, and it will be respected by planning. +// TODO(fabrizio pandini): probably it will be necessary to add another criteria +// for ordering planned task for the most complex workflows (e.g. +// init-join-upgrade and then join again) +// e.g. it should be something like "action group" where each action +// group is a list of actions +func NewExecutionPlan(cfg *config.Config, actionNames []string) (ExecutionPlan, error) { + // for each actionName + var plan = ExecutionPlan{} + for i, name := range actionNames { + // get the action implementation instance + actionImpl, err := GetAction(name) + if err != nil { + return nil, err + } + // for each logical tasks defined for the action + for j, t := range actionImpl.Tasks() { + // get the list of target nodes in the current topology + targetNodes := t.TargetNodes(cfg) + for _, n := range targetNodes { + // creates the planned task + taskContext := &PlannedTask{ + Node: n, + Task: t, + actionIndex: i, + taskIndex: j, + } + plan = append(plan, taskContext) + } + } + } + + // sorts the list of planned task ensuring a predictable, "kubeadm friendly" + // and consistent execution order + sort.Sort(plan) + return plan, nil +} + +// Len of the ExecutionPlan. +// It is required for making ExecutionPlan sortable. +func (t ExecutionPlan) Len() int { + return len(t) +} + +// Less return the lower between two elements of the ExecutionPlan, where the +// lower element should be executed before the other. +// It is required for making ExecutionPlan sortable. +func (t ExecutionPlan) Less(i, j int) bool { + return t[i].ExecutionOrder() < t[j].ExecutionOrder() +} + +// Swap two elements of the ExecutionPlan. +// It is required for making ExecutionPlan sortable. +func (t ExecutionPlan) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + +// ExecutionOrder returns a string that can be used for sorting planned tasks +// into a predictable, "kubeadm friendly" and consistent order. +// NB. we are using a string to combine all the item considered into something +// that can be easily sorted using a lexicographical order +func (p *PlannedTask) ExecutionOrder() string { + return fmt.Sprintf("Node.ProvisioningOrder: %d - Node.Name: %s - actionIndex: %d - taskIndex: %d", + // Then PlannedTask are grouped by machines, respecting the kubeadm node + // ProvisioningOrder: first complete provisioning on bootstrap control + // plane, then complete provisioning of secondary control planes, and + // finally provision worker nodes. + p.Node.ProvisioningOrder(), + p.Node.Name, + + // When planning task for one machine, the given order of actions will + // be respected and, for each action, the predefined order of tasks + // will be used + p.actionIndex, + p.taskIndex, + ) +} + +// SelectAllNodes is a NodeSelector that returns all the nodes defined in +// the `kind` Config +func SelectAllNodes(cfg *config.Config) config.NodeList { + return cfg.Nodes() +} + +// SelectControlPlaneNodes is a NodeSelector that returns all the nodes +// with control-plane role +func SelectControlPlaneNodes(cfg *config.Config) config.NodeList { + return cfg.ControlPlanes() +} + +// SelectBootstrapControlPlaneNode is a NodeSelector that returns the +// first node with control-plane role +func SelectBootstrapControlPlaneNode(cfg *config.Config) config.NodeList { + if cfg.BootStrapControlPlane() != nil { + return config.NodeList{cfg.BootStrapControlPlane()} + } + return nil +} + +// SelectSecondaryControlPlaneNodes is a NodeSelector that returns all +// the nodes with control-plane roleexcept the BootStrapControlPlane +// node, if any, +func SelectSecondaryControlPlaneNodes(cfg *config.Config) config.NodeList { + return cfg.SecondaryControlPlanes() +} + +// SelectWorkerNodes is a NodeSelector that returns all the nodes with +// Worker role, if any +func SelectWorkerNodes(cfg *config.Config) config.NodeList { + return cfg.Workers() +} + +// SelectExternalEtcdNode is a NodeSelector that returns the node with +//external-etcd role, if defined +func SelectExternalEtcdNode(cfg *config.Config) config.NodeList { + if cfg.ExternalEtcd() != nil { + return config.NodeList{cfg.ExternalEtcd()} + } + return nil +} + +// SelectExternalLoadBalancerNode is a NodeSelector that returns the node +// with external-load-balancer role, if defined +func SelectExternalLoadBalancerNode(cfg *config.Config) config.NodeList { + if cfg.ExternalLoadBalancer() != nil { + return config.NodeList{cfg.ExternalLoadBalancer()} + } + return nil +} diff --git a/pkg/cluster/actions_test.go b/pkg/cluster/actions_test.go new file mode 100644 index 0000000000..dc00e639d7 --- /dev/null +++ b/pkg/cluster/actions_test.go @@ -0,0 +1,252 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "reflect" + "sort" + "testing" + + "sigs.k8s.io/kind/pkg/cluster/config" +) + +func TestExecutionPlanSorting(t *testing.T) { + cases := []struct { + TestName string + actual ExecutionPlan + expected ExecutionPlan + }{ + { + TestName: "ExecutionPlan is ordered by provisioning order as a first criteria", + actual: ExecutionPlan{ + &PlannedTask{Node: &config.Node{Name: "worker2", Role: config.WorkerRole}}, + &PlannedTask{Node: &config.Node{Name: "control-plane2", Role: config.ControlPlaneRole}}, + &PlannedTask{Node: &config.Node{Name: "etcd", Role: config.ExternalEtcdRole}}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}}, + }, + expected: ExecutionPlan{ + &PlannedTask{Node: &config.Node{Name: "etcd", Role: config.ExternalEtcdRole}}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}}, + &PlannedTask{Node: &config.Node{Name: "control-plane2", Role: config.ControlPlaneRole}}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}}, + &PlannedTask{Node: &config.Node{Name: "worker2", Role: config.WorkerRole}}, + }, + }, + { + TestName: "ExecutionPlan respects the given action order as a second criteria", + actual: ExecutionPlan{ + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 3}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 2}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 1}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 1}, + }, + expected: ExecutionPlan{ + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 1}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 2}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 1}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 3}, + }, + }, + { + TestName: "ExecutionPlan respects the predefined order for each action as a third criteria", + actual: ExecutionPlan{ + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 1, taskIndex: 2}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 1, taskIndex: 2}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 1, taskIndex: 1}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 1, taskIndex: 1}, + }, + expected: ExecutionPlan{ + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 1, taskIndex: 1}, + &PlannedTask{Node: &config.Node{Name: "control-plane1", Role: config.ControlPlaneRole}, actionIndex: 1, taskIndex: 2}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 1, taskIndex: 1}, + &PlannedTask{Node: &config.Node{Name: "worker1", Role: config.WorkerRole}, actionIndex: 1, taskIndex: 2}, + }, + }, + } + + for _, c := range cases { + t.Run(c.TestName, func(t2 *testing.T) { + // sorting planned task + sort.Sort(c.actual) + + // cheching planned tasks are properly sorted + if !reflect.DeepEqual(c.actual, c.expected) { + t2.Errorf("Expected machineSets") + for _, m := range c.expected { + t2.Logf(" %s on %s, actionIndex %d taskIndex %d", m.Task.Description, m.Node.Name, m.actionIndex, m.taskIndex) + } + t2.Log("Saw") + for _, m := range c.actual { + t2.Logf(" %s on %s, actionIndex %d taskIndex %d", m.Task.Description, m.Node.Name, m.actionIndex, m.taskIndex) + } + } + }) + } +} + +// dummy action with single task targeting all nodes +type action0 struct{} + +func newAction0() Action { + return &action0{} +} + +func (b *action0) Tasks() []Task { + return []Task{ + { + Description: "action0 - task 0/all", + TargetNodes: SelectAllNodes, + }, + } +} + +// dummy action with single task targeting control-plane nodes +type action1 struct{} + +func newAction1() Action { + return &action1{} +} + +func (b *action1) Tasks() []Task { + return []Task{ + { + Description: "action1 - task 0/control-planes", + TargetNodes: SelectControlPlaneNodes, + }, + } +} + +// dummy action with multiple tasks each with different targets +type action2 struct{} + +func newAction2() Action { + return &action2{} +} + +func (b *action2) Tasks() []Task { + return []Task{ + { + Description: "action2 - task 0/all", + TargetNodes: SelectAllNodes, + }, + { + Description: "action2 - task 1/control-planes", + TargetNodes: SelectControlPlaneNodes, + }, + { + Description: "action2 - task 2/workers", + TargetNodes: SelectWorkerNodes, + }, + } +} + +func TestNewExecutionPlan(t *testing.T) { + testTopology := []*config.Node{ + {Name: "control-plane", Role: config.ControlPlaneRole}, + {Name: "worker1", Role: config.WorkerRole}, + {Name: "worker2", Role: config.WorkerRole}, + } + + RegisterAction("action0", newAction0) // Task 0 -> allMachines + RegisterAction("action1", newAction1) // Task 0 -> controlPlaneMachines + RegisterAction("action2", newAction2) // Task 0 -> allMachines, Task 1 -> controlPlaneMachines, Task 2 -> workerMachines + + cases := []struct { + TestName string + Actions []string + Nodes []*config.Node + ExpextedPlan []string + }{ + { + TestName: "Action with task targeting all machines is planned", + Actions: []string{"action0"}, + Nodes: testTopology, + ExpextedPlan: []string{ + "action0 - task 0/all on control-plane", + "action0 - task 0/all on worker1", + "action0 - task 0/all on worker2", + }, + }, + { + TestName: "Action with task targeting control-plane nodes is planned", + Actions: []string{"action1"}, + Nodes: testTopology, + ExpextedPlan: []string{ + "action1 - task 0/control-planes on control-plane", + }, + }, + { + TestName: "Action with many task and targets is planned", + Actions: []string{"action2"}, + Nodes: testTopology, + ExpextedPlan: []string{ // task are grouped by machine/provision order and task order is preserved + "action2 - task 0/all on control-plane", + "action2 - task 1/control-planes on control-plane", + "action2 - task 0/all on worker1", + "action2 - task 2/workers on worker1", + "action2 - task 0/all on worker2", + "action2 - task 2/workers on worker2", + }, + }, + { + TestName: "Many actions are planned", + Actions: []string{"action0", "action1", "action2"}, + Nodes: testTopology, + ExpextedPlan: []string{ // task are grouped by machine/provision order and action order/task order is preserved + "action0 - task 0/all on control-plane", + "action1 - task 0/control-planes on control-plane", + "action2 - task 0/all on control-plane", + "action2 - task 1/control-planes on control-plane", + "action0 - task 0/all on worker1", + "action2 - task 0/all on worker1", + "action2 - task 2/workers on worker1", + "action0 - task 0/all on worker2", + "action2 - task 0/all on worker2", + "action2 - task 2/workers on worker2", + }, + }, + } + + for _, c := range cases { + t.Run(c.TestName, func(t2 *testing.T) { + var cfg = &config.Config{} + // Adding nodes to the config + for _, n := range c.Nodes { + if err := cfg.Add(n); err != nil { + t2.Fatalf("unexpected error while adding nodes: %v", err) + break + } + } + // Creating the execution plane + tasks, _ := NewExecutionPlan(cfg, c.Actions) + + // Checking planned task are properly created (and sorted) + if len(tasks) != len(c.ExpextedPlan) { + t2.Fatalf("Invalid PlannedTask expected %d elements, saw %d", len(c.ExpextedPlan), len(tasks)) + } + + for i, mt := range tasks { + r := fmt.Sprintf("%s on %s", mt.Task.Description, mt.Node.Name) + if r != c.ExpextedPlan[i] { + t2.Errorf("Invalid PlannedTask %d expected %v, saw %v", i, c.ExpextedPlan[i], r) + } + } + }) + } +} diff --git a/pkg/cluster/consts/consts.go b/pkg/cluster/consts/consts.go index 0322d7a972..a2007c1060 100644 --- a/pkg/cluster/consts/consts.go +++ b/pkg/cluster/consts/consts.go @@ -19,3 +19,6 @@ package consts // ClusterLabelKey is applied to each "node" docker container for identification const ClusterLabelKey = "io.k8s.sigs.kind.cluster" + +// ClusterRoleKey is applied to each "node" docker container for categorization of nodes by role +const ClusterRoleKey = "io.k8s.sigs.kind.role" diff --git a/pkg/cluster/context.go b/pkg/cluster/context.go index de107d5c44..306b0e79fb 100644 --- a/pkg/cluster/context.go +++ b/pkg/cluster/context.go @@ -18,25 +18,19 @@ package cluster import ( "fmt" - "io/ioutil" "os" "path/filepath" "regexp" "strings" "time" - "sigs.k8s.io/kind/pkg/cluster/logs" - - "sigs.k8s.io/kind/pkg/cluster/consts" - - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "sigs.k8s.io/kind/pkg/cluster/config" - "sigs.k8s.io/kind/pkg/cluster/kubeadm" + "sigs.k8s.io/kind/pkg/cluster/consts" + "sigs.k8s.io/kind/pkg/cluster/logs" "sigs.k8s.io/kind/pkg/cluster/nodes" "sigs.k8s.io/kind/pkg/docker" - "sigs.k8s.io/kind/pkg/kustomize" logutil "sigs.k8s.io/kind/pkg/log" ) @@ -117,7 +111,7 @@ func (c *Context) KubeConfigPath() string { // TODO(bentheelder): Windows? // configDir matches the standard directory expected by kubectl etc configDir := filepath.Join(os.Getenv("HOME"), ".kube") - // note that the file name however does not, we do not want to overwite + // note that the file name however does not, we do not want to overwrite // the standard config, though in the future we may (?) merge them fileName := fmt.Sprintf("kind-config-%s", c.name) return filepath.Join(configDir, fileName) @@ -130,48 +124,52 @@ func (c *Context) Create(cfg *config.Config, retain bool, wait time.Duration) er return err } + fmt.Printf("Creating cluster '%s' ...\n", c.ClusterName()) + + // init the create context and logging cc := &createContext{ - Context: c, - config: cfg, - retain: retain, - waitForReady: wait, + Context: c, + config: cfg, + retain: retain, } - fmt.Printf("Creating cluster '%s' ...\n", c.ClusterName()) cc.status = logutil.NewStatus(os.Stdout) cc.status.MaybeWrapLogrus(log.StandardLogger()) defer cc.status.End(false) - // TODO(fabrizio pandini): usage of BootStrapControlPlane() is temporary / WIP - // kind v1alpha2 config fully supports multi nodes, but the cluster creation logic implemented in - // in this file does not (yet). - image := cfg.BootStrapControlPlane().Image - if strings.Contains(image, "@sha256:") { - image = strings.Split(image, "@sha256:")[0] - } - cc.status.Start(fmt.Sprintf("Ensuring node image (%s) πŸ–Ό", image)) - - // attempt to explicitly pull the image if it doesn't exist locally + // attempt to explicitly pull the required node images if they doesn't exist locally // we don't care if this errors, we'll still try to run which also pulls - _, _ = docker.PullIfNotPresent(cfg.BootStrapControlPlane().Image, 4) + cc.EnsureNodeImages() - // TODO(bentheelder): multiple nodes ... - kubeadmConfig, err := cc.provisionControlPlane( - fmt.Sprintf("kind-%s-control-plane", c.name), - ) + // Create node containers implementing defined config Nodes + nodeList, err := cc.provisionNodes() + if err != nil { + // In case of errors nodes are deleted (except if retain is explicitly set) + log.Error(err) + if !cc.retain { + cc.Delete() + } + return err + } c.ControlPlaneMeta = cc.ControlPlaneMeta + cc.status.End(true) - // clean up the kubeadm config file - // NOTE: in the future we will use this for other nodes first - if kubeadmConfig != "" { - defer os.Remove(kubeadmConfig) - } + // After creating node containers the Kubernetes provisioning is executed + // By default `kind` executes all the actions required to get a fully working + // Kubernetes cluster; please note that the list of actions automatically + // adapt to the topology defined in config + // TODO(fabrizio pandini): make the list of executed actions configurable from CLI + err = c.Exec(cc.config, nodeList, []string{"config", "init", "join"}, wait) if err != nil { + // In case of errors nodes are deleted (except if retain is explicitly set) + log.Error(err) + if !cc.retain { + cc.Delete() + } return err } - cc.status.End(true) fmt.Printf( "Cluster creation complete. You can now use the cluster with:\n\nexport KUBECONFIG=\"$(kind get kubeconfig-path --name=%q)\"\nkubectl cluster-info\n", cc.Name(), @@ -179,26 +177,7 @@ func (c *Context) Create(cfg *config.Config, retain bool, wait time.Duration) er return nil } -// Delete tears down a kubernetes-in-docker cluster -func (c *Context) Delete() error { - n, err := c.ListNodes() - if err != nil { - return fmt.Errorf("error listing nodes: %v", err) - } - - // try to remove the kind kube config file generated by "kind create cluster" - err = os.Remove(c.KubeConfigPath()) - if err != nil { - log.Warningf("Tried to remove %s but received error: %s\n", c.KubeConfigPath(), err) - } - - // check if $KUBECONFIG is set and let the user know to unset if so - if os.Getenv("KUBECONFIG") == c.KubeConfigPath() { - fmt.Printf("$KUBECONFIG is still set to use %s even though that file has been deleted, remember to unset it\n", c.KubeConfigPath()) - } - - return nodes.Delete(n...) -} +// TODO(bentheelder): fix this after multi-node changes (!) // ControlPlaneMeta tracks various outputs that are relevant to the control plane created with Kind. // Here we can define things like ports and listen or bind addresses as needed. @@ -208,201 +187,150 @@ type ControlPlaneMeta struct { APIServerPort int } -// provisionControlPlane provisions the control plane node -// and the cluster kubeadm config -func (cc *createContext) provisionControlPlane( - nodeName string, -) (kubeadmConfigPath string, err error) { - cc.status.Start(fmt.Sprintf("[%s] Creating node container πŸ“¦", nodeName)) - // create the "node" container (docker run, but it is paused, see createNode) - node, port, err := nodes.CreateControlPlaneNode(nodeName, cc.config.BootStrapControlPlane().Image, cc.ClusterLabel()) - if err != nil { - return "", err - } - cc.ControlPlaneMeta = &ControlPlaneMeta{ - APIServerPort: port, - // TODO (@kris-nova) add node information - } +// Ensure node images are present +func (cc *createContext) EnsureNodeImages() { + var images = map[string]bool{} - cc.status.Start(fmt.Sprintf("[%s] Fixing mounts πŸ—»", nodeName)) - // we need to change a few mounts once we have the container - // we'd do this ahead of time if we could, but --privileged implies things - // that don't seem to be configurable, and we need that flag - if err := node.FixMounts(); err != nil { - // TODO(bentheelder): logging here - if !cc.retain { - nodes.Delete(*node) + // For all the nodes defined in the `kind` config + for _, configNode := range cc.config.Nodes() { + if _, ok := images[configNode.Image]; ok { + continue } - return "", err - } - - // run any pre-boot hooks - if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PreBoot { - if err := runHook(node, &hook, "preBoot"); err != nil { - return "", err - } + // prints user friendly message + image := configNode.Image + if strings.Contains(image, "@sha256:") { + image = strings.Split(image, "@sha256:")[0] } - } + cc.status.Start(fmt.Sprintf("Ensuring node image (%s) πŸ–Ό", image)) - cc.status.Start(fmt.Sprintf("[%s] Starting systemd πŸ–₯", nodeName)) - // signal the node entrypoint to continue booting into systemd - if err := node.SignalStart(); err != nil { - // TODO(bentheelder): logging here - if !cc.retain { - nodes.Delete(*node) - } - return "", err - } + // attempt to explicitly pull the image if it doesn't exist locally + // we don't care if this errors, we'll still try to run which also pulls + _, _ = docker.PullIfNotPresent(configNode.Image, 4) - cc.status.Start(fmt.Sprintf("[%s] Waiting for docker to be ready πŸ‹", nodeName)) - // wait for docker to be ready - if !node.WaitForDocker(time.Now().Add(time.Second * 30)) { - // TODO(bentheelder): logging here - if !cc.retain { - nodes.Delete(*node) - } - return "", fmt.Errorf("timed out waiting for docker to be ready on node") + // marks the images as already pulled + images[configNode.Image] = true } +} - // load the docker image artifacts into the docker daemon - node.LoadImages() +// provisionNodes takes care of creating all the containers +// that will host `kind` nodes +func (cc *createContext) provisionNodes() (nodeList map[string]*nodes.Node, err error) { + nodeList = map[string]*nodes.Node{} - // get installed kubernetes version from the node image - kubeVersion, err := node.KubeVersion() - if err != nil { - // TODO(bentheelder): logging here - if !cc.retain { - nodes.Delete(*node) - } - return "", fmt.Errorf("failed to get kubernetes version from node: %v", err) - } + // For all the nodes defined in the `kind` config + for _, configNode := range cc.config.Nodes() { - // create kubeadm config file - kubeadmConfig, err := cc.createKubeadmConfig( - cc.config, - kubeadm.ConfigData{ - ClusterName: cc.ClusterName(), - KubernetesVersion: kubeVersion, - APIBindPort: port, - }, - ) - if err != nil { - if !cc.retain { - nodes.Delete(*node) - } - return "", fmt.Errorf("failed to create kubeadm config: %v", err) - } + cc.status.Start(fmt.Sprintf("[%s] Creating node container πŸ“¦", configNode.Name)) + // create the node into a container (docker run, but it is paused, see createNode) + var name = fmt.Sprintf("kind-%s-%s", cc.name, configNode.Name) + var node *nodes.Node - // copy the config to the node - if err := node.CopyTo(kubeadmConfig, "/kind/kubeadm.conf"); err != nil { - // TODO(bentheelder): logging here - if !cc.retain { - nodes.Delete(*node) + switch configNode.Role { + case config.ControlPlaneRole: + node, err = nodes.CreateControlPlaneNode(name, configNode.Image, cc.ClusterLabel()) + case config.WorkerRole: + node, err = nodes.CreateWorkerNode(name, configNode.Image, cc.ClusterLabel()) + } + if err != nil { + return nodeList, err + } + nodeList[configNode.Name] = node + + cc.status.Start(fmt.Sprintf("[%s] Fixing mounts πŸ—»", configNode.Name)) + // we need to change a few mounts once we have the container + // we'd do this ahead of time if we could, but --privileged implies things + // that don't seem to be configurable, and we need that flag + if err := node.FixMounts(); err != nil { + // TODO(bentheelder): logging here + return nodeList, err } - return kubeadmConfig, errors.Wrap(err, "failed to copy kubeadm config to node") - } - // run any pre-kubeadm hooks - if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PreKubeadm { - if err := runHook(node, &hook, "preKubeadm"); err != nil { - return kubeadmConfig, err + // run any pre-boot hooks + if configNode.ControlPlane != nil && configNode.ControlPlane.NodeLifecycle != nil { + for _, hook := range configNode.ControlPlane.NodeLifecycle.PreBoot { + if err := runHook(node, &hook, "preBoot"); err != nil { + return nodeList, err + } } } - } - // run kubeadm - cc.status.Start( - fmt.Sprintf( - "[%s] Starting Kubernetes (this may take a minute) ☸", - nodeName, - )) - if err := node.Command( - // init because this is the control plane node - "kubeadm", "init", - // preflight errors are expected, in particular for swap being enabled - // TODO(bentheelder): limit the set of acceptable errors - "--ignore-preflight-errors=all", - // specify our generated config file - "--config=/kind/kubeadm.conf", - ).Run(); err != nil { - // TODO(bentheelder): logging here - // TODO(bentheelder): add a flag to retain the broken nodes for debugging - return kubeadmConfig, errors.Wrap(err, "failed to init node with kubeadm") - } + cc.status.Start(fmt.Sprintf("[%s] Starting systemd πŸ–₯", configNode.Name)) + // signal the node container entrypoint to continue booting into systemd + if err := node.SignalStart(); err != nil { + // TODO(bentheelder): logging here + return nodeList, err + } - // run any post-kubeadm hooks - if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PostKubeadm { - if err := runHook(node, &hook, "postKubeadm"); err != nil { - return kubeadmConfig, err - } + cc.status.Start(fmt.Sprintf("[%s] Waiting for docker to be ready πŸ‹", configNode.Name)) + // wait for docker to be ready + if !node.WaitForDocker(time.Now().Add(time.Second * 30)) { + // TODO(bentheelder): logging here + return nodeList, fmt.Errorf("timed out waiting for docker to be ready on node") } - } - // set up the $KUBECONFIG - kubeConfigPath := cc.KubeConfigPath() - if err = node.WriteKubeConfig(kubeConfigPath); err != nil { - // TODO(bentheelder): logging here - // TODO(bentheelder): add a flag to retain the broken nodes for debugging - return kubeadmConfig, errors.Wrap(err, "failed to get kubeconfig from node") - } + // load the docker image artifacts into the docker daemon + cc.status.Start(fmt.Sprintf("[%s] Pre-loading images πŸ‹", configNode.Name)) + node.LoadImages() - // TODO(bentheelder): support other overlay networks - if err = node.Command( - "/bin/sh", "-c", - `kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version --kubeconfig=/etc/kubernetes/admin.conf | base64 | tr -d '\n')"`, - ).Run(); err != nil { - return kubeadmConfig, errors.Wrap(err, "failed to apply overlay network") } - // if we are only provisioning one node, remove the master taint - // https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#master-isolation - // TODO(bentheelder): put this back when we have multi-node - //if cfg.NumNodes == 1 { - if err = node.Command( - "kubectl", "--kubeconfig=/etc/kubernetes/admin.conf", - "taint", "nodes", "--all", "node-role.kubernetes.io/master-", - ).Run(); err != nil { - return kubeadmConfig, errors.Wrap(err, "failed to remove master taint") + return nodeList, nil +} + +// Exec actions on kubernetes-in-docker cluster +// Actions are repetitive, high level abstractions/workflows composed +// by one or more lower level tasks, that automatically adapt to the +// current cluster topology +// TODO(fabrizio pandini): make Exec accessible from CLI via +// a separated kind exec cluster command or something similar +func (c *Context) Exec(cfg *config.Config, nodeList map[string]*nodes.Node, actions []string, wait time.Duration) error { + // validate config first + if err := cfg.Validate(); err != nil { + return err } - //} - // add the default storage class - if err := addDefaultStorageClass(node); err != nil { - return kubeadmConfig, errors.Wrap(err, "failed to add default storage class") + // init the exec context and logging + ec := &execContext{ + Context: c, + config: cfg, + nodes: nodeList, + waitForReady: wait, } - // run any post-overlay hooks - if cc.config.BootStrapControlPlane().ControlPlane != nil && cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle != nil { - for _, hook := range cc.config.BootStrapControlPlane().ControlPlane.NodeLifecycle.PostSetup { - if err := runHook(node, &hook, "postSetup"); err != nil { - return kubeadmConfig, err - } - } + ec.status = logutil.NewStatus(os.Stdout) + ec.status.MaybeWrapLogrus(log.StandardLogger()) + + defer ec.status.End(false) + + // Create an ExecutionPlan that applies the given actions to the topology defined + // in the config + executionPlan, err := NewExecutionPlan(ec.config, actions) + if err != nil { + return err } - // Wait for the control plane node to reach Ready status. - isReady := nodes.WaitForReady(node, time.Now().Add(cc.waitForReady)) - if cc.waitForReady > 0 { - if !isReady { - log.Warn("timed out waiting for control plane to be ready") + // Executes all the selected action + // TODO(fabrizio pandini): add a flag to a filter PlannedTask by node + // (e.g. execute only on this node) or by other criteria tbd + for _, plannedTask := range executionPlan { + ec.status.Start(fmt.Sprintf("[%s] %s", plannedTask.Node.Name, plannedTask.Task.Description)) + + err := plannedTask.Task.Run(ec, plannedTask.Node) + if err != nil { + // in case of error, the execution plan is halted + log.Error(err) + return err } } + ec.status.End(true) - return kubeadmConfig, nil + return nil } -func addDefaultStorageClass(controlPlane *nodes.Node) error { - in := strings.NewReader(defaultStorageClassManifest) - cmd := controlPlane.Command( - "kubectl", - "--kubeconfig=/etc/kubernetes/admin.conf", "apply", "-f", "-", - ) - cmd.SetStdin(in) - return cmd.Run() +func (ec *execContext) NodeFor(configNode *config.Node) (node *nodes.Node, ok bool) { + node, ok = ec.nodes[configNode.Name] + return } // runHook runs a LifecycleHook on the node @@ -427,49 +355,25 @@ func runHook(node *nodes.Node, hook *config.LifecycleHook, phase string) error { return nil } -// createKubeadmConfig creates the kubeadm config file for the cluster -// by running data through the template and writing it to a temp file -// the config file path is returned, this file should be removed later -func (c *Context) createKubeadmConfig(cfg *config.Config, data kubeadm.ConfigData) (path string, err error) { - // create kubeadm config file - f, err := ioutil.TempFile("", "") - if err != nil { - return "", errors.Wrap(err, "failed to create kubeadm config") - } - path = f.Name() - // generate the config contents - config, err := kubeadm.Config(data) - if err != nil { - os.Remove(path) - return "", err - } - // apply patches - patchedConfig, err := kustomize.Build( - []string{config}, - cfg.BootStrapControlPlane().KubeadmConfigPatches, - cfg.BootStrapControlPlane().KubeadmConfigPatchesJSON6902, - ) +// Delete tears down a kubernetes-in-docker cluster +func (c *Context) Delete() error { + n, err := c.ListNodes() if err != nil { - os.Remove(path) - return "", err + return fmt.Errorf("error listing nodes: %v", err) } - // write to the file - log.Infof("Using KubeadmConfig:\n\n%s\n", patchedConfig) - _, err = f.WriteString(patchedConfig) + + // try to remove the kind kube config file generated by "kind create cluster" + err = os.Remove(c.KubeConfigPath()) if err != nil { - os.Remove(path) - return "", err + log.Warningf("Tried to remove %s but received error: %s\n", c.KubeConfigPath(), err) } - return path, nil -} -// config has slices of string, but we want bytes for kustomize -func stringSliceToByteSliceSlice(ss []string) [][]byte { - bss := [][]byte{} - for _, s := range ss { - bss = append(bss, []byte(s)) + // check if $KUBECONFIG is set and let the user know to unset if so + if os.Getenv("KUBECONFIG") == c.KubeConfigPath() { + fmt.Printf("$KUBECONFIG is still set to use %s even though that file has been deleted, remember to unset it\n", c.KubeConfigPath()) } - return bss + + return nodes.Delete(n...) } // ListNodes returns the list of container IDs for the "nodes" in the cluster diff --git a/pkg/cluster/kubeadm-config.go b/pkg/cluster/kubeadm-config.go new file mode 100644 index 0000000000..8e615d5257 --- /dev/null +++ b/pkg/cluster/kubeadm-config.go @@ -0,0 +1,136 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "sigs.k8s.io/kind/pkg/cluster/config" + "sigs.k8s.io/kind/pkg/cluster/kubeadm" + "sigs.k8s.io/kind/pkg/kustomize" +) + +// KubeadmConfigAction implements action for creating the kubadm config +// and deployng it on the bootrap control-plane node. +type KubeadmConfigAction struct{} + +func init() { + RegisterAction("config", NewKubeadmConfigAction) +} + +// NewKubeadmConfigAction returns a new KubeadmConfigAction +func NewKubeadmConfigAction() Action { + return &KubeadmConfigAction{} +} + +// Tasks returns the list of action tasks +func (b *KubeadmConfigAction) Tasks() []Task { + return []Task{ + { + // Creates the kubeadm config file on the BootstrapControlPlaneNode + Description: "Creating the kubeadm config file β›΅", + TargetNodes: SelectBootstrapControlPlaneNode, + Run: runKubeadmConfig, + }, + } +} + +// runKubeadmConfig creates a kubeadm config file locally and then +// copies it to the node +func runKubeadmConfig(ec *execContext, configNode *config.Node) error { + // get the target node for this task + node, ok := ec.NodeFor(configNode) + if !ok { + return fmt.Errorf("unable to get the handle for operating on node: %s", configNode.Name) + } + + // get installed kubernetes version from the node image + kubeVersion, err := node.KubeVersion() + if err != nil { + // TODO(bentheelder): logging here + return errors.Wrap(err, "failed to get kubernetes version from node: %v") + } + + // create kubeadm config file writing a local temp file + kubeadmConfig, err := createKubeadmConfig( + ec.config, + kubeadm.ConfigData{ + ClusterName: ec.name, + KubernetesVersion: kubeVersion, + APIBindPort: kubeadm.APIServerPort, + Token: kubeadm.Token, + // TODO(fabriziopandini): when external load-balancer will be + // implemented also controlPlaneAddress should be added + }, + ) + if err != nil { + // TODO(bentheelder): logging here + return fmt.Errorf("failed to create kubeadm config: %v", err) + } + + // defer deletion of the local temp file + defer os.Remove(kubeadmConfig) + + // copy the config to the node + if err := node.CopyTo(kubeadmConfig, "/kind/kubeadm.conf"); err != nil { + // TODO(bentheelder): logging here + return errors.Wrap(err, "failed to copy kubeadm config to node") + } + + return nil +} + +// createKubeadmConfig creates the kubeadm config file for the cluster +// by running data through the template and writing it to a temp file +// the config file path is returned, this file should be removed later +func createKubeadmConfig(cfg *config.Config, data kubeadm.ConfigData) (path string, err error) { + // create kubeadm config file + f, err := ioutil.TempFile("", "") + if err != nil { + return "", errors.Wrap(err, "failed to create kubeadm config") + } + path = f.Name() + // generate the config contents + config, err := kubeadm.Config(data) + if err != nil { + os.Remove(path) + return "", err + } + // apply patches + patchedConfig, err := kustomize.Build( + []string{config}, + cfg.BootStrapControlPlane().KubeadmConfigPatches, + cfg.BootStrapControlPlane().KubeadmConfigPatchesJSON6902, + ) + if err != nil { + os.Remove(path) + return "", err + } + // write to the file + log.Infof("Using KubeadmConfig:\n\n%s\n", patchedConfig) + _, err = f.WriteString(patchedConfig) + if err != nil { + os.Remove(path) + return "", err + } + return path, nil +} diff --git a/pkg/cluster/kubeadm-init.go b/pkg/cluster/kubeadm-init.go new file mode 100644 index 0000000000..fdb2eb1e61 --- /dev/null +++ b/pkg/cluster/kubeadm-init.go @@ -0,0 +1,171 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/pkg/errors" + + "sigs.k8s.io/kind/pkg/cluster/config" + "sigs.k8s.io/kind/pkg/cluster/kubeadm" + "sigs.k8s.io/kind/pkg/cluster/nodes" +) + +// KubeadmInitAction implements action for executing the kubadm init +// and a set of default post init operations like e.g. install the +// CNI network plugin. +type KubeadmInitAction struct{} + +func init() { + RegisterAction("init", NewKubeadmInitAction) +} + +// NewKubeadmInitAction returns a new KubeadmInitAction +func NewKubeadmInitAction() Action { + return &KubeadmInitAction{} +} + +// Tasks returns the list of action tasks +func (b *KubeadmInitAction) Tasks() []Task { + return []Task{ + { + // Run kubeadm init on the BootstrapControlPlaneNode + Description: "Starting Kubernetes (this may take a minute) ☸", + TargetNodes: SelectBootstrapControlPlaneNode, + Run: runKubeadmInit, + }, + } +} + +// runKubeadmConfig executes kubadm init and a set of default +// post init operations. +func runKubeadmInit(ec *execContext, configNode *config.Node) error { + // get the target node for this task + node, ok := ec.NodeFor(configNode) + if !ok { + return fmt.Errorf("unable to get the handle for operating on node: %s", configNode.Name) + } + + // run any pre-kubeadm hooks + if configNode.ControlPlane != nil && configNode.ControlPlane.NodeLifecycle != nil { + for _, hook := range configNode.ControlPlane.NodeLifecycle.PreKubeadm { + if err := runHook(node, &hook, "preKubeadm"); err != nil { + return err + } + } + } + + // run kubeadm + if err := node.Command( + // init because this is the control plane node + "kubeadm", "init", + // preflight errors are expected, in particular for swap being enabled + // TODO(bentheelder): limit the set of acceptable errors + "--ignore-preflight-errors=all", + // specify our generated config file + "--config=/kind/kubeadm.conf", + ).Run(); err != nil { + return errors.Wrap(err, "failed to init node with kubeadm") + } + + // run any post-kubeadm hooks + if configNode.ControlPlane != nil && configNode.ControlPlane.NodeLifecycle != nil { + for _, hook := range configNode.ControlPlane.NodeLifecycle.PostKubeadm { + if err := runHook(node, &hook, "postKubeadm"); err != nil { + return err + } + } + } + + // copies the kubeconfig files locally in order to make the cluster + // usable with kubectl. + // the kubeconfig file created by kubeadm internally to the node + // must be modified in order to use the random host port reserved + // for the API server and exposed by the node + + // retrives the random host where the API server is exposed + // TODO(fabrizio pandini): when external load-balancer will be + // implemented this should be modified accordingly + hostPort, err := node.Ports(kubeadm.APIServerPort) + if err != nil { + return errors.Wrap(err, "failed to get kubeconfig from node") + } + + kubeConfigPath := ec.KubeConfigPath() + if err := node.WriteKubeConfig(kubeConfigPath, hostPort); err != nil { + return errors.Wrap(err, "failed to get kubeconfig from node") + } + + // install the CNI network plugin + // TODO(bentheelder): support other overlay networks + if err := node.Command( + "/bin/sh", "-c", + `kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version --kubeconfig=/etc/kubernetes/admin.conf | base64 | tr -d '\n')"`, + ).Run(); err != nil { + return errors.Wrap(err, "failed to apply overlay network") + } + + // if we are only provisioning one node, remove the master taint + // https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#master-isolation + if len(ec.config.Nodes()) == 1 { + if err := node.Command( + "kubectl", "--kubeconfig=/etc/kubernetes/admin.conf", + "taint", "nodes", "--all", "node-role.kubernetes.io/master-", + ).Run(); err != nil { + return errors.Wrap(err, "failed to remove master taint") + } + } + + // add the default storage class + if err := addDefaultStorageClass(node); err != nil { + return errors.Wrap(err, "failed to add default storage class") + } + + // run any post-setup hooks + if configNode.ControlPlane != nil && configNode.ControlPlane.NodeLifecycle != nil { + for _, hook := range configNode.ControlPlane.NodeLifecycle.PostSetup { + if err := runHook(node, &hook, "postSetup"); err != nil { + return err + } + } + } + + // Wait for the control plane node to reach Ready status. + isReady := nodes.WaitForReady(node, time.Now().Add(ec.waitForReady)) + if ec.waitForReady > 0 { + if !isReady { + log.Warn("timed out waiting for control plane to be ready") + } + } + + return nil +} + +func addDefaultStorageClass(controlPlane *nodes.Node) error { + in := strings.NewReader(defaultStorageClassManifest) + cmd := controlPlane.Command( + "kubectl", + "--kubeconfig=/etc/kubernetes/admin.conf", "apply", "-f", "-", + ) + cmd.SetStdin(in) + return cmd.Run() +} diff --git a/pkg/cluster/kubeadm-join.go b/pkg/cluster/kubeadm-join.go new file mode 100644 index 0000000000..6531a652a1 --- /dev/null +++ b/pkg/cluster/kubeadm-join.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + + "github.com/pkg/errors" + + "sigs.k8s.io/kind/pkg/cluster/config" + "sigs.k8s.io/kind/pkg/cluster/kubeadm" +) + +// KubeadmJoinAction implements action for joining nodes +// to a Kubernetes cluster. +type KubeadmJoinAction struct{} + +func init() { + RegisterAction("join", NewKubeadmJoinAction) +} + +// NewKubeadmJoinAction returns a new KubeadmJoinAction +func NewKubeadmJoinAction() Action { + return &KubeadmJoinAction{} +} + +// Tasks returns the list of action tasks +func (b *KubeadmJoinAction) Tasks() []Task { + return []Task{ + // TODO(fabrizio pandini): add Run kubeadm join --experimental-master + // on SecondaryControlPlaneNodes + { + // Run kubeadm join on the WorkeNodes + Description: "Joining worker node to Kubernetes ☸", + TargetNodes: SelectWorkerNodes, + Run: runKubeadmJoin, + }, + } +} + +// runKubeadmJoin executes kubadm join +func runKubeadmJoin(ec *execContext, configNode *config.Node) error { + // before running join, it should be retrived + + // gets the node where + // TODO(fabrizio pandini): when external load-balancer will be + // implemented this should be modified accordingly + controlPlaneHandle, ok := ec.NodeFor(ec.config.BootStrapControlPlane()) + if !ok { + return fmt.Errorf("unable to get the handle for operating on node: %s", ec.config.BootStrapControlPlane().Name) + } + + // gets the IP of the bootstrap master node + controlPlaneIP, err := controlPlaneHandle.IP() + if err != nil { + return errors.Wrap(err, "failed to get IP for node") + } + + // get the target node for this task + node, ok := ec.NodeFor(configNode) + if !ok { + return fmt.Errorf("unable to get the handle for operating on node: %s", configNode.Name) + } + + // TODO(fabrizio pandini): might be we want to run pre-kubeadm hooks on workers too + + // run kubeadm + if err := node.Command( + "kubeadm", "join", + // the control plane address uses the docker ip and a well know APIServerPort that + // are accessible only inside the docker network + fmt.Sprintf("%s:%d", controlPlaneIP, kubeadm.APIServerPort), + // uses a well known token and skipping ca certification for automating TLS bootstrap process + "--token", kubeadm.Token, + "--discovery-token-unsafe-skip-ca-verification", + // preflight errors are expected, in particular for swap being enabled + // TODO(bentheelder): limit the set of acceptable errors + "--ignore-preflight-errors=all", + ).Run(); err != nil { + return errors.Wrap(err, "failed to join node with kubeadm") + } + + // TODO(fabrizio pandini): might be we want to run post-kubeadm hooks on workers too + + // TODO(fabrizio pandini): might be we want to run post-setup hooks on workers too + + return nil +} diff --git a/pkg/cluster/kubeadm/config.go b/pkg/cluster/kubeadm/config.go index decf08a1d5..11d8fbe47c 100644 --- a/pkg/cluster/kubeadm/config.go +++ b/pkg/cluster/kubeadm/config.go @@ -32,6 +32,8 @@ type ConfigData struct { KubernetesVersion string // The API Server port APIBindPort int + // The Token for TLS bootstrap + Token string // DerivedConfigData is populated by Derive() // These auto-generated fields are available to Config templates, // but not meant to be set by hand @@ -63,8 +65,12 @@ const ConfigTemplateAlphaV1orV2 = `# config generated by kind apiVersion: kubeadm.k8s.io/v1alpha2 kind: MasterConfiguration kubernetesVersion: {{.KubernetesVersion}} -clusterName: {{.ClusterName}} -# we use a random local port for the API server +clusterName: "{{.ClusterName}}" +# we use a well know token for TLS bootstrap +bootstrapTokens: +- token: "{{ .Token }}" +# we use a well know port for making the API server discoverable inside docker network. +# from the host machine such port will be accessible via a random local port instead. api: bindPort: {{.APIBindPort}} # we need nsswitch.conf so we use /etc/hosts @@ -86,7 +92,7 @@ const ConfigTemplateAlphaV3 = `# config generated by kind apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration kubernetesVersion: {{.KubernetesVersion}} -clusterName: {{.ClusterName}} +clusterName: "{{.ClusterName}}" # we need nsswitch.conf so we use /etc/hosts # https://github.com/kubernetes/kubernetes/issues/69195 apiServerExtraVolumes: @@ -102,7 +108,11 @@ apiServerCertSANs: [localhost] --- apiVersion: kubeadm.k8s.io/v1alpha3 kind: InitConfiguration -# we use a random local port for the API server +# we use a well know token for TLS bootstrap +bootstrapTokens: +- token: "{{ .Token }}" +# we use a well know port for making the API server discoverable inside docker network. +# from the host machine such port will be accessible via a random local port instead. apiEndpoint: bindPort: {{.APIBindPort}} --- diff --git a/pkg/cluster/kubeadm/const.go b/pkg/cluster/kubeadm/const.go index 201dfaacd1..c845845abb 100644 --- a/pkg/cluster/kubeadm/const.go +++ b/pkg/cluster/kubeadm/const.go @@ -19,3 +19,6 @@ package kubeadm // APIServerPort is the expected default APIServerPort on the control plane node(s) // https://kubernetes.io/docs/reference/access-authn-authz/controlling-access/#api-server-ports-and-ips const APIServerPort = 6443 + +// Token defines a dummy, well known token for automating TLS bootstrap process +const Token = "abcdef.0123456789abcdef" diff --git a/pkg/cluster/nodes/create.go b/pkg/cluster/nodes/create.go index a04da7a941..2940f8457f 100644 --- a/pkg/cluster/nodes/create.go +++ b/pkg/cluster/nodes/create.go @@ -21,6 +21,7 @@ import ( "net" "github.com/pkg/errors" + "sigs.k8s.io/kind/pkg/cluster/kubeadm" "sigs.k8s.io/kind/pkg/docker" ) @@ -42,14 +43,43 @@ func getPort() (int, error) { return port, nil } -// CreateControlPlaneNode `docker run`s the node image, note that due to -// images/node/entrypoint being the entrypoint, this container will -// effectively be paused until we call actuallyStartNode(...) -func CreateControlPlaneNode(name, image, clusterLabel string) (handle *Node, port int, err error) { - port, err = getPort() +// CreateControlPlaneNode creates a contol-plane node +// and gets ready for exposing the the API server +func CreateControlPlaneNode(name, image, clusterLabel string) (node *Node, err error) { + // gets a random host port for the API server + port, err := getPort() + if err != nil { + return nil, errors.Wrap(err, "failed to get port for API server") + } + + node, err = createNode(name, image, clusterLabel, + // publish selected port for the API server + "--expose", fmt.Sprintf("%d", port), + "-p", fmt.Sprintf("%d:%d", port, kubeadm.APIServerPort), + ) + if err != nil { + return node, err + } + + // stores the port mapping into the node internal state + node.ports = map[int]int{kubeadm.APIServerPort: port} + + return node, nil +} + +// CreateWorkerNode creates a worker node +func CreateWorkerNode(name, image, clusterLabel string) (node *Node, err error) { + node, err = createNode(name, image, clusterLabel) if err != nil { - return nil, 0, errors.Wrap(err, "failed to get port for API server") + return node, err } + return node, nil +} + +// createNode `docker run`s the node image, note that due to +// images/node/entrypoint being the entrypoint, this container will +// effectively be paused until we call actuallyStartNode(...) +func createNode(name, image, clusterLabel string, extraArgs ...string) (handle *Node, err error) { runArgs := []string{ "-d", // run the container detached // running containers in a container requires privileged @@ -67,13 +97,13 @@ func CreateControlPlaneNode(name, image, clusterLabel string) (handle *Node, por "--name", name, // ... and set the container name // label the node with the cluster ID "--label", clusterLabel, - // publish selected port for the API server - "--expose", fmt.Sprintf("%d", port), - "-p", fmt.Sprintf("%d:%d", port, port), // explicitly set the entrypoint "--entrypoint=/usr/local/bin/entrypoint", } + // adds node specific args + runArgs = append(runArgs, extraArgs...) + if docker.UsernsRemap() { // We need this argument in order to make this command work // in systems that have userns-remap enabled on the docker daemon @@ -88,6 +118,7 @@ func CreateControlPlaneNode(name, image, clusterLabel string) (handle *Node, por "/sbin/init", }, ) + // if there is a returned ID then we did create a container // we should return a handle so the caller can clean it up // we'll return a handle with the nice name though @@ -97,7 +128,19 @@ func CreateControlPlaneNode(name, image, clusterLabel string) (handle *Node, por } } if err != nil { - return handle, 0, errors.Wrap(err, "docker run error") + return handle, errors.Wrap(err, "docker run error") + } + + // Deletes the machine-id embedded in the node imaga and regenerate a new one. + // This is necessary because both kubelet and other components like weave net + // use machine-id internally to distinguish nodes. + if err := handle.Command("rm", "-f", "/etc/machine-id").Run(); err != nil { + return handle, errors.Wrap(err, "machine-id-setup error") } - return handle, port, nil + + if err := handle.Command("systemd-machine-id-setup").Run(); err != nil { + return handle, errors.Wrap(err, "machine-id-setup error") + } + + return handle, nil } diff --git a/pkg/cluster/nodes/node.go b/pkg/cluster/nodes/node.go index ad94a7edf6..858e466ce7 100644 --- a/pkg/cluster/nodes/node.go +++ b/pkg/cluster/nodes/node.go @@ -23,6 +23,8 @@ import ( "os" "path/filepath" "regexp" + "strconv" + "strings" "time" "github.com/pkg/errors" @@ -66,6 +68,8 @@ func (n *Node) Command(command string, args ...string) exec.Cmd { // like node.nodeCache = nodeCache{} type nodeCache struct { kubernetesVersion string + ip string + ports map[int]int containerCmder exec.Cmder } @@ -202,15 +206,64 @@ func (n *Node) KubeVersion() (version string, err error) { return n.nodeCache.kubernetesVersion, nil } +// IP returns the IP address of the node +func (n *Node) IP() (ip string, err error) { + // use the cached version first + if n.nodeCache.ip != "" { + return n.nodeCache.ip, nil + } + // retrive the IP address of the node using docker inspect + lines, err := docker.Inspect(n.nameOrID, "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}") + if err != nil { + return "", errors.Wrap(err, "failed to get file") + } + if len(lines) != 1 { + return "", fmt.Errorf("file should only be one line, got %d lines", len(lines)) + } + n.nodeCache.ip = strings.Trim(lines[0], "'") + return n.nodeCache.ip, nil +} + +// Ports returns a specific port mapping for the node +// Node by convention use well known ports internally, while random port +// are used for making the `kind`Β cluster accessible from the host machine +func (n *Node) Ports(containerPort int) (hostPort int, err error) { + // use the cached version first + if hostPort, ok := n.nodeCache.ports[containerPort]; ok { + return hostPort, nil + } + // retrive the specific port mapping using docker inspect + lines, err := docker.Inspect(n.nameOrID, fmt.Sprintf("{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}", containerPort)) + if err != nil { + return -1, errors.Wrap(err, "failed to get file") + } + if len(lines) != 1 { + return -1, fmt.Errorf("file should only be one line, got %d lines", len(lines)) + } + + if n.nodeCache.ports == nil { + n.nodeCache.ports = map[int]int{} + } + + n.nodeCache.ports[containerPort], err = strconv.Atoi(strings.Trim(lines[0], "'")) + if err != nil { + return -1, errors.Wrap(err, "failed to get file") + } + return n.nodeCache.ports[containerPort], nil +} + // matches kubeconfig server entry like: // server: https://172.17.0.2:6443 // which we rewrite to: // server: https://localhost:$PORT -var serverAddressRE = regexp.MustCompile(`^(\s+server:) https://.*:(\d+)$`) +var serverAddressRE = regexp.MustCompile(`^(\s+server:) https://.*:\d+$`) // WriteKubeConfig writes a fixed KUBECONFIG to dest // this should only be called on a control plane node -func (n *Node) WriteKubeConfig(dest string) error { +// While copyng to the host machine the control plane address +// is replaced with local host and the control plane port with +// a randomly generated port reserved during node creation. +func (n *Node) WriteKubeConfig(dest string, hostPort int) error { cmd := n.Command("cat", "/etc/kubernetes/admin.conf") lines, err := exec.CombinedOutputLines(cmd) if err != nil { @@ -222,7 +275,7 @@ func (n *Node) WriteKubeConfig(dest string) error { for _, line := range lines { match := serverAddressRE.FindStringSubmatch(line) if len(match) > 1 { - line = fmt.Sprintf("%s https://localhost:%s", match[1], match[len(match)-1]) + line = fmt.Sprintf("%s https://localhost:%d", match[1], hostPort) } buff.WriteString(line) buff.WriteString("\n") diff --git a/pkg/docker/inspect.go b/pkg/docker/inspect.go new file mode 100644 index 0000000000..e5ef40424d --- /dev/null +++ b/pkg/docker/inspect.go @@ -0,0 +1,34 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package docker + +import ( + "fmt" + + "sigs.k8s.io/kind/pkg/exec" +) + +// Inspect return low-level information on containers +func Inspect(containerNameOrID, format string) ([]string, error) { + cmd := exec.Command("docker", "inspect", + "-f", // format + fmt.Sprintf("'%s'", format), + containerNameOrID, // ... against the "node" container + ) + + return exec.CombinedOutputLines(cmd) +} From c6f53d196353e4111b4a7b0de1df81466f93dc11 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Tue, 11 Dec 2018 12:50:16 +0100 Subject: [PATCH 5/7] address some feedback --- pkg/cluster/actions.go | 19 ++++++++++--------- pkg/cluster/nodes/create.go | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/cluster/actions.go b/pkg/cluster/actions.go index 9acfd9a300..11cff56649 100644 --- a/pkg/cluster/actions.go +++ b/pkg/cluster/actions.go @@ -44,7 +44,7 @@ type Task struct { // task should be executed TargetNodes NodeSelector // Run the func that implements the task action - Run func(ec *execContext, configNode *config.Node) error + Run func(*execContext, *config.Node) error } // NodeSelector defines a function returning a subset of nodes where tasks @@ -151,12 +151,6 @@ func (t ExecutionPlan) Less(i, j int) bool { return t[i].ExecutionOrder() < t[j].ExecutionOrder() } -// Swap two elements of the ExecutionPlan. -// It is required for making ExecutionPlan sortable. -func (t ExecutionPlan) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} - // ExecutionOrder returns a string that can be used for sorting planned tasks // into a predictable, "kubeadm friendly" and consistent order. // NB. we are using a string to combine all the item considered into something @@ -168,9 +162,10 @@ func (p *PlannedTask) ExecutionOrder() string { // plane, then complete provisioning of secondary control planes, and // finally provision worker nodes. p.Node.ProvisioningOrder(), + // Node name is considered in order to get a predictable/repeatable ordering + // in case of many nodes with the same ProvisioningOrder p.Node.Name, - - // When planning task for one machine, the given order of actions will + // If both the two criteria above are equal, the given order of actions will // be respected and, for each action, the predefined order of tasks // will be used p.actionIndex, @@ -178,6 +173,12 @@ func (p *PlannedTask) ExecutionOrder() string { ) } +// Swap two elements of the ExecutionPlan. +// It is required for making ExecutionPlan sortable. +func (t ExecutionPlan) Swap(i, j int) { + t[i], t[j] = t[j], t[i] +} + // SelectAllNodes is a NodeSelector that returns all the nodes defined in // the `kind` Config func SelectAllNodes(cfg *config.Config) config.NodeList { diff --git a/pkg/cluster/nodes/create.go b/pkg/cluster/nodes/create.go index 2940f8457f..146c19aa7e 100644 --- a/pkg/cluster/nodes/create.go +++ b/pkg/cluster/nodes/create.go @@ -131,7 +131,7 @@ func createNode(name, image, clusterLabel string, extraArgs ...string) (handle * return handle, errors.Wrap(err, "docker run error") } - // Deletes the machine-id embedded in the node imaga and regenerate a new one. + // Deletes the machine-id embedded in the node image and regenerate a new one. // This is necessary because both kubelet and other components like weave net // use machine-id internally to distinguish nodes. if err := handle.Command("rm", "-f", "/etc/machine-id").Run(); err != nil { From 91c7a920c6e7af651cb9214aa8ae365b83249b81 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 8 Jan 2019 13:00:44 -0800 Subject: [PATCH 6/7] correct rebase (add back execContext) --- pkg/cluster/context.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/cluster/context.go b/pkg/cluster/context.go index 306b0e79fb..0df0f8d7ca 100644 --- a/pkg/cluster/context.go +++ b/pkg/cluster/context.go @@ -50,6 +50,22 @@ type createContext struct { ControlPlaneMeta *ControlPlaneMeta } +// execContext is a superset of Context used by helpers for Context.Create() +// and Context.Exec() command +// TODO(fabrizio pandini): might be we want to move all the actions in a separated +// package e.g. pkg/cluster/actions +// In order to do this a circular dependency should be avoided: +// pkg/cluster -- use -- pkg/cluster/actions +// pkg/cluster/actions -- use pkg/cluster execContext +type execContext struct { + *Context + status *logutil.Status + config *config.Config + // nodes contains the list of actual nodes (a node is a container implementing a config node) + nodes map[string]*nodes.Node + waitForReady time.Duration // Wait for the control plane node to be ready +} + // similar to valid docker container names, but since we will prefix // and suffix this name, we can relax it a little // see NewContext() for usage From 1371151a5e41ecf4c847d9081c4baa2d3d6fabeb Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 8 Jan 2019 14:58:58 -0800 Subject: [PATCH 7/7] add bootstrap token to betav1 --- pkg/cluster/kubeadm/config.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/kubeadm/config.go b/pkg/cluster/kubeadm/config.go index 11d8fbe47c..e02005e9ca 100644 --- a/pkg/cluster/kubeadm/config.go +++ b/pkg/cluster/kubeadm/config.go @@ -134,7 +134,7 @@ const ConfigTemplateBetaV1 = `# config generated by kind apiVersion: kubeadm.k8s.io/v1beta1 kind: ClusterConfiguration kubernetesVersion: {{.KubernetesVersion}} -clusterName: {{.ClusterName}} +clusterName: "{{.ClusterName}}" # on docker for mac we have to expose the api server via port forward, # so we need to ensure the cert is valid for localhost so we can talk # to the cluster after rewriting the kubeconfig to point to localhost @@ -143,7 +143,11 @@ apiServer: --- apiVersion: kubeadm.k8s.io/v1beta1 kind: InitConfiguration -# we use a random local port for the API server +# we use a well know token for TLS bootstrap +bootstrapTokens: +- token: "{{ .Token }}" +# we use a well know port for making the API server discoverable inside docker network. +# from the host machine such port will be accessible via a random local port instead. localAPIEndpoint: bindPort: {{.APIBindPort}} ---