From 27dd1acaeba758d9b4f897c2d7878a57dcec2f13 Mon Sep 17 00:00:00 2001 From: hsinhoyeh Date: Tue, 31 Jan 2023 14:13:04 +0800 Subject: [PATCH] feat: add multiple node support with --with_workers=N --- cmd/multikf/cmd_add.go | 3 +++ cmd/multikf/helper.go | 9 +++++++++ pkg/machine/docker/hostmachine.go | 1 + pkg/machine/docker/template/config.go | 3 ++- pkg/machine/machine.go | 1 + pkg/machine/vagrant/template/config.go | 3 ++- pkg/machine/vagrant/vagrant.go | 1 + pkg/template/config/config.go | 21 +++++++++++++++++++-- pkg/template/kind_template.go | 13 ++++++++++--- pkg/template/kind_template_test.go | 18 +++++++++++++++--- pkg/template/template.go | 4 ++++ 11 files changed, 67 insertions(+), 10 deletions(-) diff --git a/cmd/multikf/cmd_add.go b/cmd/multikf/cmd_add.go index 61a6167..cf56286 100644 --- a/cmd/multikf/cmd_add.go +++ b/cmd/multikf/cmd_add.go @@ -22,6 +22,7 @@ func NewAddCommand(logger log.Logger, ioStreams genericclioptions.IOStreams) *co withKubeflowDefaultPassword string // with kubeflow defaultpassword withIP string // with specific IP withAudit bool // with audit enabled + withWorkers int // with workers exportPorts string // export ports on hostmachine forceOverwrite bool // force overwrite existing config ) @@ -54,6 +55,7 @@ func NewAddCommand(logger log.Logger, ioStreams genericclioptions.IOStreams) *co exportPorts: exportPorts, forceOverwrite: forceOverwrite, auditEnabled: withAudit, + workers: withWorkers, }) if err != nil { return err @@ -89,6 +91,7 @@ func NewAddCommand(logger log.Logger, ioStreams genericclioptions.IOStreams) *co cmd.Flags().IntVar(&useGPUs, "use_gpus", 0, "use gpu resources (default: 0), possible value (0 or 1)") cmd.Flags().StringVar(&withIP, "with_ip", "0.0.0.0", "with a specific ip address for kubeapi (default: 0.0.0.0)") cmd.Flags().StringVar(&exportPorts, "export_ports", "", "export ports to host, delimited by comma(example: 8443:443 stands for mapping host port 8443 to container port 443)") + cmd.Flags().IntVar(&withWorkers, "with_workers", 0, "use workers (default: 0)") return cmd } diff --git a/cmd/multikf/helper.go b/cmd/multikf/helper.go index 0bfce1a..03c922e 100644 --- a/cmd/multikf/helper.go +++ b/cmd/multikf/helper.go @@ -58,6 +58,10 @@ func newMachineFactoryWithProvisioner(p machine.Provisioner, logger log.Logger) return vag, nil } +var ( + _ machine.MachineConfiger = &machineConfig{} +) + type machineConfig struct { logger log.Logger cpus int @@ -68,6 +72,7 @@ type machineConfig struct { defaultPassword string forceOverwrite bool auditEnabled bool + workers int } func (m machineConfig) GetCPUs() int { @@ -124,6 +129,10 @@ func (m machineConfig) GetForceOverwriteConfig() bool { return m.forceOverwrite } +func (m machineConfig) GetWorkers() int { + return m.workers +} + type kubeflowPlugin struct { withKubeflowDefaultPassword string kubeflowVersion plugins.TypePluginVersion diff --git a/pkg/machine/docker/hostmachine.go b/pkg/machine/docker/hostmachine.go index aa94ae4..3d66ff2 100644 --- a/pkg/machine/docker/hostmachine.go +++ b/pkg/machine/docker/hostmachine.go @@ -116,6 +116,7 @@ func (h *HostMachine) prepareFiles() error { h.options.GetExportPorts(), h.options.AuditEnabled(), filepath.Join(h.hostMachineDir, "audit-policy.yaml"), + h.options.GetWorkers(), ) vfolder := NewHostFolder(h.hostMachineDir) diff --git a/pkg/machine/docker/template/config.go b/pkg/machine/docker/template/config.go index 10a7006..aeb0514 100644 --- a/pkg/machine/docker/template/config.go +++ b/pkg/machine/docker/template/config.go @@ -9,7 +9,7 @@ type DockerHostmachineTemplateConfig struct { *pkgtemplateconfig.DefaultTemplateConfig } -func NewDockerHostmachineTemplateConfig(name string, cpus int, memory int, sshport int, kubeApiPort int, kubeApiIP string, gpus int, exportPorts []machine.ExportPortPair, auditEnabled bool, auditFileAbsolutePath string) *DockerHostmachineTemplateConfig { +func NewDockerHostmachineTemplateConfig(name string, cpus int, memory int, sshport int, kubeApiPort int, kubeApiIP string, gpus int, exportPorts []machine.ExportPortPair, auditEnabled bool, auditFileAbsolutePath string, workerCount int) *DockerHostmachineTemplateConfig { return &DockerHostmachineTemplateConfig{ DefaultTemplateConfig: pkgtemplateconfig.NewDefaultTemplateConfig( name, @@ -22,6 +22,7 @@ func NewDockerHostmachineTemplateConfig(name string, cpus int, memory int, sshpo exportPorts, auditEnabled, auditFileAbsolutePath, + workerCount, ), } } diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index 9131440..1a9acb1 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -16,6 +16,7 @@ type MachineConfiger interface { GetExportPorts() []ExportPortPair GetForceOverwriteConfig() bool AuditEnabled() bool + GetWorkers() int } type ExportPortPair struct { diff --git a/pkg/machine/vagrant/template/config.go b/pkg/machine/vagrant/template/config.go index 93f336e..2545eda 100644 --- a/pkg/machine/vagrant/template/config.go +++ b/pkg/machine/vagrant/template/config.go @@ -9,7 +9,7 @@ type VagrantTemplateConfig struct { *pkgtemplateconfig.DefaultTemplateConfig } -func NewVagrantTemplateConfig(name string, cpus int, memory int, sshport int, kubeApiPort int, kubeApiIP string, gpus int, exportPorts []machine.ExportPortPair, auditEnabled bool, auditFileAbsolutePath string) *VagrantTemplateConfig { +func NewVagrantTemplateConfig(name string, cpus int, memory int, sshport int, kubeApiPort int, kubeApiIP string, gpus int, exportPorts []machine.ExportPortPair, auditEnabled bool, auditFileAbsolutePath string, workerCount int) *VagrantTemplateConfig { return &VagrantTemplateConfig{ DefaultTemplateConfig: pkgtemplateconfig.NewDefaultTemplateConfig( name, @@ -22,6 +22,7 @@ func NewVagrantTemplateConfig(name string, cpus int, memory int, sshport int, ku exportPorts, auditEnabled, auditFileAbsolutePath, + workerCount, ), } } diff --git a/pkg/machine/vagrant/vagrant.go b/pkg/machine/vagrant/vagrant.go index d8589ba..d7cfa7d 100644 --- a/pkg/machine/vagrant/vagrant.go +++ b/pkg/machine/vagrant/vagrant.go @@ -206,6 +206,7 @@ func (v *VagrantMachine) prepareFiles() error { v.options.GetExportPorts(), v.options.AuditEnabled(), "/tmp/audit-policy.yaml", /*for vagrant, we will copy the file under /tmp and run local installation*/ + v.options.GetWorkers(), ) vfolder := NewVagrantFolder(v.vagrantMachineDir) diff --git a/pkg/template/config/config.go b/pkg/template/config/config.go index 4fa2c82..2b1d6e1 100644 --- a/pkg/template/config/config.go +++ b/pkg/template/config/config.go @@ -1,6 +1,13 @@ package config -import "github.com/footprintai/multikf/pkg/machine" +import ( + "github.com/footprintai/multikf/pkg/machine" + "github.com/footprintai/multikf/pkg/template" +) + +var ( + _ template.KindConfiger = &DefaultTemplateConfig{} +) type DefaultTemplateConfig struct { name string @@ -13,9 +20,10 @@ type DefaultTemplateConfig struct { exportPorts []machine.ExportPortPair auditEnabled bool auditFileAbsolutePath string + workerCount int } -func NewDefaultTemplateConfig(name string, cpus int, memory int, sshport int, kubeApiPort int, kubeApiIP string, gpus int, exportPorts []machine.ExportPortPair, auditEnabled bool, auditFileAbsolutePath string) *DefaultTemplateConfig { +func NewDefaultTemplateConfig(name string, cpus int, memory int, sshport int, kubeApiPort int, kubeApiIP string, gpus int, exportPorts []machine.ExportPortPair, auditEnabled bool, auditFileAbsolutePath string, workerCount int) *DefaultTemplateConfig { return &DefaultTemplateConfig{ name: name, cpus: cpus, @@ -27,6 +35,7 @@ func NewDefaultTemplateConfig(name string, cpus int, memory int, sshport int, ku exportPorts: exportPorts, auditEnabled: auditEnabled, auditFileAbsolutePath: auditFileAbsolutePath, + workerCount: workerCount, } } @@ -69,3 +78,11 @@ func (t *DefaultTemplateConfig) AuditEnabled() bool { func (t *DefaultTemplateConfig) AuditFileAbsolutePath() string { return t.auditFileAbsolutePath } + +func (t *DefaultTemplateConfig) GetWorkerIDs() []int { + ids := make([]int, t.workerCount, t.workerCount) + for i := 0; i < t.workerCount; i++ { + ids[i] = i + } + return ids +} diff --git a/pkg/template/kind_template.go b/pkg/template/kind_template.go index faed451..cebfc06 100644 --- a/pkg/template/kind_template.go +++ b/pkg/template/kind_template.go @@ -29,20 +29,21 @@ func (k *KindFileTemplate) Execute(w io.Writer) error { return nil } -type kindConfig interface { +type KindConfiger interface { NameGetter KubeAPIPortGetter KubeAPIIPGetter GpuGetter ExportPortsGetter AuditEnabler + WorkerIDsGetter } func (k *KindFileTemplate) Populate(v interface{}) error { - if _, isKindConfiger := v.(kindConfig); !isKindConfiger { + if _, isKindConfiger := v.(KindConfiger); !isKindConfiger { return fmt.Errorf("not implements kindConfig interface") } - c := v.(kindConfig) + c := v.(KindConfiger) k.Name = c.GetName() k.KubeAPIPort = c.GetKubeAPIPort() k.KubeAPIIP = c.GetKubeAPIIP() @@ -50,6 +51,7 @@ func (k *KindFileTemplate) Populate(v interface{}) error { k.ExportPorts = c.GetExportPorts() k.AuditEnabled = c.AuditEnabled() k.AuditFileAbsolutePath = c.AuditFileAbsolutePath() + k.WorkerIDs = c.GetWorkerIDs() return nil } @@ -63,6 +65,7 @@ type KindFileTemplate struct { ExportPorts []machine.ExportPortPair AuditEnabled bool AuditFileAbsolutePath string + WorkerIDs []int } var kindDefaultFileTemplate string = ` @@ -116,6 +119,10 @@ nodes: containerPath: /etc/kubernetes/policies/audit-policy.yaml readOnly: true {{- end}} +{{- range .WorkerIDs }} +- role: worker + image: kindest/node:v1.23.12@sha256:9402cf1330bbd3a0d097d2033fa489b2abe40d479cc5ef47d0b6a6960613148a +{{- end}} networking: apiServerAddress: {{.KubeAPIIP}} apiServerPort: {{.KubeAPIPort}} diff --git a/pkg/template/kind_template_test.go b/pkg/template/kind_template_test.go index a369362..2921082 100644 --- a/pkg/template/kind_template_test.go +++ b/pkg/template/kind_template_test.go @@ -34,6 +34,10 @@ func (s staticConfig) AuditFileAbsolutePath() string { return "" } +func (s staticConfig) GetWorkerIDs() []int { + return []int{1, 2, 3} +} + func (s staticConfig) GetExportPorts() []machine.ExportPortPair { return []machine.ExportPortPair{ machine.ExportPortPair{ @@ -67,7 +71,6 @@ nodes: nodeRegistration: kubeletExtraArgs: node-labels: "ingress-ready=true" - # image: footprintai/kind-node:v1.21.9 image: kindest/node:v1.23.12@sha256:9402cf1330bbd3a0d097d2033fa489b2abe40d479cc5ef47d0b6a6960613148a gpus: true extraPortMappings: @@ -77,6 +80,12 @@ nodes: - containerPort: 8083 hostPort: 443 protocol: TCP +- role: worker + image: kindest/node:v1.23.12@sha256:9402cf1330bbd3a0d097d2033fa489b2abe40d479cc5ef47d0b6a6960613148a +- role: worker + image: kindest/node:v1.23.12@sha256:9402cf1330bbd3a0d097d2033fa489b2abe40d479cc5ef47d0b6a6960613148a +- role: worker + image: kindest/node:v1.23.12@sha256:9402cf1330bbd3a0d097d2033fa489b2abe40d479cc5ef47d0b6a6960613148a networking: apiServerAddress: 1.2.3.4 apiServerPort: 8443 @@ -117,6 +126,10 @@ func (s auditConfig) AuditFileAbsolutePath() string { return "foo.bar.yaml" } +func (s auditConfig) GetWorkerIDs() []int { + return []int{} +} + func TestKindTemplateWithAudit(t *testing.T) { kt := NewKindTemplate() assert.NoError(t, kt.Populate(auditConfig{})) @@ -159,8 +172,7 @@ nodes: nodeRegistration: kubeletExtraArgs: node-labels: "ingress-ready=true" - # image: footprintai/kind-node:v1.21.9 - image: kindest/node:v1.21.14 + image: kindest/node:v1.23.12@sha256:9402cf1330bbd3a0d097d2033fa489b2abe40d479cc5ef47d0b6a6960613148a gpus: false extraPortMappings: - containerPort: 8081 diff --git a/pkg/template/template.go b/pkg/template/template.go index b654027..17cb3cf 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -49,3 +49,7 @@ type AuditEnabler interface { AuditEnabled() bool AuditFileAbsolutePath() string } + +type WorkerIDsGetter interface { + GetWorkerIDs() []int +}