From e2dc955003ec3d2d59664561203d6d89133b3e9c Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Mon, 15 Oct 2018 14:30:35 -0700 Subject: [PATCH] pkg/asset: add ClusterK8sIO, machines.Worker assets Add assets that can create the machinesets for workers. This depends on the cluster-apiserver to be deployed by the machine-api-operator, so the tectonic.sh will block on creating these machinesets objects until then. Using specific AMI for worker machinesets to keep consistency with current MAO implementation. --- pkg/asset/machines/aws/aws.go | 76 ++++++++++++ pkg/asset/machines/cluster_k8s_io.go | 73 +++++++++++ pkg/asset/machines/doc.go | 2 + pkg/asset/machines/libvirt/libvirt.go | 61 ++++++++++ pkg/asset/machines/userdata.go | 35 ++++++ pkg/asset/machines/worker.go | 127 ++++++++++++++++++++ pkg/asset/manifests/machine-api-operator.go | 6 +- pkg/asset/manifests/tectonic.go | 42 ++++--- 8 files changed, 402 insertions(+), 20 deletions(-) create mode 100644 pkg/asset/machines/aws/aws.go create mode 100644 pkg/asset/machines/cluster_k8s_io.go create mode 100644 pkg/asset/machines/doc.go create mode 100644 pkg/asset/machines/libvirt/libvirt.go create mode 100644 pkg/asset/machines/userdata.go create mode 100644 pkg/asset/machines/worker.go diff --git a/pkg/asset/machines/aws/aws.go b/pkg/asset/machines/aws/aws.go new file mode 100644 index 00000000000..177c97657a4 --- /dev/null +++ b/pkg/asset/machines/aws/aws.go @@ -0,0 +1,76 @@ +// Package aws generates Machine objects for aws. +package aws + +import ( + "text/template" + + "github.com/openshift/installer/pkg/types" +) + +// Config is used to generate the machine. +type Config struct { + ClusterName string + Replicas int64 + AMIID string + Tags map[string]string + Region string + Machine types.AWSMachinePoolPlatform +} + +// WorkerMachineSetTmpl is template for worker machineset. +var WorkerMachineSetTmpl = template.Must(template.New("aws-worker-machineset").Parse(` +apiVersion: cluster.k8s.io/v1alpha1 +kind: MachineSet +metadata: + name: {{.ClusterName}}-worker-0 + namespace: openshift-cluster-api + labels: + sigs.k8s.io/cluster-api-cluster: {{.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: worker + sigs.k8s.io/cluster-api-machine-type: worker +spec: + replicas: {{.Replicas}} + selector: + matchLabels: + sigs.k8s.io/cluster-api-machineset: worker + sigs.k8s.io/cluster-api-cluster: {{.ClusterName}} + template: + metadata: + labels: + sigs.k8s.io/cluster-api-machineset: worker + sigs.k8s.io/cluster-api-cluster: {{.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: worker + sigs.k8s.io/cluster-api-machine-type: worker + spec: + providerConfig: + value: + apiVersion: aws.cluster.k8s.io/v1alpha1 + kind: AWSMachineProviderConfig + ami: + id: {{.AMIID}} + instanceType: {{.Machine.InstanceType}} + placement: + region: {{.Region}} + subnet: + filters: + - name: "tag:Name" + values: + - "{{.ClusterName}}-worker-*" + iamInstanceProfile: + id: "{{.ClusterName}}-worker-profile" + tags: +{{- range $key,$value := .Tags}} + - name: "{{$key}}" + value: "{{$value}}" +{{- end}} + securityGroups: + - filters: + - name: "tag:Name" + values: + - "{{.ClusterName}}_worker_sg" + userDataSecret: + name: worker-user-data + versions: + kubelet: "" + controlPlane: "" +`)) diff --git a/pkg/asset/machines/cluster_k8s_io.go b/pkg/asset/machines/cluster_k8s_io.go new file mode 100644 index 00000000000..ef474f1ea29 --- /dev/null +++ b/pkg/asset/machines/cluster_k8s_io.go @@ -0,0 +1,73 @@ +package machines + +import ( + "bytes" + "text/template" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/types" +) + +// ClusterK8sIO generates the `Cluster.cluster.k8s.io/v1alpha1` object. +type ClusterK8sIO struct { + Raw []byte +} + +var _ asset.Asset = (*ClusterK8sIO)(nil) + +// Name returns a human friendly name for the ClusterK8sIO Asset. +func (c *ClusterK8sIO) Name() string { + return "Cluster.cluster.k8s.io/v1alpha1" +} + +// Dependencies returns all of the dependencies directly needed by the +// ClusterK8sIO asset +func (c *ClusterK8sIO) Dependencies() []asset.Asset { + return []asset.Asset{ + &installconfig.InstallConfig{}, + } +} + +// Generate generates the Worker asset. +func (c *ClusterK8sIO) Generate(dependencies asset.Parents) error { + installconfig := &installconfig.InstallConfig{} + dependencies.Get(installconfig) + + c.Raw = clusterK8sIO(installconfig.Config) + return nil +} + +var clusterK8sIOTmpl = template.Must(template.New("cluster").Parse(` +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Cluster +metadata: + name: {{.Name}} + namespace: openshift-cluster-api +spec: + clusterNetwork: + services: + cidrBlocks: + - {{.ServiceCIDR}} + pods: + cidrBlocks: + - {{.PodCIDR}} + serviceDomain: unused +`)) + +func clusterK8sIO(ic *types.InstallConfig) []byte { + templateData := struct { + Name string + ServiceCIDR string + PodCIDR string + }{ + Name: ic.ObjectMeta.Name, + ServiceCIDR: ic.Networking.ServiceCIDR.String(), + PodCIDR: ic.Networking.PodCIDR.String(), + } + buf := &bytes.Buffer{} + if err := clusterK8sIOTmpl.Execute(buf, templateData); err != nil { + panic(err) + } + return buf.Bytes() +} diff --git a/pkg/asset/machines/doc.go b/pkg/asset/machines/doc.go new file mode 100644 index 00000000000..77156e1a63c --- /dev/null +++ b/pkg/asset/machines/doc.go @@ -0,0 +1,2 @@ +// Package machines is responsible for creating Machine objects for machinepools. +package machines diff --git a/pkg/asset/machines/libvirt/libvirt.go b/pkg/asset/machines/libvirt/libvirt.go new file mode 100644 index 00000000000..ad20b8e0d31 --- /dev/null +++ b/pkg/asset/machines/libvirt/libvirt.go @@ -0,0 +1,61 @@ +// Package libvirt generates Machine objects for libvirt. +package libvirt + +import ( + "text/template" + + "github.com/openshift/installer/pkg/types" +) + +// Config is used to generate the machine. +type Config struct { + ClusterName string + Replicas int64 + Platform types.LibvirtPlatform +} + +// WorkerMachineSetTmpl is template for worker machineset. +var WorkerMachineSetTmpl = template.Must(template.New("worker-machineset").Parse(` +apiVersion: cluster.k8s.io/v1alpha1 +kind: MachineSet +metadata: + name: {{.ClusterName}}-worker-0 + namespace: openshift-cluster-api + labels: + sigs.k8s.io/cluster-api-cluster: {{.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: worker + sigs.k8s.io/cluster-api-machine-type: worker +spec: + replicas: {{.Replicas}} + selector: + matchLabels: + sigs.k8s.io/cluster-api-machineset: worker + sigs.k8s.io/cluster-api-cluster: {{.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: worker + sigs.k8s.io/cluster-api-machine-type: worker + template: + metadata: + labels: + sigs.k8s.io/cluster-api-machineset: worker + sigs.k8s.io/cluster-api-cluster: {{.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: worker + sigs.k8s.io/cluster-api-machine-type: worker + spec: + providerConfig: + value: + apiVersion: libvirtproviderconfig/v1alpha1 + kind: LibvirtMachineProviderConfig + domainMemory: 2048 + domainVcpu: 2 + ignKey: /var/lib/libvirt/images/worker.ign + volume: + poolName: default + baseVolumeID: /var/lib/libvirt/images/coreos_base + networkInterfaceName: {{.Platform.Network.Name}} + networkInterfaceAddress: {{.Platform.Network.IPRange}} + autostart: false + uri: {{.Platform.URI}} + versions: + kubelet: "" + controlPlane: "" +`)) diff --git a/pkg/asset/machines/userdata.go b/pkg/asset/machines/userdata.go new file mode 100644 index 00000000000..d29aa35e9ed --- /dev/null +++ b/pkg/asset/machines/userdata.go @@ -0,0 +1,35 @@ +package machines + +import ( + "bytes" + "encoding/base64" + "text/template" + + "github.com/pkg/errors" +) + +var userDataTmpl = template.Must(template.New("user-data").Parse(` +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} + namespace: openshift-cluster-api +type: Opaque +data: + userData: {{.UserDataContent}} +`)) + +func userData(secretName string, content []byte) ([]byte, error) { + templateData := struct { + Name string + UserDataContent string + }{ + Name: secretName, + UserDataContent: base64.StdEncoding.EncodeToString(content), + } + buf := &bytes.Buffer{} + if err := userDataTmpl.Execute(buf, templateData); err != nil { + return nil, errors.Wrap(err, "failed to execute content.UserDataTmpl") + } + return buf.Bytes(), nil +} diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go new file mode 100644 index 00000000000..8a18e5cd10b --- /dev/null +++ b/pkg/asset/machines/worker.go @@ -0,0 +1,127 @@ +package machines + +import ( + "bytes" + "context" + "fmt" + "text/template" + "time" + + "github.com/pkg/errors" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/ignition/machine" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/machines/aws" + "github.com/openshift/installer/pkg/asset/machines/libvirt" + "github.com/openshift/installer/pkg/rhcos" + "github.com/openshift/installer/pkg/types" +) + +func defaultAWSMachinePoolPlatform() types.AWSMachinePoolPlatform { + return types.AWSMachinePoolPlatform{ + InstanceType: "t2.medium", + } +} + +// Worker generates the machinesets for `worker` machine pool. +type Worker struct { + MachineSetRaw []byte + UserDataSecretRaw []byte +} + +var _ asset.Asset = (*Worker)(nil) + +// Name returns a human friendly name for the Worker Asset. +func (w *Worker) Name() string { + return "Worker Machines" +} + +// Dependencies returns all of the dependencies directly needed by the +// Worker asset +func (w *Worker) Dependencies() []asset.Asset { + return []asset.Asset{ + &installconfig.InstallConfig{}, + &machine.Worker{}, + } +} + +// Generate generates the Worker asset. +func (w *Worker) Generate(dependencies asset.Parents) error { + installconfig := &installconfig.InstallConfig{} + wign := &machine.Worker{} + dependencies.Get(installconfig, wign) + + var err error + w.UserDataSecretRaw, err = userData("worker-user-data", wign.File.Data) + if err != nil { + return errors.Wrap(err, "failed to create user-data secret for worker machines") + } + + ic := installconfig.Config + pool := workerPool(ic.Machines) + numOfWorkers := int64(0) + if pool.Replicas != nil { + numOfWorkers = *pool.Replicas + } + + switch ic.Platform.Name() { + case "aws": + config := aws.Config{ + ClusterName: ic.ObjectMeta.Name, + Replicas: numOfWorkers, + Region: ic.Platform.AWS.Region, + Machine: defaultAWSMachinePoolPlatform(), + } + + tags := map[string]string{ + "tectonicClusterID": ic.ClusterID, + // Info: https://github.com/openshift/cluster-api-provider-aws/issues/73 + // fmt.Sprintf("kubernetes.io/cluster/%s", ic.ObjectMeta.Name): "owned", + } + for k, v := range ic.Platform.AWS.UserTags { + tags[k] = v + } + config.Tags = tags + + config.Machine.Set(ic.Platform.AWS.DefaultMachinePlatform) + config.Machine.Set(pool.Platform.AWS) + + ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second) + defer cancel() + ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, config.Region) + if err != nil { + return errors.Wrap(err, "failed to determine default AMI") + } + config.AMIID = ami + + w.MachineSetRaw = applyTemplateData(aws.WorkerMachineSetTmpl, config) + case "libvirt": + config := libvirt.Config{ + ClusterName: ic.ObjectMeta.Name, + Replicas: numOfWorkers, + Platform: *ic.Platform.Libvirt, + } + w.MachineSetRaw = applyTemplateData(libvirt.WorkerMachineSetTmpl, config) + default: + return fmt.Errorf("invalid Platform") + } + return nil +} + +func workerPool(pools []types.MachinePool) types.MachinePool { + for idx, pool := range pools { + if pool.Name == "worker" { + return pools[idx] + } + } + return types.MachinePool{} +} + +func applyTemplateData(template *template.Template, templateData interface{}) []byte { + buf := &bytes.Buffer{} + if err := template.Execute(buf, templateData); err != nil { + panic(err) + } + return buf.Bytes() +} diff --git a/pkg/asset/manifests/machine-api-operator.go b/pkg/asset/manifests/machine-api-operator.go index 5b1a06db695..fea72ae7478 100644 --- a/pkg/asset/manifests/machine-api-operator.go +++ b/pkg/asset/manifests/machine-api-operator.go @@ -108,7 +108,7 @@ func (mao *machineAPIOperator) Generate(dependencies asset.Parents) error { Region: installConfig.Config.Platform.AWS.Region, AvailabilityZone: "", Image: ami, - Replicas: int(*installConfig.Config.Machines[1].Replicas), + Replicas: 0, // setting replicas to 0 so that MAO doesn't create competing MachineSets } case installConfig.Config.Platform.Libvirt != nil: mao.Config.Libvirt = &libvirtConfig{ @@ -116,14 +116,14 @@ func (mao *machineAPIOperator) Generate(dependencies asset.Parents) error { URI: installConfig.Config.Platform.Libvirt.URI, NetworkName: installConfig.Config.Platform.Libvirt.Network.Name, IPRange: installConfig.Config.Platform.Libvirt.Network.IPRange, - Replicas: int(*installConfig.Config.Machines[1].Replicas), + Replicas: 0, // setting replicas to 0 so that MAO doesn't create competing MachineSets } case installConfig.Config.Platform.OpenStack != nil: mao.Config.OpenStack = &openstackConfig{ ClusterName: installConfig.Config.ObjectMeta.Name, ClusterID: installConfig.Config.ClusterID, Region: installConfig.Config.Platform.OpenStack.Region, - Replicas: int(*installConfig.Config.Machines[1].Replicas), + Replicas: 0, // setting replicas to 0 so that MAO doesn't create competing MachineSets } default: return errors.Errorf("unknown provider for machine-api-operator") diff --git a/pkg/asset/manifests/tectonic.go b/pkg/asset/manifests/tectonic.go index d0b360f1574..61fd65fba3c 100644 --- a/pkg/asset/manifests/tectonic.go +++ b/pkg/asset/manifests/tectonic.go @@ -7,6 +7,7 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/machines" content "github.com/openshift/installer/pkg/asset/manifests/content/tectonic" "github.com/openshift/installer/pkg/asset/tls" ) @@ -30,6 +31,8 @@ func (t *Tectonic) Dependencies() []asset.Asset { &installconfig.InstallConfig{}, &tls.IngressCertKey{}, &tls.KubeCA{}, + &machines.ClusterK8sIO{}, + &machines.Worker{}, } } @@ -38,7 +41,9 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} ingressCertKey := &tls.IngressCertKey{} kubeCA := &tls.KubeCA{} - dependencies.Get(installConfig, ingressCertKey, kubeCA) + clusterk8sio := &machines.ClusterK8sIO{} + worker := &machines.Worker{} + dependencies.Get(installConfig, ingressCertKey, kubeCA, clusterk8sio, worker) templateData := &tectonicTemplateData{ IngressCaCert: base64.StdEncoding.EncodeToString(kubeCA.Cert()), @@ -55,22 +60,25 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { } assetData := map[string][]byte{ - "99_binding-discovery.yaml": []byte(content.BindingDiscovery), - "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), - "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), - "99_kube-core-00-appversion.yaml": []byte(content.AppVersionKubeCore), - "99_kube-core-00-operator.yaml": applyTemplateData(content.KubeCoreOperator, templateData), - "99_role-admin.yaml": []byte(content.RoleAdmin), - "99_role-user.yaml": []byte(content.RoleUser), - "99_tectonic-ingress-00-appversion.yaml": []byte(content.AppVersionTectonicIngress), - "99_tectonic-ingress-01-cluster-config.yaml": applyTemplateData(content.ClusterConfigTectonicIngress, templateData), - "99_tectonic-ingress-02-tls.yaml": applyTemplateData(content.TLSTectonicIngress, templateData), - "99_tectonic-ingress-03-pull.json": applyTemplateData(content.PullTectonicIngress, templateData), - "99_tectonic-ingress-04-svc-account.yaml": []byte(content.SvcAccountTectonicIngress), - "99_tectonic-ingress-05-operator.yaml": applyTemplateData(content.TectonicIngressControllerOperator, templateData), - "99_tectonic-system-00-binding-admin.yaml": []byte(content.BindingAdmin), - "99_tectonic-system-01-ca-cert.yaml": applyTemplateData(content.CaCertTectonicSystem, templateData), - "99_tectonic-system-02-pull.json": applyTemplateData(content.PullTectonicSystem, templateData), + "99_binding-discovery.yaml": []byte(content.BindingDiscovery), + "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), + "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), + "99_kube-core-00-appversion.yaml": []byte(content.AppVersionKubeCore), + "99_kube-core-00-operator.yaml": applyTemplateData(content.KubeCoreOperator, templateData), + "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, + "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, + "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, + "99_role-admin.yaml": []byte(content.RoleAdmin), + "99_role-user.yaml": []byte(content.RoleUser), + "99_tectonic-ingress-00-appversion.yaml": []byte(content.AppVersionTectonicIngress), + "99_tectonic-ingress-01-cluster-config.yaml": applyTemplateData(content.ClusterConfigTectonicIngress, templateData), + "99_tectonic-ingress-02-tls.yaml": applyTemplateData(content.TLSTectonicIngress, templateData), + "99_tectonic-ingress-03-pull.json": applyTemplateData(content.PullTectonicIngress, templateData), + "99_tectonic-ingress-04-svc-account.yaml": []byte(content.SvcAccountTectonicIngress), + "99_tectonic-ingress-05-operator.yaml": applyTemplateData(content.TectonicIngressControllerOperator, templateData), + "99_tectonic-system-00-binding-admin.yaml": []byte(content.BindingAdmin), + "99_tectonic-system-01-ca-cert.yaml": applyTemplateData(content.CaCertTectonicSystem, templateData), + "99_tectonic-system-02-pull.json": applyTemplateData(content.PullTectonicSystem, templateData), } t.FileList = make([]*asset.File, 0, len(assetData))