diff --git a/pkg/asset/machines/aws/master.go b/pkg/asset/machines/aws/master.go new file mode 100644 index 00000000000..ee40ac74f4b --- /dev/null +++ b/pkg/asset/machines/aws/master.go @@ -0,0 +1,73 @@ +// Package aws generates Machine objects for aws. +package aws + +import ( + "text/template" +) + +// MasterConfig is used to generate master machines +type MasterConfig struct { + MachineConfig + Instances []MasterInstance +} + +// MasterInstance contains information specific to each +// master machine instance to create. +type MasterInstance struct { + AvailabilityZone string +} + +// MasterMachineTmpl is a template for a list of master machines. +var MasterMachineTmpl = template.Must(template.New("aws-master-machine").Parse(` +{{- $c := . -}} +kind: List +apiVersion: v1 +metadata: + resourceVersion: "" + selfLink: "" +items: +{{- range $index,$instance := $c.Instances}} +- apiVersion: cluster.k8s.io/v1alpha1 + kind: Machine + metadata: + name: {{$c.ClusterName}}-master-{{$index}} + namespace: openshift-cluster-api + labels: + sigs.k8s.io/cluster-api-cluster: {{$c.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: master + sigs.k8s.io/cluster-api-machine-type: master + spec: + providerConfig: + value: + apiVersion: aws.cluster.k8s.io/v1alpha1 + kind: AWSMachineProviderConfig + ami: + id: {{$c.AMIID}} + instanceType: {{$c.Machine.InstanceType}} + placement: + region: {{$c.Region}} + availabilityZone: {{$instance.AvailabilityZone}} + subnet: + filters: + - name: "tag:Name" + values: + - "{{$c.ClusterName}}-master-{{$instance.AvailabilityZone}}" + iamInstanceProfile: + id: "{{$c.ClusterName}}-master-profile" + tags: +{{- range $key,$value := $c.Tags}} + - name: "{{$key}}" + value: "{{$value}}" +{{- end}} + securityGroups: + - filters: + - name: "tag:Name" + values: + - "{{$c.ClusterName}}_master_sg" + userDataSecret: + name: "master-user-data-{{$index}}" + versions: + kubelet: "" + controlPlane: "" +{{- end}} +`)) diff --git a/pkg/asset/machines/aws/aws.go b/pkg/asset/machines/aws/worker.go similarity index 89% rename from pkg/asset/machines/aws/aws.go rename to pkg/asset/machines/aws/worker.go index 177c97657a4..5d398a954bf 100644 --- a/pkg/asset/machines/aws/aws.go +++ b/pkg/asset/machines/aws/worker.go @@ -7,10 +7,16 @@ import ( "github.com/openshift/installer/pkg/types" ) -// Config is used to generate the machine. -type Config struct { +// WorkerConfig is used to generate the worker machineset. +type WorkerConfig struct { + Replicas int64 + MachineConfig +} + +// MachineConfig contains fields common to worker and master +// machine configurations +type MachineConfig struct { ClusterName string - Replicas int64 AMIID string Tags map[string]string Region string diff --git a/pkg/asset/machines/aws/zones.go b/pkg/asset/machines/aws/zones.go new file mode 100644 index 00000000000..e0100993641 --- /dev/null +++ b/pkg/asset/machines/aws/zones.go @@ -0,0 +1,48 @@ +package aws + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" +) + +// AvailabilityZones retrieves a list of availability zones for the given region. +func AvailabilityZones(region string) ([]string, error) { + ec2Client := ec2Client(region) + zones, err := fetchAvailabilityZones(ec2Client, region) + if err != nil { + return nil, fmt.Errorf("cannot fetch availability zones: %v", err) + } + return zones, nil +} + +func ec2Client(region string) *ec2.EC2 { + ssn := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String(region), + }, + })) + return ec2.New(ssn) +} + +func fetchAvailabilityZones(client *ec2.EC2, region string) ([]string, error) { + zoneFilter := &ec2.Filter{ + Name: aws.String("region-name"), + Values: []*string{aws.String(region)}, + } + req := &ec2.DescribeAvailabilityZonesInput{ + Filters: []*ec2.Filter{zoneFilter}, + } + resp, err := client.DescribeAvailabilityZones(req) + if err != nil { + return nil, err + } + zones := []string{} + for _, zone := range resp.AvailabilityZones { + zones = append(zones, *zone.ZoneName) + } + return zones, nil +} diff --git a/pkg/asset/machines/libvirt/master.go b/pkg/asset/machines/libvirt/master.go new file mode 100644 index 00000000000..2be2555d587 --- /dev/null +++ b/pkg/asset/machines/libvirt/master.go @@ -0,0 +1,55 @@ +// Package libvirt generates Machine objects for libvirt. +package libvirt + +import ( + "text/template" + + "github.com/openshift/installer/pkg/types" +) + +// MasterConfig is used to generate the master machine list. +type MasterConfig struct { + ClusterName string + Instances []string + Platform types.LibvirtPlatform +} + +// MasterMachinesTmpl is the template for master machines +var MasterMachinesTmpl = template.Must(template.New("master-machines").Parse(` +{{- $c := . -}} +kind: List +apiVersion: v1 +metadata: + resourceVersion: "" + selfLink: "" +items: +{{- range $index,$instance := .Instances}} +- apiVersion: cluster.k8s.io/v1alpha1 + kind: Machine + metadata: + name: {{$c.ClusterName}}-master-{{$index}} + namespace: openshift-cluster-api + labels: + sigs.k8s.io/cluster-api-cluster: {{$c.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: master + sigs.k8s.io/cluster-api-machine-type: master + spec: + providerConfig: + value: + apiVersion: libvirtproviderconfig/v1alpha1 + kind: LibvirtMachineProviderConfig + domainMemory: 2048 + domainVcpu: 2 + ignKey: /var/lib/libvirt/images/master-{{$index}}.ign + volume: + poolName: default + baseVolumeID: /var/lib/libvirt/images/coreos_base + networkInterfaceName: {{$c.Platform.Network.Name}} + networkInterfaceAddress: {{$c.Platform.Network.IPRange}} + autostart: false + uri: {{$c.Platform.URI}} + versions: + kubelet: "" + controlPlane: "" +{{- end }} +`)) diff --git a/pkg/asset/machines/libvirt/libvirt.go b/pkg/asset/machines/libvirt/worker.go similarity index 100% rename from pkg/asset/machines/libvirt/libvirt.go rename to pkg/asset/machines/libvirt/worker.go diff --git a/pkg/asset/machines/master.go b/pkg/asset/machines/master.go new file mode 100644 index 00000000000..54e18a0806e --- /dev/null +++ b/pkg/asset/machines/master.go @@ -0,0 +1,148 @@ +package machines + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/ignition/machine" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/asset/machines/aws" + "github.com/openshift/installer/pkg/asset/machines/libvirt" + "github.com/openshift/installer/pkg/asset/machines/openstack" + "github.com/openshift/installer/pkg/rhcos" + "github.com/openshift/installer/pkg/types" +) + +// Master generates the machines for the `master` machine pool. +type Master struct { + MachinesRaw []byte + UserDataSecretsRaw []byte +} + +var _ asset.Asset = (*Master)(nil) + +// Name returns a human friendly name for the Master Asset. +func (m *Master) Name() string { + return "Master Machines" +} + +// Dependencies returns all of the dependencies directly needed by the +// Master asset +func (m *Master) Dependencies() []asset.Asset { + return []asset.Asset{ + &installconfig.InstallConfig{}, + &machine.Master{}, + } +} + +// Generate generates the Master asset. +func (m *Master) Generate(dependencies asset.Parents) error { + installconfig := &installconfig.InstallConfig{} + mign := &machine.Master{} + dependencies.Get(installconfig, mign) + + userDataContent := map[string][]byte{} + for i, file := range mign.FileList { + userDataContent[fmt.Sprintf("master-user-data-%d", i)] = file.Data + } + + var err error + m.UserDataSecretsRaw, err = userDataList(userDataContent) + if err != nil { + return errors.Wrap(err, "failed to create user-data secrets for master machines") + } + + ic := installconfig.Config + pool := masterPool(ic.Machines) + numOfMasters := int64(0) + if pool.Replicas != nil { + numOfMasters = *pool.Replicas + } + + switch ic.Platform.Name() { + case "aws": + config := aws.MasterConfig{} + config.ClusterName = ic.ObjectMeta.Name + config.Region = ic.Platform.AWS.Region + config.Machine = defaultAWSMachinePoolPlatform() + + tags := map[string]string{ + "tectonicClusterID": ic.ClusterID, + } + for k, v := range ic.Platform.AWS.UserTags { + tags[k] = v + } + config.Tags = tags + + config.Machine.Set(ic.Platform.AWS.DefaultMachinePlatform) + config.Machine.Set(pool.Platform.AWS) + + ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second) + defer cancel() + ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, config.Region) + if err != nil { + return errors.Wrap(err, "failed to determine default AMI") + } + config.AMIID = ami + azs, err := aws.AvailabilityZones(config.Region) + if err != nil { + return errors.Wrap(err, "failed to fetch availability zones") + } + + for i := 0; i < int(numOfMasters); i++ { + azIndex := i % len(azs) + config.Instances = append(config.Instances, aws.MasterInstance{AvailabilityZone: azs[azIndex]}) + } + + m.MachinesRaw = applyTemplateData(aws.MasterMachineTmpl, config) + case "libvirt": + instances := []string{} + for i := 0; i < int(numOfMasters); i++ { + instances = append(instances, fmt.Sprintf("master-%d", i)) + } + config := libvirt.MasterConfig{ + ClusterName: ic.ObjectMeta.Name, + Instances: instances, + Platform: *ic.Platform.Libvirt, + } + m.MachinesRaw = applyTemplateData(libvirt.MasterMachinesTmpl, config) + case "openstack": + instances := []string{} + for i := 0; i < int(numOfMasters); i++ { + instances = append(instances, fmt.Sprintf("master-%d", i)) + } + config := openstack.MasterConfig{ + ClusterName: ic.ObjectMeta.Name, + Instances: instances, + Image: ic.Platform.OpenStack.BaseImage, + Region: ic.Platform.OpenStack.Region, + Machine: defaultOpenStackMachinePoolPlatform(), + } + + tags := map[string]string{ + "tectonicClusterID": ic.ClusterID, + } + config.Tags = tags + + config.Machine.Set(ic.Platform.OpenStack.DefaultMachinePlatform) + config.Machine.Set(pool.Platform.OpenStack) + + m.MachinesRaw = applyTemplateData(openstack.MasterMachinesTmpl, config) + default: + return fmt.Errorf("invalid Platform") + } + return nil +} + +func masterPool(pools []types.MachinePool) types.MachinePool { + for idx, pool := range pools { + if pool.Name == "master" { + return pools[idx] + } + } + return types.MachinePool{} +} diff --git a/pkg/asset/machines/openstack/master.go b/pkg/asset/machines/openstack/master.go new file mode 100644 index 00000000000..b365d57c2b6 --- /dev/null +++ b/pkg/asset/machines/openstack/master.go @@ -0,0 +1,70 @@ +// Package openstack generates Machine objects for openstack. +package openstack + +import ( + "text/template" + + "github.com/openshift/installer/pkg/types" +) + +// MasterConfig is used to generate the machine. +type MasterConfig struct { + ClusterName string + Instances []string + Image string + Tags map[string]string + Region string + Machine types.OpenStackMachinePoolPlatform +} + +// MasterMachinesTmpl is the template for master machines. +var MasterMachinesTmpl = template.Must(template.New("openstack-master-machines").Parse(` +{{- $c := . -}} +kind: List +apiVersion: v1 +metadata: + resourceVersion: "" + selfLink: "" +items: +{{- range $index,$instance := .Instances}} +- apiVersion: cluster.k8s.io/v1alpha1 + kind: Machine + metadata: + name: {{$c.ClusterName}}-master-{{$index}} + namespace: openshift-cluster-api + labels: + sigs.k8s.io/cluster-api-cluster: {{$c.ClusterName}} + sigs.k8s.io/cluster-api-machine-role: master + sigs.k8s.io/cluster-api-machine-type: master + spec: + providerConfig: + value: + apiVersion: openstack.cluster.k8s.io/v1alpha1 + kind: OpenStackMachineProviderConfig + image: + id: {{$c.Image}} + flavor: {{$c.Machine.FlavorName}} + placement: + region: {{$c.Region}} + subnet: + filters: + - name: "tag:Name" + values: + - "{{$c.ClusterName}}-master-*" + tags: +{{- range $key,$value := $c.Tags}} + - name: "{{$key}}" + value: "{{$value}}" +{{- end}} + securityGroups: + - filters: + - name: "tag:Name" + values: + - "{{$c.ClusterName}}_master_sg" + userDataSecret: + name: master-user-data-{{$index}} + versions: + kubelet: "" + controlPlane: "" +{{- end -}} +`)) diff --git a/pkg/asset/machines/openstack/openstack.go b/pkg/asset/machines/openstack/worker.go similarity index 100% rename from pkg/asset/machines/openstack/openstack.go rename to pkg/asset/machines/openstack/worker.go diff --git a/pkg/asset/machines/userdata.go b/pkg/asset/machines/userdata.go index d29aa35e9ed..c1687eda340 100644 --- a/pkg/asset/machines/userdata.go +++ b/pkg/asset/machines/userdata.go @@ -8,28 +8,33 @@ import ( "github.com/pkg/errors" ) -var userDataTmpl = template.Must(template.New("user-data").Parse(` +var userDataListTmpl = template.Must(template.New("user-data-list").Parse(` +kind: List apiVersion: v1 -kind: Secret metadata: - name: {{.Name}} - namespace: openshift-cluster-api -type: Opaque -data: - userData: {{.UserDataContent}} + resourceVersion: "" + selfLink: "" +items: +{{- range $name, $content := . }} +- apiVersion: v1 + kind: Secret + metadata: + name: {{$name}} + namespace: openshift-cluster-api + type: Opaque + data: + userData: {{$content}} +{{- end}} `)) -func userData(secretName string, content []byte) ([]byte, error) { - templateData := struct { - Name string - UserDataContent string - }{ - Name: secretName, - UserDataContent: base64.StdEncoding.EncodeToString(content), +func userDataList(data map[string][]byte) ([]byte, error) { + encodedData := map[string]string{} + for name, content := range data { + encodedData[name] = base64.StdEncoding.EncodeToString(content) } buf := &bytes.Buffer{} - if err := userDataTmpl.Execute(buf, templateData); err != nil { - return nil, errors.Wrap(err, "failed to execute content.UserDataTmpl") + if err := userDataListTmpl.Execute(buf, encodedData); err != nil { + return nil, errors.Wrap(err, "failed to execute content.UserDataListTmpl") } return buf.Bytes(), nil } diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index 0a51c6605e6..8024d69d777 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -60,7 +60,8 @@ func (w *Worker) Generate(dependencies asset.Parents) error { dependencies.Get(installconfig, wign) var err error - w.UserDataSecretRaw, err = userData("worker-user-data", wign.File.Data) + userDataMap := map[string][]byte{"worker-user-data": wign.File.Data} + w.UserDataSecretRaw, err = userDataList(userDataMap) if err != nil { return errors.Wrap(err, "failed to create user-data secret for worker machines") } @@ -74,12 +75,11 @@ func (w *Worker) Generate(dependencies asset.Parents) error { switch ic.Platform.Name() { case "aws": - config := aws.Config{ - ClusterName: ic.ObjectMeta.Name, - Replicas: numOfWorkers, - Region: ic.Platform.AWS.Region, - Machine: defaultAWSMachinePoolPlatform(), - } + config := aws.WorkerConfig{} + config.ClusterName = ic.ObjectMeta.Name + config.Replicas = numOfWorkers + config.Region = ic.Platform.AWS.Region + config.Machine = defaultAWSMachinePoolPlatform() tags := map[string]string{ "tectonicClusterID": ic.ClusterID, diff --git a/pkg/asset/manifests/tectonic.go b/pkg/asset/manifests/tectonic.go index 1a4791911be..ae56d76d39d 100644 --- a/pkg/asset/manifests/tectonic.go +++ b/pkg/asset/manifests/tectonic.go @@ -47,6 +47,7 @@ func (t *Tectonic) Dependencies() []asset.Asset { &tls.KubeCA{}, &machines.ClusterK8sIO{}, &machines.Worker{}, + &machines.Master{}, &kubeAddonOperator{}, } } @@ -58,8 +59,9 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { kubeCA := &tls.KubeCA{} clusterk8sio := &machines.ClusterK8sIO{} worker := &machines.Worker{} + master := &machines.Master{} addon := &kubeAddonOperator{} - dependencies.Get(installConfig, ingressCertKey, kubeCA, clusterk8sio, worker, addon) + dependencies.Get(installConfig, ingressCertKey, kubeCA, clusterk8sio, worker, master, addon) templateData := &tectonicTemplateData{ IngressCaCert: base64.StdEncoding.EncodeToString(kubeCA.Cert()), @@ -76,23 +78,25 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { } assetData := map[string][]byte{ - "99_binding-discovery.yaml": []byte(content.BindingDiscovery), - "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), - "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), - "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, - "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, - "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, - "99_role-admin.yaml": []byte(content.RoleAdmin), - "99_role-user.yaml": []byte(content.RoleUser), - "99_tectonic-ingress-00-appversion.yaml": []byte(content.AppVersionTectonicIngress), - "99_tectonic-ingress-01-cluster-config.yaml": applyTemplateData(content.ClusterConfigTectonicIngress, templateData), - "99_tectonic-ingress-02-tls.yaml": applyTemplateData(content.TLSTectonicIngress, templateData), - "99_tectonic-ingress-03-pull.json": applyTemplateData(content.PullTectonicIngress, templateData), - "99_tectonic-ingress-04-svc-account.yaml": []byte(content.SvcAccountTectonicIngress), - "99_tectonic-ingress-05-operator.yaml": applyTemplateData(content.TectonicIngressControllerOperator, templateData), - "99_tectonic-system-00-binding-admin.yaml": []byte(content.BindingAdmin), - "99_tectonic-system-01-ca-cert.yaml": applyTemplateData(content.CaCertTectonicSystem, templateData), - "99_tectonic-system-02-pull.json": applyTemplateData(content.PullTectonicSystem, templateData), + "99_binding-discovery.yaml": []byte(content.BindingDiscovery), + "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), + "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), + "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, + "99_openshift-cluster-api_master-machines.yaml": master.MachinesRaw, + "99_openshift-cluster-api_master-user-data-secrets.yaml": master.UserDataSecretsRaw, + "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, + "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, + "99_role-admin.yaml": []byte(content.RoleAdmin), + "99_role-user.yaml": []byte(content.RoleUser), + "99_tectonic-ingress-00-appversion.yaml": []byte(content.AppVersionTectonicIngress), + "99_tectonic-ingress-01-cluster-config.yaml": applyTemplateData(content.ClusterConfigTectonicIngress, templateData), + "99_tectonic-ingress-02-tls.yaml": applyTemplateData(content.TLSTectonicIngress, templateData), + "99_tectonic-ingress-03-pull.json": applyTemplateData(content.PullTectonicIngress, templateData), + "99_tectonic-ingress-04-svc-account.yaml": []byte(content.SvcAccountTectonicIngress), + "99_tectonic-ingress-05-operator.yaml": applyTemplateData(content.TectonicIngressControllerOperator, templateData), + "99_tectonic-system-00-binding-admin.yaml": []byte(content.BindingAdmin), + "99_tectonic-system-01-ca-cert.yaml": applyTemplateData(content.CaCertTectonicSystem, templateData), + "99_tectonic-system-02-pull.json": applyTemplateData(content.PullTectonicSystem, templateData), } // addon goes to openshift system