diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 753ef3e0671..949d70968a9 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -59,6 +59,7 @@ func (a *Bootstrap) Dependencies() []asset.Asset { &kubeconfig.AdminClient{}, &kubeconfig.Kubelet{}, &machines.Master{}, + &machines.Worker{}, &manifests.Manifests{}, &manifests.Openshift{}, &tls.AdminKubeConfigCABundle{}, @@ -371,6 +372,7 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) { &manifests.Manifests{}, &manifests.Openshift{}, &machines.Master{}, + &machines.Worker{}, } { dependencies.Get(asset) a.Config.Storage.Files = append(a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, "root", 0644, asset)...) diff --git a/pkg/asset/machines/machineconfig/manifest.go b/pkg/asset/machines/machineconfig/manifest.go index 9b1a4cb29ff..8970e63cdc4 100644 --- a/pkg/asset/machines/machineconfig/manifest.go +++ b/pkg/asset/machines/machineconfig/manifest.go @@ -46,8 +46,12 @@ func Manifests(configs []*mcfgv1.MachineConfig, role, directory string) ([]*asse } // IsManifest tests whether the specified filename is a MachineConfig manifest. -func IsManifest(role, filename string) bool { - return fmt.Sprintf(machineConfigFileName, role) == filename +func IsManifest(filename string) (bool, error) { + matched, err := filepath.Match(machineConfigFileNamePattern, filename) + if err != nil { + return false, err + } + return matched, nil } // Load loads the MachineConfig manifests. diff --git a/pkg/asset/machines/master.go b/pkg/asset/machines/master.go index fe28413de24..3c38f7238e8 100644 --- a/pkg/asset/machines/master.go +++ b/pkg/asset/machines/master.go @@ -250,21 +250,28 @@ func (m *Master) Machines() ([]machineapi.Machine, error) { return machines, nil } -// IsMasterManifest tests whether a file is a manifest that belongs to the -// Master Machines asset. -func IsMasterManifest(file *asset.File) bool { +// IsMachineManifest tests whether a file is a manifest that belongs to the +// Master Machines or Worker Machines asset. +func IsMachineManifest(file *asset.File) bool { if filepath.Dir(file.Filename) != directory { return false } filename := filepath.Base(file.Filename) - if filename == masterUserDataFileName { + if filename == masterUserDataFileName || filename == workerUserDataFileName { return true } - if machineconfig.IsManifest("master", filename) { + if matched, err := machineconfig.IsManifest(filename); err != nil { + panic(err) + } else if matched { return true } if matched, err := filepath.Match(masterMachineFileNamePattern, filename); err != nil { panic("bad format for master machine file name pattern") + } else if matched { + return true + } + if matched, err := filepath.Match(workerMachineSetFileNamePattern, filename); err != nil { + panic("bad format for worker machine file name pattern") } else { return matched } diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index 885e898e9ae..63b244ef27e 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -2,12 +2,21 @@ package machines import ( "fmt" + "os" "path/filepath" "github.com/ghodss/yaml" + libvirtapi "github.com/openshift/cluster-api-provider-libvirt/pkg/apis" + libvirtprovider "github.com/openshift/cluster-api-provider-libvirt/pkg/apis/libvirtproviderconfig/v1alpha1" + machineapi "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + awsapi "sigs.k8s.io/cluster-api-provider-aws/pkg/apis" + awsprovider "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" + openstackapi "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis" + openstackprovider "sigs.k8s.io/cluster-api-provider-openstack/pkg/apis/openstackproviderconfig/v1alpha1" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/ignition/machine" @@ -23,17 +32,22 @@ import ( nonetypes "github.com/openshift/installer/pkg/types/none" openstacktypes "github.com/openshift/installer/pkg/types/openstack" vspheretypes "github.com/openshift/installer/pkg/types/vsphere" - mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ) const ( - // workerMachineSetFileName is the filename used for the worker MachineSet manifest. - workerMachineSetFileName = "99_openshift-cluster-api_worker-machineset.yaml" + // workerMachineSetFileName is the format string for constructing the worker MachineSet filenames. + workerMachineSetFileName = "99_openshift-cluster-api_worker-machineset-%s.yaml" // workerUserDataFileName is the filename used for the worker user-data secret. workerUserDataFileName = "99_openshift-cluster-api_worker-user-data-secret.yaml" ) +var ( + workerMachineSetFileNamePattern = fmt.Sprintf(workerMachineSetFileName, "*") + + _ asset.WritableAsset = (*Worker)(nil) +) + func defaultAWSMachinePoolPlatform() awstypes.MachinePool { return awstypes.MachinePool{ EC2RootVolume: awstypes.EC2RootVolume{ @@ -57,11 +71,9 @@ func defaultOpenStackMachinePoolPlatform(flavor string) openstacktypes.MachinePo type Worker struct { UserDataFile *asset.File MachineConfigFiles []*asset.File - MachineSetFile *asset.File + MachineSetFiles []*asset.File } -var _ asset.Asset = (*Worker)(nil) - // Name returns a human friendly name for the Worker Asset. func (w *Worker) Name() string { return "Worker Machines" @@ -172,44 +184,88 @@ func (w *Worker) Generate(dependencies asset.Parents) error { return errors.Wrap(err, "failed to create MachineConfig manifests for worker machines") } - if len(machineSets) == 0 { - return nil - } - list := &metav1.List{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "List", - }, - Items: make([]runtime.RawExtension, len(machineSets)), - } - for i, set := range machineSets { - list.Items[i] = runtime.RawExtension{Object: set} - } - data, err = yaml.Marshal(list) - if err != nil { - return errors.Wrap(err, "failed to marshal") - } - w.MachineSetFile = &asset.File{ - Filename: filepath.Join(directory, workerMachineSetFileName), - Data: data, + w.MachineSetFiles = make([]*asset.File, len(machineSets)) + padFormat := fmt.Sprintf("%%0%dd", len(fmt.Sprintf("%d", len(machineSets)))) + for i, machineSet := range machineSets { + data, err := yaml.Marshal(machineSet) + if err != nil { + return errors.Wrapf(err, "marshal worker %d", i) + } + + padded := fmt.Sprintf(padFormat, i) + w.MachineSetFiles[i] = &asset.File{ + Filename: filepath.Join(directory, fmt.Sprintf(workerMachineSetFileName, padded)), + Data: data, + } } + return nil } // Files returns the files generated by the asset. func (w *Worker) Files() []*asset.File { - files := make([]*asset.File, 0, 1+len(w.MachineConfigFiles)+1) + files := make([]*asset.File, 0, 1+len(w.MachineConfigFiles)+len(w.MachineSetFiles)) if w.UserDataFile != nil { files = append(files, w.UserDataFile) } files = append(files, w.MachineConfigFiles...) - if w.MachineSetFile != nil { - files = append(files, w.MachineSetFile) - } + files = append(files, w.MachineSetFiles...) return files } -// Load returns false since this asset is not written to disk by the installer. +// Load reads the asset files from disk. func (w *Worker) Load(f asset.FileFetcher) (found bool, err error) { - return false, nil + file, err := f.FetchByName(filepath.Join(directory, workerUserDataFileName)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + w.UserDataFile = file + + w.MachineConfigFiles, err = machineconfig.Load(f, "worker", directory) + if err != nil { + return true, err + } + + fileList, err := f.FetchByPattern(filepath.Join(directory, workerMachineSetFileNamePattern)) + if err != nil { + return true, err + } + + w.MachineSetFiles = fileList + return true, nil +} + +// MachineSets returns MachineSet manifest structures. +func (w *Worker) MachineSets() ([]machineapi.MachineSet, error) { + scheme := runtime.NewScheme() + awsapi.AddToScheme(scheme) + libvirtapi.AddToScheme(scheme) + openstackapi.AddToScheme(scheme) + decoder := serializer.NewCodecFactory(scheme).UniversalDecoder( + awsprovider.SchemeGroupVersion, + libvirtprovider.SchemeGroupVersion, + openstackprovider.SchemeGroupVersion, + ) + + machineSets := []machineapi.MachineSet{} + for i, file := range w.MachineSetFiles { + machineSet := &machineapi.MachineSet{} + err := yaml.Unmarshal(file.Data, &machineSet) + if err != nil { + return machineSets, errors.Wrapf(err, "unmarshal worker %d", i) + } + + obj, _, err := decoder.Decode(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, nil, nil) + if err != nil { + return machineSets, errors.Wrapf(err, "unmarshal worker %d", i) + } + + machineSet.Spec.Template.Spec.ProviderSpec.Value = &runtime.RawExtension{Object: obj} + machineSets = append(machineSets, *machineSet) + } + + return machineSets, nil } diff --git a/pkg/asset/manifests/openshift.go b/pkg/asset/manifests/openshift.go index 9ed55e85441..4289440faaa 100644 --- a/pkg/asset/manifests/openshift.go +++ b/pkg/asset/manifests/openshift.go @@ -39,7 +39,6 @@ func (o *Openshift) Name() string { func (o *Openshift) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, - &machines.Worker{}, &password.KubeadminPassword{}, &openshift.BindingDiscovery{}, @@ -53,8 +52,7 @@ func (o *Openshift) Dependencies() []asset.Asset { func (o *Openshift) Generate(dependencies asset.Parents) error { installConfig := &installconfig.InstallConfig{} kubeadminPassword := &password.KubeadminPassword{} - worker := &machines.Worker{} - dependencies.Get(installConfig, worker, kubeadminPassword) + dependencies.Get(installConfig, kubeadminPassword) var cloudCreds cloudCredsSecretData platform := installConfig.Config.Platform.Name() switch platform { @@ -133,7 +131,6 @@ func (o *Openshift) Generate(dependencies asset.Parents) error { Data: data, }) } - o.FileList = append(o.FileList, worker.Files()...) asset.SortFiles(o.FileList) @@ -153,7 +150,7 @@ func (o *Openshift) Load(f asset.FileFetcher) (bool, error) { } for _, file := range fileList { - if machines.IsMasterManifest(file) { + if machines.IsMachineManifest(file) { continue } diff --git a/pkg/asset/store/assetcreate_test.go b/pkg/asset/store/assetcreate_test.go index e72d743c648..0c6c878619b 100644 --- a/pkg/asset/store/assetcreate_test.go +++ b/pkg/asset/store/assetcreate_test.go @@ -88,6 +88,7 @@ func TestCreatedAssetsAreNotDirty(t *testing.T) { emptyAssets := map[string]bool{ "Master Machines": true, // no files for the 'none' platform + "Worker Machines": true, // no files for the 'none' platform "Metadata": true, // read-only } for _, a := range tc.targets { diff --git a/pkg/asset/targets/targets.go b/pkg/asset/targets/targets.go index f3cf542274d..b89982310f9 100644 --- a/pkg/asset/targets/targets.go +++ b/pkg/asset/targets/targets.go @@ -23,6 +23,7 @@ var ( // Manifests are the manifests targeted assets. Manifests = []asset.WritableAsset{ &machines.Master{}, + &machines.Worker{}, &manifests.Manifests{}, &manifests.Openshift{}, }