From b2ad52a4a06cb85c58926c14be8c925fc747c715 Mon Sep 17 00:00:00 2001 From: Kenny Leung Date: Wed, 16 May 2018 11:46:32 -0700 Subject: [PATCH] Populate machineset status This PR was split off from PR #165 to make it a smaller PR. There are edge cases that will be implemented in a separate PR. --- pkg/controller/machineset/controller.go | 53 +++-- pkg/controller/machineset/reconcile_test.go | 23 ++- pkg/controller/machineset/status.go | 165 ++++++++++++++++ pkg/controller/machineset/status_test.go | 204 ++++++++++++++++++++ 4 files changed, 424 insertions(+), 21 deletions(-) create mode 100644 pkg/controller/machineset/status.go create mode 100644 pkg/controller/machineset/status_test.go diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index 545647c97bc2..318f66c162c9 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -18,16 +18,24 @@ package machineset import ( "fmt" + "github.com/golang/glog" "github.com/kubernetes-incubator/apiserver-builder/pkg/builders" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" - machineclientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset" + clusterapiclientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset" listers "sigs.k8s.io/cluster-api/pkg/client/listers_generated/cluster/v1alpha1" "sigs.k8s.io/cluster-api/pkg/controller/sharedinformers" ) +const ( + // The number of times we retry updating a ReplicaSet's status. + statusUpdateRetries = 1 +) + // controllerKind contains the schema.GroupVersionKind for this controller type. var controllerKind = v1alpha1.SchemeGroupVersion.WithKind("MachineSet") @@ -35,8 +43,11 @@ var controllerKind = v1alpha1.SchemeGroupVersion.WithKind("MachineSet") type MachineSetControllerImpl struct { builders.DefaultControllerFns - // machineClient a client that knows how to consume Machine resources - machineClient machineclientset.Interface + // kubernetesClient a client that knows how to consume Node resources + kubernetesClient kubernetes.Interface + + // clusterAPIClient a client that knows how to consume Cluster API resources + clusterAPIClient clusterapiclientset.Interface // machineSetsLister indexes properties about MachineSet machineSetsLister listers.MachineSetLister @@ -48,13 +59,15 @@ type MachineSetControllerImpl struct { // Init initializes the controller and is called by the generated code // Register watches for additional resource types here. func (c *MachineSetControllerImpl) Init(arguments sharedinformers.ControllerInitArguments) { + c.kubernetesClient = arguments.GetSharedInformers().KubernetesClientSet + c.machineSetsLister = arguments.GetSharedInformers().Factory.Cluster().V1alpha1().MachineSets().Lister() c.machineLister = arguments.GetSharedInformers().Factory.Cluster().V1alpha1().Machines().Lister() var err error - c.machineClient, err = machineclientset.NewForConfig(arguments.GetRestConfig()) + c.clusterAPIClient, err = clusterapiclientset.NewForConfig(arguments.GetRestConfig()) if err != nil { - glog.Fatalf("error building clientset for machineClient: %v", err) + glog.Fatalf("error building clientset for clusterAPIClient: %v", err) } } @@ -68,7 +81,18 @@ func (c *MachineSetControllerImpl) Reconcile(machineSet *v1alpha1.MachineSet) er return err } - return c.syncReplicas(machineSet, filteredMachines) + syncErr := c.syncReplicas(machineSet, filteredMachines) + + ms := machineSet.DeepCopy() + newStatus := c.calculateStatus(ms, filteredMachines) + + // Always updates status as machines come up or die. + _, err = updateMachineSetStatus(c.clusterAPIClient.ClusterV1alpha1().MachineSets(machineSet.Namespace), machineSet, newStatus) + if err != nil { + return fmt.Errorf("failed to update machine set status, %v", err) + } + + return syncErr } func (c *MachineSetControllerImpl) Get(namespace, name string) (*v1alpha1.MachineSet, error) { @@ -93,11 +117,8 @@ func (c *MachineSetControllerImpl) syncReplicas(machineSet *v1alpha1.MachineSet, diff *= -1 for i := 0; i < diff; i++ { glog.V(2).Infof("creating a machine ( spec.replicas(%d) > currentMachineCount(%d) )", desiredReplicas, currentMachineCount) - machine, err := c.createMachine(machineSet) - if err != nil { - return err - } - _, err = c.machineClient.ClusterV1alpha1().Machines(machineSet.Namespace).Create(machine) + machine := c.createMachine(machineSet) + _, err := c.clusterAPIClient.ClusterV1alpha1().Machines(machineSet.Namespace).Create(machine) if err != nil { glog.Errorf("unable to create a machine = %s, due to %v", machine.Name, err) result = err @@ -109,7 +130,7 @@ func (c *MachineSetControllerImpl) syncReplicas(machineSet *v1alpha1.MachineSet, // TODO: Define machines deletion policies. // see: https://github.com/kubernetes/kube-deploy/issues/625 machineToDelete := machines[i] - err := c.machineClient.ClusterV1alpha1().Machines(machineSet.Namespace).Delete(machineToDelete.Name, &metav1.DeleteOptions{}) + err := c.clusterAPIClient.ClusterV1alpha1().Machines(machineSet.Namespace).Delete(machineToDelete.Name, &metav1.DeleteOptions{}) if err != nil { glog.Errorf("unable to delete a machine = %s, due to %v", machineToDelete.Name, err) result = err @@ -122,7 +143,7 @@ func (c *MachineSetControllerImpl) syncReplicas(machineSet *v1alpha1.MachineSet, // createMachine creates a machine resource. // the name of the newly created resource is going to be created by the API server, we set the generateName field -func (c *MachineSetControllerImpl) createMachine(machineSet *v1alpha1.MachineSet) (*v1alpha1.Machine, error) { +func (c *MachineSetControllerImpl) createMachine(machineSet *v1alpha1.MachineSet) *v1alpha1.Machine { gv := v1alpha1.SchemeGroupVersion machine := &v1alpha1.Machine{ TypeMeta: metav1.TypeMeta{ @@ -133,9 +154,9 @@ func (c *MachineSetControllerImpl) createMachine(machineSet *v1alpha1.MachineSet Spec: machineSet.Spec.Template.Spec, } machine.ObjectMeta.GenerateName = fmt.Sprintf("%s-", machineSet.Name) - machine.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(machineSet, controllerKind),} + machine.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(machineSet, controllerKind)} - return machine, nil + return machine } // getMachines returns a list of machines that match on machineSet.Spec.Selector @@ -182,7 +203,7 @@ func (c *MachineSetControllerImpl) adoptOrphan(machineSet *v1alpha1.MachineSet, newRef := *metav1.NewControllerRef(machineSet, controllerKind) ownerRefs = append(ownerRefs, newRef) machine.ObjectMeta.SetOwnerReferences(ownerRefs) - if _, err := c.machineClient.ClusterV1alpha1().Machines(machineSet.Namespace).Update(machine); err != nil { + if _, err := c.clusterAPIClient.ClusterV1alpha1().Machines(machineSet.Namespace).Update(machine); err != nil { glog.Warningf("Failed to update machine owner reference. %v", err) } } diff --git a/pkg/controller/machineset/reconcile_test.go b/pkg/controller/machineset/reconcile_test.go index 809ee5db0a08..fd496ff40b6d 100644 --- a/pkg/controller/machineset/reconcile_test.go +++ b/pkg/controller/machineset/reconcile_test.go @@ -150,12 +150,13 @@ func TestMachineSetControllerReconcileHandler(t *testing.T) { if err != nil { t.Fatal(err) } + rObjects = append(rObjects, amachineset) } fakeClient := fake.NewSimpleClientset(rObjects...) machineLister := v1alpha1listers.NewMachineLister(machinesIndexer) machineSetLister := v1alpha1listers.NewMachineSetLister(machineSetIndexer) target := &MachineSetControllerImpl{} - target.machineClient = fakeClient + target.clusterAPIClient = fakeClient target.machineSetsLister = machineSetLister target.machineLister = machineLister @@ -171,6 +172,7 @@ func TestMachineSetControllerReconcileHandler(t *testing.T) { // validate actions := fakeClient.Actions() + actions = getFilteredActions(actions, "machines") if len(actions) != len(test.expectedActions) { t.Fatalf("unexpected actions: %v, expected %d actions got %d", actions, len(test.expectedActions), len(actions)) } @@ -222,15 +224,16 @@ func createMachineSet(replicas int, machineSetName string, machineName string, n Namespace: namespace, }, Spec: v1alpha1.MachineSetSpec{ - Replicas: &replicasInt32, - Selector:metav1.LabelSelector{ - MatchLabels: map[string]string{labelKey:"strongMachine"}, + Replicas: &replicasInt32, + MinReadySeconds: 600, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{labelKey: "strongMachine"}, }, Template: v1alpha1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: machineName, Namespace: namespace, - Labels: map[string]string{labelKey:"strongMachine"}, + Labels: map[string]string{labelKey: "strongMachine"}, }, Spec: v1alpha1.MachineSpec{ ProviderConfig: v1alpha1.ProviderConfig{ @@ -298,3 +301,13 @@ func setNonControllerRef(m *v1alpha1.Machine) *v1alpha1.Machine { m.ObjectMeta.OwnerReferences[0].Controller = &controller return m } + +func getFilteredActions(actions []clienttesting.Action, resource string) []clienttesting.Action { + var filteredActions []clienttesting.Action + for _, action := range actions { + if action.GetResource().Resource == resource { + filteredActions = append(filteredActions, action) + } + } + return filteredActions +} diff --git a/pkg/controller/machineset/status.go b/pkg/controller/machineset/status.go new file mode 100644 index 000000000000..b27093e4db3e --- /dev/null +++ b/pkg/controller/machineset/status.go @@ -0,0 +1,165 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machineset + +import ( + "fmt" + "time" + + "github.com/golang/glog" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + machinesetclientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" +) + +func (c *MachineSetControllerImpl) calculateStatus(ms *v1alpha1.MachineSet, filteredMachines []*v1alpha1.Machine) v1alpha1.MachineSetStatus { + newStatus := ms.Status + // Count the number of machines that have labels matching the labels of the machine + // template of the replica set, the matching machines may have more + // labels than are in the template. Because the label of machineTemplateSpec is + // a superset of the selector of the replica set, so the possible + // matching machines must be part of the filteredMachines. + fullyLabeledReplicasCount := 0 + readyReplicasCount := 0 + availableReplicasCount := 0 + templateLabel := labels.Set(ms.Spec.Template.Labels).AsSelectorPreValidated() + for _, machine := range filteredMachines { + if templateLabel.Matches(labels.Set(machine.Labels)) { + fullyLabeledReplicasCount++ + } + node, err := c.getMachineNode(machine) + if err != nil { + glog.Warningf("Unable to get node for machine %v, %v", machine.Name, err) + continue + } + if isNodeReady(node) { + readyReplicasCount++ + if isNodeAvailable(node, ms.Spec.MinReadySeconds, metav1.Now()) { + availableReplicasCount++ + } + } + } + + newStatus.Replicas = int32(len(filteredMachines)) + newStatus.FullyLabeledReplicas = int32(fullyLabeledReplicasCount) + newStatus.ReadyReplicas = int32(readyReplicasCount) + newStatus.AvailableReplicas = int32(availableReplicasCount) + return newStatus +} + +// updateMachineSetStatus attempts to update the Status.Replicas of the given MachineSet, with a single GET/PUT retry. +func updateMachineSetStatus(c machinesetclientset.MachineSetInterface, ms *v1alpha1.MachineSet, newStatus v1alpha1.MachineSetStatus) (*v1alpha1.MachineSet, error) { + // This is the steady state. It happens when the MachineSet doesn't have any expectations, since + // we do a periodic relist every 30s. If the generations differ but the replicas are + // the same, a caller might've resized to the same replica count. + if ms.Status.Replicas == newStatus.Replicas && + ms.Status.FullyLabeledReplicas == newStatus.FullyLabeledReplicas && + ms.Status.ReadyReplicas == newStatus.ReadyReplicas && + ms.Status.AvailableReplicas == newStatus.AvailableReplicas && + ms.Generation == ms.Status.ObservedGeneration { + return ms, nil + } + + // Save the generation number we acted on, otherwise we might wrongfully indicate + // that we've seen a spec update when we retry. + // TODO: This can clobber an update if we allow multiple agents to write to the + // same status. + newStatus.ObservedGeneration = ms.Generation + + var getErr, updateErr error + var updatedMS *v1alpha1.MachineSet + for i := 0; ; i++ { + glog.V(4).Infof(fmt.Sprintf("Updating status for %v: %s/%s, ", ms.Kind, ms.Namespace, ms.Name) + + fmt.Sprintf("replicas %d->%d (need %d), ", ms.Status.Replicas, newStatus.Replicas, *(ms.Spec.Replicas)) + + fmt.Sprintf("fullyLabeledReplicas %d->%d, ", ms.Status.FullyLabeledReplicas, newStatus.FullyLabeledReplicas) + + fmt.Sprintf("readyReplicas %d->%d, ", ms.Status.ReadyReplicas, newStatus.ReadyReplicas) + + fmt.Sprintf("availableReplicas %d->%d, ", ms.Status.AvailableReplicas, newStatus.AvailableReplicas) + + fmt.Sprintf("sequence No: %v->%v", ms.Status.ObservedGeneration, newStatus.ObservedGeneration)) + + ms.Status = newStatus + updatedMS, updateErr = c.UpdateStatus(ms) + if updateErr == nil { + return updatedMS, nil + } + // Stop retrying if we exceed statusUpdateRetries - the machineSet will be requeued with a rate limit. + if i >= statusUpdateRetries { + break + } + // Update the MachineSet with the latest resource version for the next poll + if ms, getErr = c.Get(ms.Name, metav1.GetOptions{}); getErr != nil { + // If the GET fails we can't trust status.Replicas anymore. This error + // is bound to be more interesting than the update failure. + return nil, getErr + } + } + + return nil, updateErr +} + +func isNodeAvailable(node *corev1.Node, minReadySeconds int32, now metav1.Time) bool { + if !isNodeReady(node) { + return false + } + + if minReadySeconds == 0 { + return true + } + + minReadySecondsDuration := time.Duration(minReadySeconds) * time.Second + _, readyCondition := getNodeCondition(&node.Status, corev1.NodeReady) + + if !readyCondition.LastTransitionTime.IsZero() && + readyCondition.LastTransitionTime.Add(minReadySecondsDuration).Before(now.Time) { + return true + } + + return false +} + +// getNodeCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func getNodeCondition(status *corev1.NodeStatus, conditionType corev1.NodeConditionType) (int, *corev1.NodeCondition) { + if status == nil { + return -1, nil + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType { + return i, &status.Conditions[i] + } + } + return -1, nil +} + +// isNodeReady returns true if a node is ready; false otherwise. +func isNodeReady(node *corev1.Node) bool { + for _, c := range node.Status.Conditions { + if c.Type == corev1.NodeReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +func getMachinesToDelete(filteredMachines []*v1alpha1.Machine, diff int) []*v1alpha1.Machine { + // TODO: Define machines deletion policies. + // see: https://github.com/kubernetes/kube-deploy/issues/625 + return filteredMachines[:diff] +} diff --git a/pkg/controller/machineset/status_test.go b/pkg/controller/machineset/status_test.go new file mode 100644 index 000000000000..130027f253af --- /dev/null +++ b/pkg/controller/machineset/status_test.go @@ -0,0 +1,204 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machineset + +import ( + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +func TestMachineSetController_calculateStatus(t *testing.T) { + readyNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "rNode"}, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + corev1.NodeCondition{ + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: time.Now()}, + }, + }, + }, + } + notReadyNode := readyNode.DeepCopy() + notReadyNode.Name = "nrNode" + notReadyNode.Status.Conditions[0].Status = corev1.ConditionFalse + unknownStatusNode := readyNode.DeepCopy() + unknownStatusNode.Name = "usNode" + unknownStatusNode.Status.Conditions[0].Status = corev1.ConditionUnknown + noConditionNode := readyNode.DeepCopy() + noConditionNode.Name = "ncNode" + noConditionNode.Status.Conditions = []corev1.NodeCondition{} + availableNode := readyNode.DeepCopy() + availableNode.Name = "aNode" + availableNode.Status.Conditions[0].LastTransitionTime.Time = time.Now().Add(time.Duration(-6) * time.Minute) + + tests := []struct { + name string + machines []*v1alpha1.Machine + setMinReadySeconds bool + minReadySeconds int32 + expectedReplicas int32 + expectedLabeledReplicas int32 + expectedReadyReplicas int32 + expectedAvailableReplicas int32 + }{ + { + name: "scenario 1: empty machinset.", + }, + { + name: "scenario 2: 1 replica, 1 labeled machine", + machines: []*v1alpha1.Machine{ + machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), + }, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + }, + { + name: "scenario 3: 1 replica, 0 labeled machine", + machines: []*v1alpha1.Machine{ + setDifferentLabels(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1")), + }, + expectedReplicas: 1, + }, + { + name: "scenario 4: 1 replica, 1 ready machine", + machines: []*v1alpha1.Machine{ + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), readyNode), + }, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + expectedReadyReplicas: 1, + }, + { + name: "scenario 5: 1 replica, 0 ready machine, not ready node", + machines: []*v1alpha1.Machine{ + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), notReadyNode), + }, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + }, + { + name: "scenario 6: 1 replica, 0 ready machine, unknown node", + machines: []*v1alpha1.Machine{ + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), unknownStatusNode), + }, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + }, + { + name: "scenario 7: 1 replica, 0 ready machine, missing condition node", + machines: []*v1alpha1.Machine{ + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), noConditionNode), + }, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + }, + { + name: "scenario 8: 1 replica, 1 available machine, minReadySeconds = 0", + machines: []*v1alpha1.Machine{ + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), readyNode), + }, + setMinReadySeconds: true, + minReadySeconds: 0, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + expectedReadyReplicas: 1, + expectedAvailableReplicas: 1, + }, + { + name: "scenario 9: 1 replica, 1 available machine, 360s elapsed, need 300s", + machines: []*v1alpha1.Machine{ + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), availableNode), + }, + setMinReadySeconds: true, + minReadySeconds: 300, + expectedReplicas: 1, + expectedLabeledReplicas: 1, + expectedReadyReplicas: 1, + expectedAvailableReplicas: 1, + }, + { + name: "scenario 10: 4 replicas, 3 labeled, 2 ready, 1 available machine", + machines: []*v1alpha1.Machine{ + setDifferentLabels(setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), noConditionNode)), + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), notReadyNode), + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), readyNode), + setNode(machineFromMachineSet(createMachineSet(1, "foo", "bar1", "acme"), "bar1"), availableNode), + }, + setMinReadySeconds: true, + minReadySeconds: 300, + expectedReplicas: 4, + expectedLabeledReplicas: 3, + expectedReadyReplicas: 2, + expectedAvailableReplicas: 1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + rObjects := []runtime.Object{readyNode, notReadyNode, unknownStatusNode, noConditionNode, availableNode} + k8sClient := fake.NewSimpleClientset(rObjects...) + + c := &MachineSetControllerImpl{} + c.kubernetesClient = k8sClient + + ms := createMachineSet(len(test.machines), "foo", "bar1", "acme") + if test.setMinReadySeconds { + ms.Spec.MinReadySeconds = test.minReadySeconds + } + + status := c.calculateStatus(ms, test.machines) + + if status.Replicas != test.expectedReplicas { + t.Errorf("got %v replicas, expected %v replicas", status.Replicas, test.expectedReplicas) + } + + if status.FullyLabeledReplicas != test.expectedLabeledReplicas { + t.Errorf("got %v fully labeled replicas, expected %v fully labeled replicas", status.FullyLabeledReplicas, test.expectedLabeledReplicas) + } + + if status.ReadyReplicas != test.expectedReadyReplicas { + t.Errorf("got %v ready replicas, expected %v ready replicas", status.ReadyReplicas, test.expectedReadyReplicas) + } + + if status.AvailableReplicas != test.expectedAvailableReplicas { + t.Errorf("got %v available replicas, expected %v available replicas", status.AvailableReplicas, test.expectedAvailableReplicas) + } + + }) + } +} + +func setNode(machine *v1alpha1.Machine, node *corev1.Node) *v1alpha1.Machine { + machine.Status.NodeRef = getNodeRef(node) + return machine +} + +func getNodeRef(node *corev1.Node) *corev1.ObjectReference { + return &corev1.ObjectReference{ + Kind: "Node", + Name: node.ObjectMeta.Name, + } +}