From 548a3ca6d8b868516e944d0e248cfbe97f51697a Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Fri, 10 Sep 2021 13:59:27 +1000 Subject: [PATCH 01/11] Changed shell to lookup an annotation on VM for ssh endpoint info --- pkg/shell/shell.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index 4924ab45..f32fac32 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -127,7 +127,7 @@ func (sp ShellProxy) ConnectFunc(w http.ResponseWriter, r *http.Request) { } // get the host and port - host := vm.Status.PublicIP + host := vm.Annotations["sshEndpoint"] port := "22" if sshDev == "true" { if sshDevHost != "" { From 0c238ba69348dd5af6adb903e807449f144263c4 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Tue, 14 Sep 2021 10:01:33 +1000 Subject: [PATCH 02/11] Fall back to publicIP if annotation is missing --- pkg/shell/shell.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index f32fac32..a3e9146c 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -127,7 +127,10 @@ func (sp ShellProxy) ConnectFunc(w http.ResponseWriter, r *http.Request) { } // get the host and port - host := vm.Annotations["sshEndpoint"] + host, ok := vm.Annotations["sshEndpoint"] + if !ok { + host = vm.Status.PublicIP + } port := "22" if sshDev == "true" { if sshDevHost != "" { From 22a64416f6e77d635f4037f0fb53d90a46032243 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Thu, 16 Sep 2021 17:34:28 +1000 Subject: [PATCH 03/11] Dropped dynamicbindrequests --- .../dynamicbindcontroller.go | 4 +- .../vmclaimcontroller/vmclaimcontroller.go | 491 +++++++----------- pkg/sessionserver/sessionserver.go | 1 + 3 files changed, 204 insertions(+), 292 deletions(-) diff --git a/pkg/controllers/dynamicbindcontroller/dynamicbindcontroller.go b/pkg/controllers/dynamicbindcontroller/dynamicbindcontroller.go index 2409b16c..3a70af3b 100644 --- a/pkg/controllers/dynamicbindcontroller/dynamicbindcontroller.go +++ b/pkg/controllers/dynamicbindcontroller/dynamicbindcontroller.go @@ -144,6 +144,7 @@ func (d *DynamicBindController) reconcileDynamicBindRequest(dynamicBindRequest * if err != nil { glog.Errorf("error retrieving corresponding virtual machine claim %s for dynamic bind request %s", dynamicBindRequest.Spec.VirtualMachineClaim, dynamicBindRequest.Spec.Id) + return err } var dbcSelector metav1.ListOptions @@ -161,6 +162,7 @@ func (d *DynamicBindController) reconcileDynamicBindRequest(dynamicBindRequest * if err != nil { glog.Errorf("Error while retrieving dynamic bind configurations, %v", err) + return err } var chosenDynamicBindConfiguration *hfv1.DynamicBindConfiguration @@ -179,7 +181,7 @@ func (d *DynamicBindController) reconcileDynamicBindRequest(dynamicBindRequest * } if err != nil { glog.Errorf("Error while retrieving environment %v", err) - return nil + return err } if !environment.Spec.BurstCapable { diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index 66db03a4..2ec75fc3 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -3,8 +3,9 @@ package vmclaimcontroller import ( "context" "fmt" + "github.com/hobbyfarm/gargantua/pkg/controllers/scheduledevent" + "github.com/hobbyfarm/gargantua/pkg/sessionserver" "math/rand" - "strings" "time" "github.com/golang/glog" @@ -13,6 +14,7 @@ import ( hfInformers "github.com/hobbyfarm/gargantua/pkg/client/informers/externalversions" hfListers "github.com/hobbyfarm/gargantua/pkg/client/listers/hobbyfarm.io/v1" "github.com/hobbyfarm/gargantua/pkg/util" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -66,7 +68,7 @@ func NewVMClaimController(hfClientSet hfClientset.Interface, hfInformerFactory h vmInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ AddFunc: vmClaimController.enqueueVM, UpdateFunc: func(old, new interface{}) { - + vmClaimController.enqueueVM(new) }, DeleteFunc: vmClaimController.enqueueVM, }, time.Minute*30) @@ -137,26 +139,36 @@ func (v *VMClaimController) processNextVM() bool { } err := func() error { - defer v.vmWorkqueue.Done(obj) _, objName, err := cache.SplitMetaNamespaceKey(obj.(string)) vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, objName, metav1.GetOptions{}) if err != nil { + // ideally should put logic here to determine if we need to retry and push this vm back onto the workqueue - glog.Errorf("error while retrieving vm %s: %v", objName, err) - return nil + if errors.IsNotFound(err) { + return nil + + } else { + glog.Errorf("error while retrieving vm %s: %v, will be requeued", objName, err) + return err + } } - if vm.Spec.VirtualMachineClaimId != "" { + // trigger reconcile on vmClaims only when associated VM is running + // this should avoid triggering unwanted reconciles of VMClaims until the VM's are running + if vm.Spec.VirtualMachineClaimId != "" && vm.Status.Status == hfv1.VmStatusRunning { v.vmClaimWorkqueue.Add(vm.Spec.VirtualMachineClaimId) } return nil }() if err != nil { - glog.Errorf("vm claim controller process next vm returned an error %v", err) + // return and requeue the object + //v.vmWorkqueue.Add(obj) return true } + //vm event has been processed successfully ignore it + v.vmWorkqueue.Done(obj) return true } @@ -170,252 +182,38 @@ func (v *VMClaimController) processNextVMClaim() bool { } err := func() error { - defer v.vmClaimWorkqueue.Done(obj) - glog.V(4).Infof("processing vm claim in vm claim controller: %v", obj) - _, objName, err := cache.SplitMetaNamespaceKey(obj.(string)) // this is actually not necessary because VM's are not namespaced yet... + _, objName, err := cache.SplitMetaNamespaceKey(obj.(string)) if err != nil { glog.Errorf("error while splitting meta namespace key %v", err) - return nil + return err } + // fetch vmClaim vmClaim, err := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Get(v.ctx, objName, metav1.GetOptions{}) if err != nil { - glog.Errorf("error while retrieving virtual machine claim %s, likely deleted %v", objName, err) - //v.vmClaimWorkqueue.Forget(obj) - return nil - } - - if vmClaim.Status.Tainted { - //v.vmClaimWorkqueue.Forget(obj) - glog.V(8).Infof("vm claim %s tainted, forgetting", objName) - return nil - } - - if vmClaim.Status.Bound && vmClaim.Status.Ready { - //v.vmClaimWorkqueue.Forget(obj) - glog.V(8).Infof("vm claim %s already bound and ready, forgetting", objName) - return nil - } - - if vmClaim.Status.Bound && !vmClaim.Status.Ready { - vmClaimIsReady := true - for _, needed := range vmClaim.Spec.VirtualMachines { - if needed.VirtualMachineId != "" { - vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, needed.VirtualMachineId, metav1.GetOptions{}) - - if err != nil { - glog.Errorf("error while retrieving vm from k8s api: %v", err) - //v.vmClaimWorkqueue.AddRateLimited(obj) - v.vmClaimWorkqueue.Add(obj) - return nil - } - - if vm.Status.Status != hfv1.VmStatusRunning { - vmClaimIsReady = false - break - } - } else { - glog.Errorf("found vm claim marked as bound but vm ID was not populated") - vmClaimIsReady = false - break - } - } - - if vmClaimIsReady { - v.updateVMClaimStatus(true, true, vmClaim.Spec.Id) - //v.vmClaimWorkqueue.Forget(obj) - glog.V(8).Infof("vm claim %s is now bound and ready, forgetting", objName) - return nil - } - glog.V(8).Infof("vm claim %s is not ready yet, requeuing", objName) - //v.vmClaimWorkqueue.AddRateLimited(obj) - v.vmClaimWorkqueue.Add(obj) - return nil - } - - if vmClaim.Status.BindMode == "dynamic" { - - // let's check to see if there is an active DynamicBindRequest - - dynamicBindRequest, err := v.hfClientSet.HobbyfarmV1().DynamicBindRequests().Get(v.ctx, vmClaim.Status.DynamicBindRequestId, metav1.GetOptions{}) - - if err != nil { - glog.Errorf("Error while attempting to retrieve the dynamic bind request. Perhaps this is a transient error, queuing again.") - //v.vmClaimWorkqueue.AddRateLimited(obj) - v.vmClaimWorkqueue.Add(obj) + if errors.IsNotFound(err) { + glog.Infof("vmClaim %s not found on queue.. ignoring", objName) return nil - } - - if !dynamicBindRequest.Status.Expired { - if dynamicBindRequest.Status.Fulfilled { // we are ready to bind this vm claim - for vmName, vmId := range dynamicBindRequest.Status.VirtualMachineIds { - v.updateVMClaimWithVM(vmName, vmId, objName) - } - v.updateVMClaimStatus(true, false, objName) - } } else { - v.updateVMClaimBindMode("static", "", vmClaim.Spec.Id) - } - //v.vmClaimWorkqueue.AddRateLimited(obj) - v.vmClaimWorkqueue.Add(obj) - - return nil - } else { - - if vmClaim.Status.BindMode != "static" && vmClaim.Status.StaticBindAttempts == 0 { - v.updateVMClaimBindMode("static", "", vmClaim.Spec.Id) + glog.Errorf("error while retrieving vmclaim %s from queue with err %v", objName, err) + return err } - - needed := make(map[string]int) - - for _, template := range vmClaim.Spec.VirtualMachines { - if template.VirtualMachineId == "" { - if val, ok := needed[template.Template]; ok { - needed[template.Template] = val + 1 - } else { - needed[template.Template] = 1 - } - } - } - - if len(needed) == 0 { - glog.V(8).Infof("vm claim %s does not need any vms, marking ready and bound", objName) - v.updateVMClaimStatus(true, true, objName) - //v.vmClaimWorkqueue.Forget(obj) - return nil - } - - envList, err := v.hfClientSet.HobbyfarmV1().Environments().List(v.ctx, metav1.ListOptions{}) - - if err != nil { - glog.Error(err) - } - - var chosenEnvironmentId string - - environments := envList.Items - - rand.Seed(time.Now().UnixNano()) - - rand.Shuffle(len(environments), func(i, j int) { - environments[i], environments[j] = environments[j], environments[i] - }) - - for _, env := range environments { - acceptable := true - for t, n := range needed { - vmLabels := labels.Set{ - "bound": "false", - "environment": env.Name, - "ready": "true", // do we really want to be marking ready as a label - "template": t, - } - if vmClaim.Spec.RestrictedBind { - vmLabels["restrictedbind"] = "true" - vmLabels["restrictedbindvalue"] = vmClaim.Spec.RestrictedBindValue - } else { - vmLabels["restrictedbind"] = "false" - } - - vms, err := v.vmLister.List(vmLabels.AsSelector()) - - if err != nil { - glog.Error(err) - acceptable = false - break - } - // if the number of vm's available in this environment is less than the number of vm's we need for this template - if len(vms) < n { - acceptable = false - break - } - - } - if acceptable { - chosenEnvironmentId = env.Name // @todo: change to using the ID instead of name - break - } - } - - if chosenEnvironmentId == "" { - glog.Errorf("error while trying to find matching environment for vm claim %s", vmClaim.Name) - if vmClaim.Status.StaticBindAttempts > StaticBindAttemptThreshold && vmClaim.Spec.DynamicCapable { - // need to create a dynamic bind request - dbrName := strings.Join([]string{vmClaim.Name + "-", fmt.Sprintf("%08x", rand.Uint32())}, "-") - dbr := &hfv1.DynamicBindRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: dbrName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "VirtualMachineClaim", - Name: vmClaim.Name, - UID: vmClaim.UID, - }, - }, - Labels: map[string]string{ - "hobbyfarm.io/session": vmClaim.Labels["hobbyfarm.io/session"], - }, - }, - Spec: hfv1.DynamicBindRequestSpec{ - Id: dbrName, - VirtualMachineClaim: vmClaim.Spec.Id, - Attempts: DynamicBindAttemptThreshold, - }, - Status: hfv1.DynamicBindRequestStatus{ - CurrentAttempts: 0, - Expired: false, - Fulfilled: false, - DynamicBindConfigurationId: "", - }, - } - - dbr, err := v.hfClientSet.HobbyfarmV1().DynamicBindRequests().Create(v.ctx, dbr, metav1.CreateOptions{}) - if err != nil { - glog.Errorf("Error creating dynamic bind request for VMClaim %s: %v", vmClaim.Spec.Id, err) - //v.vmClaimWorkqueue.AddRateLimited(obj) - v.vmClaimWorkqueue.Add(obj) - return nil - } - - v.updateVMClaimBindMode("dynamic", dbr.Spec.Id, vmClaim.Spec.Id) - glog.V(6).Infof("Created dynamic bind request %s for VM Claim %s", dbr.Spec.Id, vmClaim.Spec.Id) - } else { - v.updateVMClaimStaticBindAttempts(vmClaim.Status.StaticBindAttempts+1, vmClaim.Spec.Id) - } - //v.vmClaimWorkqueue.AddRateLimited(obj) - v.vmClaimWorkqueue.Add(obj) - return nil - } - - for name, vmStruct := range vmClaim.Spec.VirtualMachines { - if vmStruct.VirtualMachineId == "" { - vmId, err := v.assignNextFreeVM(vmClaim.Spec.Id, vmClaim.Spec.UserId, vmStruct.Template, chosenEnvironmentId, vmClaim.Spec.RestrictedBind, vmClaim.Spec.RestrictedBindValue) - if err != nil { - glog.Fatalf("error while assigning next free VM %v", err) - } - - err = v.updateVMClaimWithVM(name, vmId, vmClaim.Spec.Id) - if err != nil { - glog.Fatalf("error while updating VM Claim with VM %v", err) - } - } - } - - v.updateVMClaimStatus(true, true, vmClaim.Spec.Id) - - //v.vmClaimWorkqueue.Forget(obj) - glog.V(4).Infof("vmclaim processed and assigned by controller %v", objName) - - return nil } + // ignore vm objects which are being deleted + if vmClaim.DeletionTimestamp.IsZero() { + return v.processVMClaim(vmClaim) + } + return nil }() if err != nil { + // requeue object + //v.vmClaimWorkqueue.Add(obj) return true } + v.vmClaimWorkqueue.Done(obj) return true } @@ -489,7 +287,7 @@ func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, temp } -func (v *VMClaimController) updateVMClaimWithVM(vmName string, vmId string, vmClaimId string) error { +func (v *VMClaimController) updateVMClaimWithVM(vmDetails map[string]string, vmClaimId string) error { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Get(v.ctx, vmClaimId, metav1.GetOptions{}) @@ -497,11 +295,12 @@ func (v *VMClaimController) updateVMClaimWithVM(vmName string, vmId string, vmCl return fmt.Errorf("Error retrieving latest version of Virtual Machine Claim %s: %v", vmClaimId, getErr) } - vmClaimVM := result.Spec.VirtualMachines[vmName] - - vmClaimVM.VirtualMachineId = vmId + for vmName, vmId := range vmDetails { + vmClaimVM := result.Spec.VirtualMachines[vmName] + vmClaimVM.VirtualMachineId = vmId - result.Spec.VirtualMachines[vmName] = vmClaimVM + result.Spec.VirtualMachines[vmName] = vmClaimVM + } vmc, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Update(v.ctx, result, metav1.UpdateOptions{}) glog.V(4).Infof("updated result for virtual machine claim") @@ -522,18 +321,12 @@ func (v *VMClaimController) updateVMClaimWithVM(vmName string, vmId string, vmCl return nil } -func (v *VMClaimController) updateVMClaimStatus(bound bool, ready bool, vmClaimId string) error { +func (v *VMClaimController) updateVMClaimStatus(bound bool, ready bool, vmc *hfv1.VirtualMachineClaim) error { + vmc.Status.Bound = bound + vmc.Status.Ready = ready retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Get(v.ctx, vmClaimId, metav1.GetOptions{}) - if getErr != nil { - return fmt.Errorf("Error retrieving latest version of Virtual Machine Claim %s: %v", vmClaimId, getErr) - } - - result.Status.Bound = bound - result.Status.Ready = ready - - vmc, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Update(v.ctx, result, metav1.UpdateOptions{}) + vmc, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Update(v.ctx, vmc, metav1.UpdateOptions{}) if updateErr != nil { return updateErr } @@ -547,69 +340,185 @@ func (v *VMClaimController) updateVMClaimStatus(bound bool, ready bool, vmClaimI return nil }) if retryErr != nil { - return fmt.Errorf("Error updating Virtual Machine Claim: %s, %v", vmClaimId, retryErr) + return fmt.Errorf("Error updating Virtual Machine Claim: %s, %v", vmc.Name, retryErr) } return nil } -func (v *VMClaimController) updateVMClaimBindMode(bindMode string, dynamicBindRequestId string, vmClaimId string) error { +func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err error) { + if vmc.Status.Tainted { + glog.Infof("vmclaim %v is tainted.. ignoring", vmc.Name) + return nil + } - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Get(v.ctx, vmClaimId, metav1.GetOptions{}) - if getErr != nil { - return fmt.Errorf("Error retrieving latest version of Virtual Machine Claim %s: %v", vmClaimId, getErr) + if !vmc.Status.Bound && !vmc.Status.Ready { + // submit VM requests // + // update status + err = v.submitVirtualMachines(vmc) + if err != nil { + return err } + return v.updateVMClaimStatus(true, false, vmc) + } - if bindMode == "static" { - result.Status.StaticBindAttempts = 0 + if vmc.Status.Bound && !vmc.Status.Ready { + // reconcile triggered by VM being ready + // lets check the VM's + ready, err := v.checkVMStatus(vmc) + if err != nil { + glog.Errorf("error checking vmStatus for vmc: %s %v", vmc.Name, err) + return err } - result.Status.BindMode = bindMode - result.Status.DynamicBindRequestId = dynamicBindRequestId + // update status + glog.V(4).Infof("vm's have been requested for vmclaim: %s", vmc.Name) + return v.updateVMClaimStatus(true, ready, vmc) + } - vmc, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Update(v.ctx, result, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr + if vmc.Status.Bound && vmc.Status.Ready { + // nothing else needs to be done.. ignore and move along + glog.V(4).Infof("vmclaim %s is ready", vmc.Name) + } + + return nil +} + +func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) (err error) { + accessCode, ok := vmc.Labels[sessionserver.AccessCodeLabel] + if !ok { + glog.Error("accessCode label not set on vmc, aborting") + return fmt.Errorf("accessCode label not set on vmc, aborting") + } + + env, seName, dbc, err := v.findEnvironmentForVM(accessCode) + if err != nil { + glog.Errorf("error fetching environment for access code %s %v", accessCode, err) + return err + } + vmMap := make(map[string]hfv1.VirtualMachineClaimVM) + for vmName, vmDetails := range vmc.Spec.VirtualMachines { + genName := fmt.Sprintf("%s-%08x", vmc.Name, rand.Uint32()) + vm := &hfv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: genName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "hobbyfarm.io/v1", + Kind: "VirtualMachineClaim", + Name: vmc.Name, + UID: vmc.UID, + }, + }, + Labels: map[string]string{ + "dynamic": "true", + "vmc": vmc.Name, + "template": vmDetails.Template, + "environment": env.Name, + "bound": "true", + "ready": "false", + scheduledevent.ScheduledEventLabel: seName, + }, + }, + Spec: hfv1.VirtualMachineSpec{ + Id: genName, + VirtualMachineTemplateId: vmDetails.Template, + KeyPair: "", + VirtualMachineClaimId: vmc.Name, + UserId: vmc.Spec.UserId, + Provision: true, + VirtualMachineSetId: "", + }, + Status: hfv1.VirtualMachineStatus{ + Status: hfv1.VmStatusRFP, + Allocated: true, + Tainted: false, + WsEndpoint: env.Spec.WsEndpoint, + EnvironmentId: env.Name, + PublicIP: "", + PrivateIP: "", + }, + } + // used to later repopulate the info back // + vmMap[vmName] = hfv1.VirtualMachineClaimVM{ + Template: vmDetails.Template, + VirtualMachineId: genName, + } + sshUser, exists := env.Spec.TemplateMapping[vmDetails.Template]["ssh_username"] + if exists { + vm.Spec.SshUsername = sshUser } - glog.V(4).Infof("updated result for virtual machine claim") - verifyErr := util.VerifyVMClaim(v.vmClaimLister, vmc) + // extra label to indicate external provisioning so tfpcontroller ignores this request // + if provisionMethod, ok := env.Annotations["hobbyfarm.io/provisioner"]; ok { + vm.Labels["hobbyfarm.io/provisioner"] = provisionMethod + vm.Spec.Provision = false + } - if verifyErr != nil { - return verifyErr + if dbc.Spec.RestrictedBind { + vm.ObjectMeta.Labels["restrictedbind"] = "true" + vm.ObjectMeta.Labels["restrictedbindvalue"] = dbc.Spec.RestrictedBindValue + } else { + vm.ObjectMeta.Labels["restrictedbind"] = "false" } - return nil - }) - if retryErr != nil { - return fmt.Errorf("Error updating Virtual Machine Claim: %s, %v", vmClaimId, retryErr) + + vm.Labels["hobbyfarm.io/vmtemplate"] = vm.Spec.VirtualMachineTemplateId + + _, err = v.hfClientSet.HobbyfarmV1().VirtualMachines().Create(v.ctx, vm, metav1.CreateOptions{}) } + + vmc.Spec.VirtualMachines = vmMap return nil } -func (v *VMClaimController) updateVMClaimStaticBindAttempts(staticBindAttempts int, vmClaimId string) error { +func (v *VMClaimController) findEnvironmentForVM(accessCode string) (env *hfv1.Environment, seName string, dbc *hfv1.DynamicBindConfiguration, err error) { + // find scheduledEvent for this accessCode + var envName string + seList, err := v.hfClientSet.HobbyfarmV1().ScheduledEvents().List(v.ctx, metav1.ListOptions{}) + if err != nil { + return env, seName, dbc, err + } - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Get(v.ctx, vmClaimId, metav1.GetOptions{}) - if getErr != nil { - return fmt.Errorf("Error retrieving latest version of Virtual Machine Claim %s: %v", vmClaimId, getErr) + for _, se := range seList.Items { + if se.Spec.AccessCode == accessCode { + seName = se.Name + for k, _ := range se.Spec.RequiredVirtualMachines { + envName = k + } } + } - result.Status.StaticBindAttempts = staticBindAttempts + env, err = v.hfClientSet.HobbyfarmV1().Environments().Get(v.ctx, envName, metav1.GetOptions{}) - vmc, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Update(v.ctx, result, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr - } - glog.V(4).Infof("updated result for virtual machine claim") + dbcList, err := v.hfClientSet.HobbyfarmV1().DynamicBindConfigurations().List(v.ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("restrictedbindvalue=%s", seName), + }) - verifyErr := util.VerifyVMClaim(v.vmClaimLister, vmc) + if err != nil { + glog.Errorf("error listing dbc %v", err) + return env, seName, dbc, err + } - if verifyErr != nil { - return verifyErr + if len(dbcList.Items) != 1 { + return env, seName, dbc, fmt.Errorf("incorrect number of dbc matching sessionName found") + } + + dbc = &hfv1.DynamicBindConfiguration{} + dbc = &dbcList.Items[0] + return env, seName, dbc, err +} + +func (v *VMClaimController) checkVMStatus(vmc *hfv1.VirtualMachineClaim) (ready bool, err error) { + ready = true + for _, vmTemplate := range vmc.Spec.VirtualMachines { + vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, vmTemplate.VirtualMachineId, metav1.GetOptions{}) + if err != nil { + return ready, err + } + if vm.Status.Status == hfv1.VmStatusRunning { + ready = ready && true + } else { + ready = ready && false } - return nil - }) - if retryErr != nil { - return fmt.Errorf("Error updating Virtual Machine Claim: %s, %v", vmClaimId, retryErr) } - return nil + + return ready, err } diff --git a/pkg/sessionserver/sessionserver.go b/pkg/sessionserver/sessionserver.go index c2aa9a1a..de329bea 100644 --- a/pkg/sessionserver/sessionserver.go +++ b/pkg/sessionserver/sessionserver.go @@ -220,6 +220,7 @@ func (sss SessionServer) NewSessionFunc(w http.ResponseWriter, r *http.Request) labels := make(map[string]string) labels[vmcSessionLabel] = session.Name // map vmc to session labels[UserSessionLabel] = user.Spec.Id // map session to user in a way that is searchable + labels[AccessCodeLabel] = session.Labels[AccessCodeLabel] virtualMachineClaim.Labels = labels virtualMachineClaim.Spec.Id = vmcId virtualMachineClaim.Name = vmcId From 4a6fac9e363125af30e9b2f35b0818fb9de816e7 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Fri, 24 Sep 2021 09:59:00 +1000 Subject: [PATCH 04/11] cleaned up handling of missing api objects in util.VerifyVM, cleaned up vmclaimcontroller --- .../vmclaimcontroller/vmclaimcontroller.go | 76 +------------------ .../vmsetcontroller/vmsetcontroller.go | 2 + pkg/util/util.go | 10 ++- 3 files changed, 11 insertions(+), 77 deletions(-) diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index 2ec75fc3..cf63514a 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -3,11 +3,12 @@ package vmclaimcontroller import ( "context" "fmt" - "github.com/hobbyfarm/gargantua/pkg/controllers/scheduledevent" - "github.com/hobbyfarm/gargantua/pkg/sessionserver" "math/rand" "time" + "github.com/hobbyfarm/gargantua/pkg/controllers/scheduledevent" + "github.com/hobbyfarm/gargantua/pkg/sessionserver" + "github.com/golang/glog" hfv1 "github.com/hobbyfarm/gargantua/pkg/apis/hobbyfarm.io/v1" hfClientset "github.com/hobbyfarm/gargantua/pkg/client/clientset/versioned" @@ -16,7 +17,6 @@ import ( "github.com/hobbyfarm/gargantua/pkg/util" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" @@ -217,76 +217,6 @@ func (v *VMClaimController) processNextVMClaim() bool { return true } -func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, template string, environmentId string, restrictedBind bool, restrictedBindValue string) (string, error) { - - vmLabels := labels.Set{ - "bound": "false", - "environment": environmentId, - "ready": "true", - "template": template, - } - - if restrictedBind { - vmLabels["restrictedbind"] = "true" - vmLabels["restrictedbindvalue"] = restrictedBindValue - } else { - vmLabels["restrictedbind"] = "false" - } - - vms, err := v.vmLister.List(vmLabels.AsSelector()) - - if err != nil { - return "", fmt.Errorf("error while listing all vms %v", err) - } - - assigned := false - vmId := "" - for _, vm := range vms { - if !vm.Status.Allocated && vm.Status.Status == hfv1.VmStatusRunning && !vm.Status.Tainted { - // we can assign this vm - assigned = true - vmId = vm.Spec.Id - - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, vmId, metav1.GetOptions{}) - if getErr != nil { - return fmt.Errorf("Error retrieving latest version of Virtual Machine %s: %v", vmId, getErr) - } - - result.Status.Allocated = true - result.Spec.VirtualMachineClaimId = vmClaimId - result.Spec.UserId = user - - result.Labels["bound"] = "true" - - vm, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachines().Update(v.ctx, result, metav1.UpdateOptions{}) - if updateErr != nil { - return updateErr - } - glog.V(4).Infof("updated result for virtual machine") - - verifyErr := util.VerifyVM(v.vmLister, vm) - - if verifyErr != nil { - return verifyErr - } - return nil - }) - if retryErr != nil { - return "", fmt.Errorf("Error updating Virtual Machine: %s, %v", vmId, retryErr) - } - break - } - } - - if assigned { - return vmId, nil - } - - return vmId, fmt.Errorf("unknown error while assigning next free vm") - -} - func (v *VMClaimController) updateVMClaimWithVM(vmDetails map[string]string, vmClaimId string) error { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { diff --git a/pkg/controllers/vmsetcontroller/vmsetcontroller.go b/pkg/controllers/vmsetcontroller/vmsetcontroller.go index 9a649bd4..ef977394 100644 --- a/pkg/controllers/vmsetcontroller/vmsetcontroller.go +++ b/pkg/controllers/vmsetcontroller/vmsetcontroller.go @@ -196,6 +196,8 @@ func (v *VirtualMachineSetController) processNextVMSet() bool { return true } + // successfully reconcilled, mark object as done + v.vmSetWorkqueue.Done(obj) return true } diff --git a/pkg/util/util.go b/pkg/util/util.go index bb04b945..a2d4e3a7 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -11,6 +11,8 @@ import ( "encoding/json" "encoding/pem" "fmt" + mrand "math/rand" + "github.com/golang/glog" hfv1 "github.com/hobbyfarm/gargantua/pkg/apis/hobbyfarm.io/v1" hfListers "github.com/hobbyfarm/gargantua/pkg/client/listers/hobbyfarm.io/v1" @@ -18,13 +20,13 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" - mrand "math/rand" - hfClientset "github.com/hobbyfarm/gargantua/pkg/client/clientset/versioned" "net/http" "strconv" "strings" "time" + + hfClientset "github.com/hobbyfarm/gargantua/pkg/client/clientset/versioned" ) type HTTPMessage struct { @@ -178,10 +180,10 @@ func VerifyVM(vmLister hfListers.VirtualMachineLister, vm *hfv1.VirtualMachine) var fromCache *hfv1.VirtualMachine fromCache, err = vmLister.Get(vm.Name) if err != nil { - glog.Error(err) if apierrors.IsNotFound(err) { - continue + return nil } + glog.Error(err) return err } if ResourceVersionAtLeast(fromCache.ResourceVersion, vm.ResourceVersion) { From 6f583fea0b2de5534e176b9ce0aeacc04318c7b9 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Mon, 27 Sep 2021 11:20:49 +1000 Subject: [PATCH 05/11] reorganized logic, vmccontroller now reaps vmc's and session controller immediately reaps finished sessions. Also dropped needed for reaping dbr's when session is finished --- pkg/controllers/session/sessioncontroller.go | 24 +++---------------- .../vmclaimcontroller/vmclaimcontroller.go | 9 +++++-- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/pkg/controllers/session/sessioncontroller.go b/pkg/controllers/session/sessioncontroller.go index 63ebed7e..4b56d29a 100644 --- a/pkg/controllers/session/sessioncontroller.go +++ b/pkg/controllers/session/sessioncontroller.go @@ -153,27 +153,9 @@ func (s *SessionController) reconcileSession(ssName string) error { timeUntilExpires := expires.Sub(now) - // clean up old (3 hours later) sessions, and only if they are finished - if expires.Add(SessionExpireTime).Before(now) && ss.Status.Finished { - // first we need to delete the DynamicBindRequests - err := s.hfClientSet.HobbyfarmV1().DynamicBindRequests().DeleteCollection(s.ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("hobbyfarm.io/session=%s", ss.Name), - }) - - if err != nil { - return fmt.Errorf("error deleting dynamicbindrequests with session label %s: %s", ss.Name, err) - } - - // then we need to delete the vmclaims - err = s.hfClientSet.HobbyfarmV1().VirtualMachineClaims().DeleteCollection(s.ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("hobbyfarm.io/session=%s", ss.Name), - }) - - if err != nil { - return fmt.Errorf("error deleting vmclaims with session label %s: %s", ss.Name, err) - } - - glog.V(6).Infof("deleted vmclaims for old session %s", ss.Name) + // clean up sessions if they are finished + if ss.Status.Finished { + glog.V(6).Infof("deleted finished session %s", ss.Name) // now that the vmclaims are deleted, go ahead and delete the session err = s.hfClientSet.HobbyfarmV1().Sessions().Delete(s.ctx, ss.Name, metav1.DeleteOptions{}) diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index cf63514a..7a4b78e6 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -277,8 +277,8 @@ func (v *VMClaimController) updateVMClaimStatus(bound bool, ready bool, vmc *hfv func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err error) { if vmc.Status.Tainted { - glog.Infof("vmclaim %v is tainted.. ignoring", vmc.Name) - return nil + glog.Infof("vmclaim %v is tainted.. cleaning it up", vmc.Name) + return v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Delete(v.ctx, vmc.Name, metav1.DeleteOptions{}) } if !vmc.Status.Bound && !vmc.Status.Ready { @@ -304,6 +304,11 @@ func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err e return v.updateVMClaimStatus(true, ready, vmc) } + // handle tainted VM's by cleaning them up + if vmc.Status.Tainted { + return v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Delete(v.ctx, vmc.Name, metav1.DeleteOptions{}) + } + if vmc.Status.Bound && vmc.Status.Ready { // nothing else needs to be done.. ignore and move along glog.V(4).Infof("vmclaim %s is ready", vmc.Name) From 83818bd13e9cc03eef19f18ce51242fc575ce97c Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Mon, 27 Sep 2021 15:44:32 +1000 Subject: [PATCH 06/11] more cleanup for util functions --- pkg/util/util.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index a2d4e3a7..7154c7ef 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -221,10 +221,10 @@ func VerifyVMSet(vmSetLister hfListers.VirtualMachineSetLister, vms *hfv1.Virtua var fromCache *hfv1.VirtualMachineSet fromCache, err = vmSetLister.Get(vms.Name) if err != nil { - glog.Error(err) if apierrors.IsNotFound(err) { - continue + return nil } + glog.Error(err) return err } if ResourceVersionAtLeast(fromCache.ResourceVersion, vms.ResourceVersion) { @@ -245,10 +245,10 @@ func VerifyVMClaim(vmClaimLister hfListers.VirtualMachineClaimLister, vmc *hfv1. var fromCache *hfv1.VirtualMachineClaim fromCache, err = vmClaimLister.Get(vmc.Name) if err != nil { - glog.Error(err) if apierrors.IsNotFound(err) { - continue + return nil } + glog.Error(err) return err } if ResourceVersionAtLeast(fromCache.ResourceVersion, vmc.ResourceVersion) { @@ -269,10 +269,10 @@ func VerifySession(sLister hfListers.SessionLister, s *hfv1.Session) error { var fromCache *hfv1.Session fromCache, err = sLister.Get(s.Name) if err != nil { - glog.Error(err) if apierrors.IsNotFound(err) { - continue + return nil } + glog.Error(err) return err } if ResourceVersionAtLeast(fromCache.ResourceVersion, s.ResourceVersion) { From 1c8424ad9f5b3763c830cff8cc93c405615d9346 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Wed, 29 Sep 2021 15:36:43 +1000 Subject: [PATCH 07/11] cleaned up unsed functions and changed filter on dbc to handle non restricted bind events --- .../vmclaimcontroller/vmclaimcontroller.go | 51 +++++-------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index 7a4b78e6..449269d1 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -140,6 +140,9 @@ func (v *VMClaimController) processNextVM() bool { err := func() error { _, objName, err := cache.SplitMetaNamespaceKey(obj.(string)) + if err != nil { + return err + } vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, objName, metav1.GetOptions{}) if err != nil { @@ -217,40 +220,6 @@ func (v *VMClaimController) processNextVMClaim() bool { return true } -func (v *VMClaimController) updateVMClaimWithVM(vmDetails map[string]string, vmClaimId string) error { - - retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Get(v.ctx, vmClaimId, metav1.GetOptions{}) - if getErr != nil { - return fmt.Errorf("Error retrieving latest version of Virtual Machine Claim %s: %v", vmClaimId, getErr) - } - - for vmName, vmId := range vmDetails { - vmClaimVM := result.Spec.VirtualMachines[vmName] - vmClaimVM.VirtualMachineId = vmId - - result.Spec.VirtualMachines[vmName] = vmClaimVM - } - - vmc, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Update(v.ctx, result, metav1.UpdateOptions{}) - glog.V(4).Infof("updated result for virtual machine claim") - - if updateErr != nil { - return updateErr - } - - verifyErr := util.VerifyVMClaim(v.vmClaimLister, vmc) - if verifyErr != nil { - return verifyErr - } - return updateErr - }) - if retryErr != nil { - return fmt.Errorf("Error updating Virtual Machine Claim: %s, %v", vmClaimId, retryErr) - } - return nil -} - func (v *VMClaimController) updateVMClaimStatus(bound bool, ready bool, vmc *hfv1.VirtualMachineClaim) error { vmc.Status.Bound = bound @@ -270,7 +239,7 @@ func (v *VMClaimController) updateVMClaimStatus(bound bool, ready bool, vmc *hfv return nil }) if retryErr != nil { - return fmt.Errorf("Error updating Virtual Machine Claim: %s, %v", vmc.Name, retryErr) + return fmt.Errorf("error updating Virtual Machine Claim: %s, %v", vmc.Name, retryErr) } return nil } @@ -398,6 +367,9 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) vm.Labels["hobbyfarm.io/vmtemplate"] = vm.Spec.VirtualMachineTemplateId _, err = v.hfClientSet.HobbyfarmV1().VirtualMachines().Create(v.ctx, vm, metav1.CreateOptions{}) + if err != nil { + return err + } } vmc.Spec.VirtualMachines = vmMap @@ -423,8 +395,13 @@ func (v *VMClaimController) findEnvironmentForVM(accessCode string) (env *hfv1.E env, err = v.hfClientSet.HobbyfarmV1().Environments().Get(v.ctx, envName, metav1.GetOptions{}) + if err != nil { + glog.Errorf("error fetching environment %v", err) + return env, seName, dbc, err + } + dbcList, err := v.hfClientSet.HobbyfarmV1().DynamicBindConfigurations().List(v.ctx, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("restrictedbindvalue=%s", seName), + LabelSelector: fmt.Sprintf("%s=%s", scheduledevent.ScheduledEventLabel, seName), }) if err != nil { @@ -436,8 +413,8 @@ func (v *VMClaimController) findEnvironmentForVM(accessCode string) (env *hfv1.E return env, seName, dbc, fmt.Errorf("incorrect number of dbc matching sessionName found") } - dbc = &hfv1.DynamicBindConfiguration{} dbc = &dbcList.Items[0] + //dbc = &dbcList.Items[0] return env, seName, dbc, err } From c3d4ba89c5fe5cbb15c83bc02909cb80bea8c7c1 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Fri, 1 Oct 2021 15:45:04 +1000 Subject: [PATCH 08/11] handling of statically provisioned infra --- .../vmclaimcontroller/vmclaimcontroller.go | 157 +++++++++++-- .../vmsetcontroller/vmsetcontroller.go | 210 +++++++++--------- pkg/sessionserver/sessionserver.go | 23 ++ 3 files changed, 269 insertions(+), 121 deletions(-) diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index 449269d1..d779f9d3 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -17,6 +17,7 @@ import ( "github.com/hobbyfarm/gargantua/pkg/util" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" @@ -253,10 +254,22 @@ func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err e if !vmc.Status.Bound && !vmc.Status.Ready { // submit VM requests // // update status - err = v.submitVirtualMachines(vmc) - if err != nil { - return err + if vmc.Status.BindMode == "dynamic" { + err = v.submitVirtualMachines(vmc) + if err != nil { + return err + } + } else if vmc.Status.BindMode == "static" { + err = v.findVirtualMachines(vmc) + if err != nil { + glog.Errorf("error processing vmc %s - %s", vmc.Name, err.Error()) + return err + } + } else { + glog.Error("vmc bind mode needs to be either dynamic or static.. ignoring this object %s", vmc.Name) + return nil } + return v.updateVMClaimStatus(true, false, vmc) } @@ -273,11 +286,6 @@ func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err e return v.updateVMClaimStatus(true, ready, vmc) } - // handle tainted VM's by cleaning them up - if vmc.Status.Tainted { - return v.hfClientSet.HobbyfarmV1().VirtualMachineClaims().Delete(v.ctx, vmc.Name, metav1.DeleteOptions{}) - } - if vmc.Status.Bound && vmc.Status.Ready { // nothing else needs to be done.. ignore and move along glog.V(4).Infof("vmclaim %s is ready", vmc.Name) @@ -379,20 +387,11 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) func (v *VMClaimController) findEnvironmentForVM(accessCode string) (env *hfv1.Environment, seName string, dbc *hfv1.DynamicBindConfiguration, err error) { // find scheduledEvent for this accessCode var envName string - seList, err := v.hfClientSet.HobbyfarmV1().ScheduledEvents().List(v.ctx, metav1.ListOptions{}) + seName, envName, err = v.findScheduledEvent(accessCode) if err != nil { return env, seName, dbc, err } - for _, se := range seList.Items { - if se.Spec.AccessCode == accessCode { - seName = se.Name - for k, _ := range se.Spec.RequiredVirtualMachines { - envName = k - } - } - } - env, err = v.hfClientSet.HobbyfarmV1().Environments().Get(v.ctx, envName, metav1.GetOptions{}) if err != nil { @@ -434,3 +433,125 @@ func (v *VMClaimController) checkVMStatus(vmc *hfv1.VirtualMachineClaim) (ready return ready, err } + +func (v *VMClaimController) findScheduledEvent(accessCode string) (schedEvent string, envName string, err error) { + seList, err := v.hfClientSet.HobbyfarmV1().ScheduledEvents().List(v.ctx, metav1.ListOptions{}) + if err != nil { + return schedEvent, envName, err + } + + for _, se := range seList.Items { + if se.Spec.AccessCode == accessCode { + schedEvent = se.Name + for k, _ := range se.Spec.RequiredVirtualMachines { + envName = k + } + } + } + + if schedEvent == "" { + return schedEvent, envName, fmt.Errorf("no scheduled event matching access code %s found", accessCode) + } + + return schedEvent, envName, nil +} + +func (v *VMClaimController) findVirtualMachines(vmc *hfv1.VirtualMachineClaim) (err error) { + accessCode, ok := vmc.Labels[sessionserver.AccessCodeLabel] + if !ok { + glog.Error("accessCode label not set on vmc, aborting") + return fmt.Errorf("accessCode label not set on vmc, aborting") + } + _, env, err := v.findScheduledEvent(accessCode) + + if err != nil { + glog.Error("error finding scheduledevent during static bind") + return err + } + + vmMap := make(map[string]hfv1.VirtualMachineClaimVM) + for name, vmStruct := range vmc.Spec.VirtualMachines { + if vmStruct.VirtualMachineId == "" { + glog.Info("assigning a vm") + vmID, err := v.assignNextFreeVM(vmc.Spec.Id, vmc.Spec.UserId, vmStruct.Template, env, vmc.Spec.RestrictedBind, vmc.Spec.RestrictedBindValue) + if err != nil { + return err + } + vmMap[name] = hfv1.VirtualMachineClaimVM{ + Template: vmStruct.Template, + VirtualMachineId: vmID, + } + } + } + vmc.Spec.VirtualMachines = vmMap + return nil +} + +func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, template string, environmentId string, restrictedBind bool, restrictedBindValue string) (string, error) { + + vmLabels := labels.Set{ + "bound": "false", + "environment": environmentId, + "template": template, + } + + if restrictedBind { + vmLabels["restrictedbind"] = "true" + vmLabels["restrictedbindvalue"] = restrictedBindValue + } else { + vmLabels["restrictedbind"] = "false" + } + + vms, err := v.vmLister.List(vmLabels.AsSelector()) + glog.V(4).Info("found %d vm's matching this requirement", len(vms)) + if err != nil { + return "", fmt.Errorf("error while listing all vms %v", err) + } + + assigned := false + vmId := "" + for _, vm := range vms { + if !vm.Status.Allocated && !vm.Status.Tainted { + // we can assign this vm + assigned = true + vmId = vm.Spec.Id + + retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, vmId, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("error retrieving latest version of Virtual Machine %s: %v", vmId, getErr) + } + + result.Status.Allocated = true + result.Spec.VirtualMachineClaimId = vmClaimId + result.Spec.UserId = user + + result.Labels["bound"] = "true" + + vm, updateErr := v.hfClientSet.HobbyfarmV1().VirtualMachines().Update(v.ctx, result, metav1.UpdateOptions{}) + if updateErr != nil { + return updateErr + } + glog.V(4).Infof("updated result for virtual machine") + + verifyErr := util.VerifyVM(v.vmLister, vm) + + if verifyErr != nil { + return verifyErr + } + return nil + }) + if retryErr != nil { + return "", fmt.Errorf("error updating Virtual Machine: %s, %v", vmId, retryErr) + } + break + } + } + + if assigned { + return vmId, nil + } + + return vmId, fmt.Errorf("unknown error while assigning next free vm") + +} diff --git a/pkg/controllers/vmsetcontroller/vmsetcontroller.go b/pkg/controllers/vmsetcontroller/vmsetcontroller.go index ef977394..90c7af8a 100644 --- a/pkg/controllers/vmsetcontroller/vmsetcontroller.go +++ b/pkg/controllers/vmsetcontroller/vmsetcontroller.go @@ -14,6 +14,7 @@ import ( hfListers "github.com/hobbyfarm/gargantua/pkg/client/listers/hobbyfarm.io/v1" "github.com/hobbyfarm/gargantua/pkg/controllers/scheduledevent" "github.com/hobbyfarm/gargantua/pkg/util" + "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -21,13 +22,12 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" - "k8s.io/klog" ) type VirtualMachineSetController struct { hfClientSet hfClientset.Interface //vmSetWorkqueue workqueue.RateLimitingInterface - //vmWorkqueue workqueue.RateLimitingInterface + vmWorkqueue workqueue.RateLimitingInterface vmSetWorkqueue workqueue.Interface vmSetLister hfListers.VirtualMachineSetLister vmLister hfListers.VirtualMachineLister @@ -43,6 +43,7 @@ type VirtualMachineSetController struct { const ( vmEnvironmentIndex = "vm.vmclaim.controllers.hobbyfarm.io/environment-index" + vmSetFinalizer = "finalizer.hobbyfarm.io/vmset" ) func NewVirtualMachineSetController(hfClientSet hfClientset.Interface, hfInformerFactory hfInformers.SharedInformerFactory, ctx context.Context) (*VirtualMachineSetController, error) { @@ -55,8 +56,9 @@ func NewVirtualMachineSetController(hfClientSet hfClientset.Interface, hfInforme vmSetController.vmTemplateSynced = hfInformerFactory.Hobbyfarm().V1().VirtualMachineTemplates().Informer().HasSynced //vmSetController.vmSetWorkqueue = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "VMSet") - //vmSetController.vmWorkqueue = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "VM") + vmSetController.vmWorkqueue = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "VM") vmSetController.vmSetWorkqueue = workqueue.NewNamed("vmsc-vms") + //vmClaimController.hfInformerFactory = hfInformerFactory vmSetController.vmSetLister = hfInformerFactory.Hobbyfarm().V1().VirtualMachineSets().Lister() @@ -89,38 +91,13 @@ func NewVirtualMachineSetController(hfClientSet hfClientset.Interface, hfInforme } func (v *VirtualMachineSetController) handleVM(obj interface{}) { - var object metav1.Object - var ok bool - if object, ok = obj.(metav1.Object); !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - glog.Errorf("error decoding object, invalid type") - return - } - object, ok = tombstone.Obj.(metav1.Object) - if !ok { - glog.Errorf("error decoding object tombstone, invalid type") - return - } - klog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) - } - klog.V(4).Infof("Processing object: %s", object.GetName()) - if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { - // If this object is not owned by a VirtualMachineSet, we should not do anything more - // with it. - if ownerRef.Kind != "VirtualMachineSet" { - return - } - - vms, err := v.vmSetLister.Get(ownerRef.Name) - if err != nil { - klog.V(4).Infof("ignoring orphaned object '%s' of vmset '%s'", object.GetSelfLink(), ownerRef.Name) - return - } - - v.enqueueVMSet(vms) + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { return } + glog.V(8).Infof("enqueueing vm %v to check associated vmsets in vmsetcontroller", key) + v.vmWorkqueue.Add(key) } func (v *VirtualMachineSetController) enqueueVMSet(obj interface{}) { @@ -145,6 +122,7 @@ func (v *VirtualMachineSetController) Run(stopCh <-chan struct{}) error { } glog.Info("Starting vm set controller worker") go wait.Until(v.runVMSetWorker, time.Second, stopCh) + go wait.Until(v.runVMWorker, time.Second, stopCh) glog.Info("Started vm set controller worker") //if ok := cache.WaitForCacheSync(stopCh, ) <-stopCh @@ -157,6 +135,59 @@ func (v *VirtualMachineSetController) runVMSetWorker() { } } +func (v *VirtualMachineSetController) runVMWorker() { + for v.processNextVM() { + + } +} + +func (v *VirtualMachineSetController) processNextVM() bool { + obj, shutdown := v.vmWorkqueue.Get() + glog.V(8).Infof("processing VM in vmsetcontroller controller for update") + + if shutdown { + return false + } + + err := func() error { + _, objName, err := cache.SplitMetaNamespaceKey(obj.(string)) + if err != nil { + return err + } + vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Get(v.ctx, objName, metav1.GetOptions{}) + + if err != nil { + + // ideally should put logic here to determine if we need to retry and push this vm back onto the workqueue + if errors.IsNotFound(err) { + return nil + + } else { + glog.Errorf("error while retrieving vm %s: %v, will be requeued", objName, err) + return err + } + } + + // trigger reconcile on vmClaims only when associated VM is running + // this should avoid triggering unwanted reconciles of VMClaims until the VM's are running + if !vm.DeletionTimestamp.IsZero() { + glog.V(4).Infof("requeuing vmset %s to account for tainted vm %s", vm.Spec.VirtualMachineSetId, vm.Name) + defer v.vmSetWorkqueue.Add(vm.Spec.VirtualMachineSetId) + return v.removeVMFinalizer(vm) + } + + return nil + }() + + if err != nil { + // return and requeue the object + //v.vmWorkqueue.Add(obj) + return true + } + //vm event has been processed successfully ignore it + v.vmWorkqueue.Done(obj) + return true +} func (v *VirtualMachineSetController) processNextVMSet() bool { obj, shutdown := v.vmSetWorkqueue.Get() @@ -207,15 +238,15 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir "vmset": vmset.Name, }.AsSelector()) - if len(currentVMs) > vmset.Spec.Count { - // if the desired number of vms is less than the current number of VM's - // let's go through and taint/delete the ones that don't belong - + if err != nil { + glog.Errorf("error listing vms in vmset controller") + return err } if len(currentVMs) < vmset.Spec.Count { // if desired count is greater than the current provisioned // 1. let's check the environment to see if there is available capacity // 2. if available capacity is available let's create new VM's + glog.V(4).Infof("vmset %s needs %d vm's but current vm count is %d", vmset.Name, vmset.Spec.Count, len(currentVMs)) env, err := v.envLister.Get(vmset.Spec.Environment) var provision bool provision = true @@ -235,7 +266,7 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir if err != nil { return fmt.Errorf("error while retrieving virtual machine template %s %v", vmset.Spec.VMTemplate, err) } - needed := vmset.Spec.Count - vmset.Status.ProvisionedCount + needed := vmset.Spec.Count - len(currentVMs) glog.V(5).Infof("provisioning %d vms", needed) // this code is so... verbose... @@ -293,6 +324,8 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir } else { vm.ObjectMeta.Labels["restrictedbind"] = "false" } + // adding a custom finalizer for reconcile of vmsets + vm.SetFinalizers([]string{vmSetFinalizer}) vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Create(v.ctx, vm, metav1.CreateOptions{}) if err != nil { glog.Error(err) @@ -305,8 +338,6 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir } } - // no matter what we should list the vm's and delete the ones that are ready for deletion - vms, err := v.vmLister.List(labels.Set{ "vmset": string(vmset.Name), }.AsSelector()) @@ -315,25 +346,6 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir glog.Errorf("error while retrieving vms owned by vmset %s", vmset.Name) } - /* TFP Controller will be the one responsible for deleting tainted vm's - for _, x := range vms { - if x.DeletionTimestamp == nil && x.Status.Tainted { - err := v.deleteVM(x) - if err != nil { - glog.Error(err) - } - } - } - */ - - vms, err = v.vmLister.List(labels.Set{ - "vmset": string(vmset.Name), - }.AsSelector()) - - if err != nil { - glog.Errorf("error while retrieving vms owned by vmset %s", vmset.Name) - } - provisionedCount := 0 activeCount := 0 for _, x := range vms { @@ -345,55 +357,14 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir err = v.updateVMSetCount(vmset.Name, activeCount, provisionedCount) - return nil -} - -func (v *VirtualMachineSetController) deleteVM(vm *hfv1.VirtualMachine) error { - err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Delete(v.ctx, vm.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - for i := 0; i < 25; i++ { - vmFromLister, err := v.vmLister.Get(vm.Name) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return err - } - if vmFromLister.DeletionTimestamp != nil { - return nil // we are done waiting for delete to happen finalizers can happen in the background doesn't matter - } - } - return nil -} - -func (v *VirtualMachineSetController) createVM(vm *hfv1.VirtualMachine) error { - vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines().Create(v.ctx, vm, metav1.CreateOptions{}) - if err != nil { - return err - } - for i := 0; i < 25; i++ { - vmFromLister, err := v.vmLister.Get(vm.Name) - if err != nil { - if apierrors.IsNotFound(err) { - glog.V(5).Infof("vm not in lister yet %s", vm.Name) - break - } - } - if util.ResourceVersionAtLeast(vmFromLister.ResourceVersion, vm.ResourceVersion) { - return nil - } - time.Sleep(100 * time.Millisecond) - } - return nil + return err } func (v *VirtualMachineSetController) updateVMSetCount(vmSetName string, active int, prov int) error { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachineSets().Get(v.ctx, vmSetName, metav1.GetOptions{}) if getErr != nil { - return fmt.Errorf("Error retrieving latest version of Virtual Machine Set %s: %v", vmSetName, getErr) + return fmt.Errorf("error retrieving latest version of Virtual Machine Set %s: %v", vmSetName, getErr) } result.Status.ProvisionedCount = prov @@ -413,8 +384,41 @@ func (v *VirtualMachineSetController) updateVMSetCount(vmSetName string, active return nil }) if retryErr != nil { - return fmt.Errorf("Error updating Virtual Machine Set: %s, %v", vmSetName, retryErr) + return fmt.Errorf("error updating Virtual Machine Set: %s, %v", vmSetName, retryErr) } return nil } + +func (v *VirtualMachineSetController) removeVMFinalizer(vm *hfv1.VirtualMachine) (err error) { + if ContainsFinalizer(vm, vmSetFinalizer) { + RemoveFinalizer(vm, vmSetFinalizer) + _, err = v.hfClientSet.HobbyfarmV1().VirtualMachines().Update(v.ctx, vm, metav1.UpdateOptions{}) + } + return err +} + +// From ControllerUtil to save dep issues + +// RemoveFinalizer accepts an Object and removes the provided finalizer if present. +func RemoveFinalizer(vm *hfv1.VirtualMachine, finalizer string) { + f := vm.GetFinalizers() + for i := 0; i < len(f); i++ { + if f[i] == finalizer { + f = append(f[:i], f[i+1:]...) + i-- + } + } + vm.SetFinalizers(f) +} + +// ContainsFinalizer checks an Object that the provided finalizer is present. +func ContainsFinalizer(vm *hfv1.VirtualMachine, finalizer string) bool { + f := vm.GetFinalizers() + for _, e := range f { + if e == finalizer { + return true + } + } + return false +} diff --git a/pkg/sessionserver/sessionserver.go b/pkg/sessionserver/sessionserver.go index de329bea..b2d857d9 100644 --- a/pkg/sessionserver/sessionserver.go +++ b/pkg/sessionserver/sessionserver.go @@ -30,6 +30,7 @@ const ( vmcSessionLabel = "hobbyfarm.io/session" UserSessionLabel = "hobbyfarm.io/user" AccessCodeLabel = "accesscode.hobbyfarm.io" + ScheduledEventLabel = "hobbyfarm.io/scheduledevent" ) type SessionServer struct { @@ -213,6 +214,26 @@ func (sss SessionServer) NewSessionFunc(w http.ResponseWriter, r *http.Request) vms = scenario.Spec.VirtualMachines } + // find bindMode by quering the scheduledEvent + owners := accessCodeObj.GetOwnerReferences() + if len(owners) != 1 { + util.ReturnHTTPMessage(w, r, 500, "error", "access code has multiple owners.. invalid request") + return + } + + schedEvent, err := sss.hfClientSet.HobbyfarmV1().ScheduledEvents().Get(sss.ctx, owners[0].Name, metav1.GetOptions{}) + if err != nil { + util.ReturnHTTPMessage(w, r, 500, "error", "unable to find scheduledEvent") + return + } + + var bindMode string + if schedEvent.Spec.OnDemand { + bindMode = "dynamic" + } else { + bindMode = "static" + } + session.Spec.VmClaimSet = make([]string, len(vms)) for index, vmset := range vms { virtualMachineClaim := hfv1.VirtualMachineClaim{} @@ -221,6 +242,7 @@ func (sss SessionServer) NewSessionFunc(w http.ResponseWriter, r *http.Request) labels[vmcSessionLabel] = session.Name // map vmc to session labels[UserSessionLabel] = user.Spec.Id // map session to user in a way that is searchable labels[AccessCodeLabel] = session.Labels[AccessCodeLabel] + labels[ScheduledEventLabel] = schedEvent.Name virtualMachineClaim.Labels = labels virtualMachineClaim.Spec.Id = vmcId virtualMachineClaim.Name = vmcId @@ -233,6 +255,7 @@ func (sss SessionServer) NewSessionFunc(w http.ResponseWriter, r *http.Request) virtualMachineClaim.Spec.UserId = user.Spec.Id virtualMachineClaim.Status.Bound = false virtualMachineClaim.Status.Ready = false + virtualMachineClaim.Status.BindMode = bindMode virtualMachineClaim.Spec.DynamicCapable = true if restrictedBind { From 1f5a21b1c709fe063781f347ee6d5e87c559bd69 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Fri, 1 Oct 2021 15:47:35 +1000 Subject: [PATCH 09/11] gofmt/govet --- .../vmclaimcontroller/vmclaimcontroller.go | 4 ++-- pkg/sessionserver/sessionserver.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index d779f9d3..b33e3c17 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -266,7 +266,7 @@ func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err e return err } } else { - glog.Error("vmc bind mode needs to be either dynamic or static.. ignoring this object %s", vmc.Name) + glog.Errorf("vmc bind mode needs to be either dynamic or static.. ignoring this object %s", vmc.Name) return nil } @@ -503,7 +503,7 @@ func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, temp } vms, err := v.vmLister.List(vmLabels.AsSelector()) - glog.V(4).Info("found %d vm's matching this requirement", len(vms)) + glog.V(4).Infof("found %d vm's matching this requirement", len(vms)) if err != nil { return "", fmt.Errorf("error while listing all vms %v", err) } diff --git a/pkg/sessionserver/sessionserver.go b/pkg/sessionserver/sessionserver.go index b2d857d9..5a277594 100644 --- a/pkg/sessionserver/sessionserver.go +++ b/pkg/sessionserver/sessionserver.go @@ -23,13 +23,13 @@ import ( ) const ( - ssIndex = "sss.hobbyfarm.io/session-id-index" - newSSTimeout = "5m" - keepaliveSSTimeout = "5m" - pauseSSTimeout = "2h" - vmcSessionLabel = "hobbyfarm.io/session" - UserSessionLabel = "hobbyfarm.io/user" - AccessCodeLabel = "accesscode.hobbyfarm.io" + ssIndex = "sss.hobbyfarm.io/session-id-index" + newSSTimeout = "5m" + keepaliveSSTimeout = "5m" + pauseSSTimeout = "2h" + vmcSessionLabel = "hobbyfarm.io/session" + UserSessionLabel = "hobbyfarm.io/user" + AccessCodeLabel = "accesscode.hobbyfarm.io" ScheduledEventLabel = "hobbyfarm.io/scheduledevent" ) From aef1e66f2c090478bd23528a4b02417d3e9596c4 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Sat, 2 Oct 2021 12:46:35 +1000 Subject: [PATCH 10/11] fixed order of vmset reconcile to ensure vmpool is properly calculated --- pkg/controllers/vmsetcontroller/vmsetcontroller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/vmsetcontroller/vmsetcontroller.go b/pkg/controllers/vmsetcontroller/vmsetcontroller.go index 90c7af8a..55ca365e 100644 --- a/pkg/controllers/vmsetcontroller/vmsetcontroller.go +++ b/pkg/controllers/vmsetcontroller/vmsetcontroller.go @@ -172,8 +172,12 @@ func (v *VirtualMachineSetController) processNextVM() bool { // this should avoid triggering unwanted reconciles of VMClaims until the VM's are running if !vm.DeletionTimestamp.IsZero() { glog.V(4).Infof("requeuing vmset %s to account for tainted vm %s", vm.Spec.VirtualMachineSetId, vm.Name) + err = v.removeVMFinalizer(vm) + if err != nil { + glog.Errorf("error removing vm finalizer on vm %s", vm.Name) + return err + } defer v.vmSetWorkqueue.Add(vm.Spec.VirtualMachineSetId) - return v.removeVMFinalizer(vm) } return nil From 93dc54970d64dad4f3bc18ba4642f6b73cbc37f3 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Mon, 4 Oct 2021 19:35:22 +1100 Subject: [PATCH 11/11] minor fix to scheduledevent controller to handle edit --- .../scheduledeventcontroller.go | 96 ++++++++++++------- 1 file changed, 63 insertions(+), 33 deletions(-) diff --git a/pkg/controllers/scheduledevent/scheduledeventcontroller.go b/pkg/controllers/scheduledevent/scheduledeventcontroller.go index c1f20731..515fa03f 100644 --- a/pkg/controllers/scheduledevent/scheduledeventcontroller.go +++ b/pkg/controllers/scheduledevent/scheduledeventcontroller.go @@ -250,7 +250,10 @@ func (s ScheduledEventController) provisionScheduledEvent(templates *hfv1.Virtua for envName, vmtMap := range se.Spec.RequiredVirtualMachines { // get the environment we're provisioning into (envName) env, err := s.hfClientSet.HobbyfarmV1().Environments().Get(s.ctx, envName, metav1.GetOptions{}) - + if err != nil { + glog.Errorf("error retreiving environment %s", err.Error()) + return err + } // get all vmsets that are being provisioned into this environment (label selector) vmsList, err := s.hfClientSet.HobbyfarmV1().VirtualMachineSets().List(s.ctx, metav1.ListOptions{ LabelSelector: fmt.Sprintf("environment=%s", envName), @@ -281,43 +284,70 @@ func (s ScheduledEventController) provisionScheduledEvent(templates *hfv1.Virtua for templateName, count := range vmtMap { if count > 0 && !se.Spec.OnDemand { // only setup vmsets if >0 VMs are requested, and they aren't ondemand - vmsRand := fmt.Sprintf("%s-%08x", baseNameScheduledPrefix, rand.Uint32()) - vmsName := strings.Join([]string{"se", se.Name, "vms", vmsRand}, "-") - vmSets = append(vmSets, vmsName) - vms := &hfv1.VirtualMachineSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: vmsName, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "hobbyfarm.io/v1", - Kind: "ScheduledEvent", - Name: se.Name, - UID: se.UID, + if len(se.Status.VirtualMachineSets) == 0 { + vmsRand := fmt.Sprintf("%s-%08x", baseNameScheduledPrefix, rand.Uint32()) + vmsName := strings.Join([]string{"se", se.Name, "vms", vmsRand}, "-") + vmSets = append(vmSets, vmsName) + vms := &hfv1.VirtualMachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmsName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "hobbyfarm.io/v1", + Kind: "ScheduledEvent", + Name: se.Name, + UID: se.UID, + }, + }, + Labels: map[string]string{ + "environment": env.Name, + "scheduledevent": se.Name, + fmt.Sprintf("virtualmachinetemplate.hobbyfarm.io/%s", templateName): "true", }, }, - Labels: map[string]string{ - "environment": env.Name, - "scheduledevent": se.Name, - fmt.Sprintf("virtualmachinetemplate.hobbyfarm.io/%s", templateName): "true", + Spec: hfv1.VirtualMachineSetSpec{ + Count: count, + Environment: envName, + VMTemplate: templateName, + BaseName: vmsRand, }, - }, - Spec: hfv1.VirtualMachineSetSpec{ - Count: count, - Environment: envName, - VMTemplate: templateName, - BaseName: vmsRand, - }, - } - if se.Spec.RestrictedBind { - vms.Spec.RestrictedBind = true - vms.Spec.RestrictedBindValue = se.Spec.RestrictedBindValue + } + if se.Spec.RestrictedBind { + vms.Spec.RestrictedBind = true + vms.Spec.RestrictedBindValue = se.Spec.RestrictedBindValue + } else { + vms.Spec.RestrictedBind = false + } + _, err = s.hfClientSet.HobbyfarmV1().VirtualMachineSets().Create(s.ctx, vms, metav1.CreateOptions{}) + if err != nil { + glog.Error(err) + return err + } } else { - vms.Spec.RestrictedBind = false - } - vms, err := s.hfClientSet.HobbyfarmV1().VirtualMachineSets().Create(s.ctx, vms, metav1.CreateOptions{}) - if err != nil { - glog.Error(err) + vmSetName := se.Status.VirtualMachineSets[0] + vmSetOrg, err := s.hfClientSet.HobbyfarmV1().VirtualMachineSets().Get(s.ctx, vmSetName, metav1.GetOptions{}) + if err != nil { + glog.Error(err) + return err + } + + vmSetOrg.Labels["environment"] = env.Name + vmSetOrg.Spec.Count = count + vmSetOrg.Spec.Environment = envName + vmSetOrg.Spec.VMTemplate = templateName + if se.Spec.RestrictedBind { + vmSetOrg.Spec.RestrictedBind = true + + } else { + vmSetOrg.Spec.RestrictedBind = false + } + _, err = s.hfClientSet.HobbyfarmV1().VirtualMachineSets().Update(s.ctx, vmSetOrg, metav1.UpdateOptions{}) + if err != nil { + glog.Errorf("error updating vmset config %s", err.Error()) + return err + } } + } }