Add finalizer for cleanup handling

Finalizer will address an issue with cleanup when the operator is down during deletion

Catch some notFound exections when deleting gone vSphere objects

Upgrade to kopf v0.22 to avoid [404s](zalando-incubator/kopf#160)

controller.py

@@ -22,9 +22,12 @@
 CLUSTER = "cls"
 DATASTORE = "sharedVmfs-0"
 
+# finalizers
+FINALIZER = 'vsphere.vmware.com/kopf-vm-operator'
+
 # register kopf handlers
 @kopf.on.event('vsphere.vmware.com', 'v1alpha1', 'vmgroups')
-def vm_operator(event, spec, meta, status, logger, **_):
+def vm_operator(event, body, spec, meta, status, patch, logger, **_):
     # poor man's throttle: give vcenter operations some time to catch up
     sleep(3)
 
@@ -33,15 +36,30 @@ def vm_operator(event, spec, meta, status, logger, **_):
     desired_replicas = spec.get('replicas')
     now = strftime("%Y-%m-%d %H:%M:%S", gmtime())
 
+    finalizers = body.get('metadata', {}).get('finalizers', [])
+    marked_deleted = body.get('metadata', {}).get('deletionTimestamp', None)
+
     event_type = event['type']
-    if event_type == "DELETED":
+    if event_type == "DELETED" or marked_deleted:
         logger.info(f'deleting vm group "{vmgroup}"')
-        # TODO: if something goes wrong deleting the folder/VMs, we only log the exception it in the function since the custom resource in Kubernetes is gone
-        # needs garbage collection routine for vCenter objects
-        delete_vm_group(vmgroup, logger)
+        exists = vm_group_exists(vmgroup)
+        if exists:
+            # TODO: if something goes wrong deleting the folder/VMs, we only log the exception it in the function since the custom resource in Kubernetes is gone
+            # needs garbage collection routine for vCenter objects
+            delete_vm_group(vmgroup, logger)
+        else:
+            logger.info(f'vm group already gone "{vmgroup}')
+        if FINALIZER in finalizers:
+            logger.info(f'removing finalizer "{FINALIZER}"')
+            patch.setdefault('metadata', {}).setdefault('finalizers', list(finalizers))
+            patch['metadata']['finalizers'].remove(FINALIZER)
         return
 
     # ADDED/MODIFIED event
+    if not FINALIZER in finalizers:
+        logger.info(f'adding finalizer "{FINALIZER}"')
+        patch.setdefault('metadata', {}).setdefault('finalizers', list(finalizers))
+        patch['metadata']['finalizers'].append(FINALIZER)
     try:
         phase = status['vm_operator']['phase']
     # if we don't find a status, initialize the object and set its state to "PENDING"

requirements.txt

@@ -1,4 +1,4 @@
-kopf==0.20
+kopf==0.22
 pyVmomi==6.7.1.2018.12
 pydng==1.1.2
 pyvim==2.0.23

vsphere/vsphere.py

@@ -6,7 +6,7 @@
 from time import sleep
 import logging
 from pyVim.connect import SmartConnect, SmartConnectNoSSL
-from pyVmomi import vim
+from pyVmomi import vim, vmodl
 import pydng
 
 class Error(Exception):
@@ -218,13 +218,18 @@ def delete_folder(content, dc: vim.Datacenter, folder_name: str):
         folder.Destroy_Task()
     except vim.fault.VimFault as f:
         raise DestroyError(f'could not delete folder "{folder_name}": {str(f)}')
+    except vmodl.fault.ManagedObjectNotFound:
+        pass
 
 def vm_group_exists(content, dc: vim.Datacenter, vmgroup_name: str) -> bool:
     """
     vm_group_exists checks whether a given VM group "vmgroup_name" already exists in the inventory in datacenter "dc"
     """
-    obj = get_obj(content, dc, [vim.Folder], vmgroup_name)
-    if obj is None:
-        return False
-    else:
-        return True
+    try:
+        obj = get_obj(content, dc, [vim.Folder], vmgroup_name)
+        if obj is None:
+            return False
+        else:
+            return True
+    except vmodl.fault.ManagedObjectNotFound:
+        return False