Add support for virtualCleanup decorator (#400)

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.1.0
+current_version = 3.2.0.dev0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?

README-dev.md

@@ -99,6 +99,8 @@ If you want to bump the major/minor/patch version, run `bumpversion [major|minor
 
 If you want to get rid of the dev label (bump from `1.1.0.dev7` to `1.1.0`), run `bumpversion release`.
 
+Note: After bumpversion the tools unit test will need to be manually updated to test for the new version.
+
 ## Testing
 
 Currently, there are three types of SDK testing: unit, manual, and functional (blackbox).
@@ -122,7 +124,9 @@ all the standard workflows. The same workflows will be exercised by functional (
 ### Functional (blackbox) testing
 To run blackbox tests, follow these steps: 
 1. Push your code to a branch in the forked repository on Github. Let's say the branch is called `my-feature` in repository called `<username>/virtualization-sdk`.
-2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use,
+2. If you bumped the version (one of major, minor, or micro, not the dev version part), then QA will have to createa a new branch (qa-appdata-toolkits branch sdk-3-2-0 for example with version 3.2.0) and update their map before you can run the blackbox tests:
+* automation/regression/BlackBox/blackbox/appdata/virtualization_sdk/dvp_settings.py
+3. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use,
 see the next sections.
 
 At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin.

common/setup.py

@@ -4,7 +4,7 @@
 PYTHON_SRC = 'src/main/python'
 
 install_requires = [
- "dvp-api == 1.5.0",
+ "dvp-api == 1.6.0dev2",
 ]
 
 with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:

common/src/main/python/dlpx/virtualization/common/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

docs/docs/References/Decorators.md

@@ -38,6 +38,7 @@ Plugin Operation | Decorator
 [Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()`
 [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()`
 [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()`
+[Virtual Source Cleanup](Plugin_Operations.md#virtual-source-cleanup) | `@plugin.virtual.cleanup()`
 [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()`
 [Virtual Source Stop](Plugin_Operations.md#virtual-source-stop) | `@plugin.virtual.stop()`
 [VirtualSource Pre-Snapshot](Plugin_Operations.md#virtualsource-pre-snapshot) | `@plugin.virtual.pre_snapshot()`

docs/docs/References/Plugin_Operations.md

@@ -25,6 +25,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations
 [Virtual Source<br/>Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)<br/>[Virtual Source Refresh](Workflows.md#virtual-source-refresh)
 [Virtual Source<br/>Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)<br/>[Virtual Source Delete](Workflows.md#virtual-source-delete)
 [Virtual Source<br/>Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)<br/>[Virtual Source Enable](Workflows.md#virtual-source-enable)
+[Virtual Source<br/>Cleanup](#virtual-source-cleanup) | **No** | `virtual.cleanup()` | [Virtual Source Delete](Workflows.md#virtual-source-delete)
 [Virtual Source<br/>Start](#virtual-source-start) | **No** | `virtual.start()` | [Virtual Source Start](Workflows.md#virtual-source-start)
 [Virtual Source<br/>Stop](#virtual-source-stop) | **No** | `virtual.stop()` | [Virtual Source Stop](Workflows.md#virtual-source-stop)
 [Virtual Source<br/>Pre-Snapshot](#virtual-source-pre-snapshot) | **No** | `virtual.pre_snapshot()` | [Virtual Source Snapshot](Workflows.md#virtual-source-snapshot)
@@ -823,6 +824,50 @@ def reconfigure(virtual_source, repository, source_config, snapshot):
 }
 ```
 
+## Virtual Source Cleanup
+
+Intended to allow a final cleanup during a delete operation, unlike unconfigure which can be used to signal a temporary dissassociation with a database.
+
+Cleanup is called during the delete flow after unconfigure.
+
+### Required / Optional
+**Optional.**
+
+### Delphix Engine Operations
+
+* [Virtual Source Delete](Workflows.md#virtual-source-delete)
+
+### Signature
+
+`def cleanup(virtual_source, repository, source_config)`
+
+### Decorator
+
+`virtual.cleanup()`
+
+### Arguments
+
+Argument | Type | Description
+-------- | ---- | -----------
+virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation.
+repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source.
+source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source.
+
+### Returns
+None
+
+### Example
+
+```python
+from dlpx.virtualization.platform import Plugin
+
+plugin = Plugin()
+
+@plugin.virtual.cleanup()
+def cleanup(virtual_source, repository, source_config):
+ pass
+```
+
 ## Virtual Source Start
 
 Executed whenever the data should be placed in a "running" state.

dvp/src/main/python/dlpx/virtualization/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

libs/setup.py

@@ -7,7 +7,7 @@
  version = version_file.read().strip()
 
 install_requires = [
- "dvp-api == 1.5.0",
+ "dvp-api == 1.6.0dev2",
  "dvp-common == {}".format(version)
 ]
 

libs/src/main/python/dlpx/virtualization/libs/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

platform/setup.py

@@ -7,7 +7,7 @@
  version = version_file.read().strip()
 
 install_requires = [
- "dvp-api == 1.5.0",
+ "dvp-api == 1.6.0dev2",
  "dvp-common == {}".format(version),
  "enum34;python_version < '3.4'",
 ]

platform/src/main/python/dlpx/virtualization/platform/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

platform/src/main/python/dlpx/virtualization/platform/_virtual.py

@@ -25,6 +25,7 @@ class VirtualOperations(object):
  def __init__(self):
  self.configure_impl = None
  self.unconfigure_impl = None
+ self.cleanup_impl = None
  self.reconfigure_impl = None
  self.start_impl = None
  self.stop_impl = None
@@ -54,6 +55,16 @@ def unconfigure_decorator(unconfigure_impl):
 
  return unconfigure_decorator
 
+ def cleanup(self):
+ def cleanup_decorator(cleanup_impl):
+ if self.cleanup_impl:
+ raise OperationAlreadyDefinedError(Op.VIRTUAL_CLEANUP)
+ self.cleanup_impl = v.check_function(cleanup_impl,
+ Op.VIRTUAL_CLEANUP)
+ return cleanup_impl
+
+ return cleanup_decorator
+
  def reconfigure(self):
  def reconfigure_decorator(reconfigure_impl):
  if self.reconfigure_impl:
@@ -255,6 +266,58 @@ def _internal_unconfigure(self, request):
  platform_pb2.UnconfigureResult())
  return unconfigure_response
 
+ def _internal_cleanup(self, request):
+ """Cleanup operation wrapper.
+
+ Executed when deleting an existing virtual source.
+ This plugin operation is run after unconfigure.
+
+ Args:
+ request (VirtualCleanupRequest): Cleanup operation arguments.
+
+ Returns:
+ VirtualCleanupResponse: A response containing VirtualCleanupResult
+ if successful or PluginErrorResult in case of an error.
+ """
+ # Reasoning for method imports are in this file's docstring.
+ from generated.definitions import VirtualSourceDefinition
+ from generated.definitions import RepositoryDefinition
+ from generated.definitions import SourceConfigDefinition
+
+ #
+ # While virtual.cleanup() is not a required operation, this should
+ # not be called if it wasn't implemented.
+ #
+ if not self.cleanup_impl:
+ raise OperationNotDefinedError(Op.VIRTUAL_CLEANUP)
+
+ virtual_source_definition = VirtualSourceDefinition.from_dict(
+ json.loads(request.virtual_source.parameters.json))
+ mounts = [
+ VirtualOperations._from_protobuf_single_subset_mount(m)
+ for m in request.virtual_source.mounts
+ ]
+
+ virtual_source = VirtualSource(guid=request.virtual_source.guid,
+ connection=RemoteConnection.from_proto(
+ request.virtual_source.connection),
+ parameters=virtual_source_definition,
+ mounts=mounts)
+
+ repository = RepositoryDefinition.from_dict(
+ json.loads(request.repository.parameters.json))
+ source_config = SourceConfigDefinition.from_dict(
+ json.loads(request.source_config.parameters.json))
+
+ self.cleanup_impl(repository=repository,
+ source_config=source_config,
+ virtual_source=virtual_source)
+
+ virtual_cleanup_response = platform_pb2.VirtualCleanupResponse()
+ virtual_cleanup_response.return_value.CopyFrom(
+ platform_pb2.VirtualCleanupResult())
+ return virtual_cleanup_response
+
  def _internal_reconfigure(self, request):
  """Reconfigure operation wrapper.
 

platform/src/main/python/dlpx/virtualization/platform/operation.py

@@ -21,6 +21,7 @@ class Operation(Enum):
  VIRTUAL_CONFIGURE = 'virtual.configure()'
  VIRTUAL_UNCONFIGURE = 'virtual.unconfigure()'
  VIRTUAL_RECONFIGURE = 'virtual.reconfigure()'
+ VIRTUAL_CLEANUP = 'virtual.cleanup()'
  VIRTUAL_START = 'virtual.start()'
  VIRTUAL_STOP = 'virtual.stop()'
  VIRTUAL_PRE_SNAPSHOT = 'virtual.pre_snapshot()'

platform/src/test/python/dlpx/virtualization/test_plugin.py

@@ -559,6 +559,32 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config,
  " type 'unicode' but should be of class 'dlpx.virtualization."
  "fake_generated_definitions.SourceConfigDefinition'.")
 
+ @staticmethod
+ def test_virtual_cleanup(my_plugin, virtual_source, repository,
+ source_config):
+ @my_plugin.virtual.cleanup()
+ def virtual_cleanup_impl(virtual_source, repository,
+ source_config):
+ TestPlugin.assert_plugin_args(virtual_source=virtual_source,
+ repository=repository,
+ source_config=source_config)
+ return
+
+ virtual_cleanup_request = platform_pb2.VirtualCleanupRequest()
+ TestPlugin.setup_request(request=virtual_cleanup_request,
+ virtual_source=virtual_source,
+ repository=repository,
+ source_config=source_config)
+
+ expected_result = platform_pb2.VirtualCleanupResult()
+
+ virtual_cleanup_response = my_plugin.virtual._internal_cleanup(
+ virtual_cleanup_request)
+
+ # Check that the response's oneof is set to return_value and not error
+ assert virtual_cleanup_response.WhichOneof('result') == 'return_value'
+ assert virtual_cleanup_response.return_value == expected_result
+
  @staticmethod
  def test_virtual_start(my_plugin, virtual_source, repository,
  source_config):

tools/src/main/python/dlpx/virtualization/_internal/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py

@@ -330,6 +330,8 @@ def _prepare_manifest(entry_point, module_content):
  bool(plugin_object.virtual.unconfigure_impl),
  'hasVirtualReconfigure':
  bool(plugin_object.virtual.reconfigure_impl),
+ 'hasVirtualCleanup':
+ bool(plugin_object.virtual.cleanup_impl),
  'hasVirtualStart':
  bool(plugin_object.virtual.start_impl),
  'hasVirtualStop':

tools/src/main/python/dlpx/virtualization/_internal/settings.cfg

@@ -20,7 +20,7 @@
 # versions in those packages until they are shipped out of band.
 #
 [General]
-engine_api_version = 1.11.6
+engine_api_version = 1.11.11
 distribution_name = dvp-tools
 package_author = Delphix
 namespace_package = dlpx

tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml

@@ -52,6 +52,10 @@ EXPECTED_STAGED_ARGS_BY_OP:
  - repository
  - source_config
  - snapshot
+ cleanup_impl:
+ - virtual_source
+ - repository
+ - source_config
  start_impl:
  - virtual_source
  - repository
@@ -112,6 +116,10 @@ EXPECTED_DIRECT_ARGS_BY_OP:
  - repository
  - source_config
  - snapshot
+ cleanup_impl:
+ - virtual_source
+ - repository
+ - source_config
  start_impl:
  - virtual_source
  - repository

tools/src/test/python/dlpx/virtualization/_internal/conftest.py

@@ -404,6 +404,7 @@ def mount_specification(virtual_source, repository):
  virtual.mount_specification_impl = mount_specification
  virtual.status_impl = None
  virtual.initialize_impl = None
+ virtual.cleanup_impl = None
 
  return virtual
 
@@ -447,7 +448,8 @@ def plugin_manifest(upgrade_operation):
  'hasVirtualMountSpecification': True,
  'hasVirtualStatus': False,
  'hasInitialize': False,
- 'migrationIdList': upgrade_operation.migration_id_list
+ 'migrationIdList': upgrade_operation.migration_id_list,
+ 'hasVirtualCleanup': False,
  }
  return manifest
 
@@ -691,7 +693,7 @@ def artifact_content(engine_api, virtual_source_definition,
 
 @pytest.fixture
 def engine_api():
- return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 6}
+ return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 11}
 
 
 @pytest.fixture

tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py

@@ -75,6 +75,11 @@ def unconfigure(repository, source_config, virtual_source):
  pass
 
 
+@vfiles.virtual.cleanup()
+def cleanup(repository, source_config, virtual_source, bad_arg):
+ pass
+
+
 @vfiles.upgrade.repository('2019.10.30')
 def repo_upgrade(old_repository):
  return old_repository

tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py

@@ -81,6 +81,11 @@ def unconfigure(repository, source_config, virtual_source):
  pass
 
 
+@direct.virtual.cleanup()
+def cleanup(repository, source_config, virtual_source):
+ pass
+
+
 @direct.upgrade.repository('1.3', MigrationType.LUA)
 def repo_upgrade(old_repository):
  return old_repository

tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py

@@ -74,6 +74,11 @@ def unconfigure(repository, source_config, virtual_source):
  pass
 
 
+@direct.virtual.cleanup()
+def cleanup(repository, source_config, virtual_source):
+ pass
+
+
 @direct.upgrade.repository('2019.11.20')
 def repo_upgrade(old_repository):
  return old_repository

tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py

@@ -10,22 +10,22 @@
 class TestPackageUtil:
  @staticmethod
  def test_get_version():
- assert package_util.get_version() == '3.1.0'
+ assert package_util.get_version() == '3.2.0.dev0'
 
  @staticmethod
  def test_get_virtualization_api_version():
- assert package_util.get_virtualization_api_version() == '1.5.0'
+ assert package_util.get_virtualization_api_version() == '1.6.0'
 
  @staticmethod
  def test_get_engine_api_version():
- assert package_util.get_engine_api_version_from_settings() == '1.11.6'
+ assert package_util.get_engine_api_version_from_settings() == '1.11.11'
 
  @staticmethod
  def test_get_build_api_version_json():
  build_api_version = {
  'type': 'APIVersion',
  'major': 1,
- 'minor': 5,
+ 'minor': 6,
  'micro': 0
  }
  assert package_util.get_build_api_version() == build_api_version
@@ -36,7 +36,7 @@ def test_get_engine_api_version_json():
  'type': 'APIVersion',
  'major': 1,
  'minor': 11,
- 'micro': 6
+ 'micro': 11
  }
  assert package_util.get_engine_api_version() == engine_api_version