Merge pull request #2668 from ClusterHQ/plugin-loading-FLOC-4245

[FLOC-4245] Factor out plugin loading.

flocker/acceptance/testtools.py

@@ -48,7 +48,8 @@
 from ..ca import treq_with_authentication, UserCredential
 from ..testtools import random_name
 from ..apiclient import FlockerClient, DatasetState
-from ..node.script import get_backend, get_api
+from ..node.backends import backend_loader
+from ..node.script import get_api
 from ..node import dockerpy_client
 from ..node.agents.blockdevice import _SyncToThreadedAsyncAPIAdapter
 from ..provision import reinstall_flocker_from_package_source
@@ -247,7 +248,7 @@ def get_backend_api(cluster_id):
     backend_config = full_backend_config.get(backend_name)
     if 'backend' in backend_config:
         backend_config.pop('backend')
-    backend = get_backend(backend_name)
+    backend = backend_loader.get(backend_name)
     return get_api(backend, pmap(backend_config), reactor, cluster_id)
 
 

flocker/common/plugin.py

@@ -0,0 +1,135 @@
+# Copyright ClusterHQ Inc.  See LICENSE file for details.
+
+"""
+Tools for loading third-party plugins.
+"""
+
+from characteristic import attributes
+from pyrsistent import PClass, field, PVector, pvector
+from twisted.python.reflect import namedAny
+
+
+@attributes(["plugin_name"])
+class PluginNotFound(Exception):
+    """
+    A plugin with the given name was not found.
+
+    :attr str plugin_name: Name of the plugin looked for.
+    """
+    def __str__(self):
+        return (
+            "'{!s}' is neither a built-in plugin nor a 3rd party "
+            "module.".format(self.plugin_name)
+        )
+
+
+class InvalidPlugin(Exception):
+    """
+    A module with the given plugin name was found, but doesn't
+    provide a valid flocker plugin.
+    """
+
+
+@attributes(["plugin_name", "module_attribute"])
+class MissingPluginAttribute(InvalidPlugin):
+    """
+    The named module doesn't have the attribute expected of plugins.
+    """
+    def __str__(self):
+        return (
+            "The 3rd party plugin '{plugin_name!s}' does not "
+            "correspond to the expected interface. "
+            "`{plugin_name!s}.{module_attribute!s}` is not defined."
+            .format(
+                plugin_name=self.plugin_name,
+                module_attribute=self.module_attribute,
+            )
+        )
+
+
+@attributes(["plugin_name", "plugin_type", "actual_type", "module_attribute"])
+class InvalidPluginType(InvalidPlugin):
+    """
+    A plugin with the given name was not found.
+    """
+    def __str__(self):
+        return (
+            "The 3rd party plugin '{plugin_name!s}' does not "
+            "correspond to the expected interface. "
+            "`{plugin_name!s}.{module_attribute!s}` is of "
+            "type `{actual_type.__name__}`, not `{plugin_type.__name__}`."
+            .format(
+                plugin_name=self.plugin_name,
+                actual_type=self.actual_type,
+                plugin_type=self.plugin_type,
+                module_attribute=self.module_attribute,
+            )
+        )
+
+
+class PluginLoader(PClass):
+    """
+    :ivar PVector builtin_plugins: The plugins shipped with flocker.
+    :ivar str module_attribute: The module attribute that third-party plugins
+        should declare.
+    :ivar type plugin_type: The type describing a plugin.
+    """
+    builtin_plugins = field(PVector, mandatory=True, factory=pvector)
+    module_attribute = field(str, mandatory=True)
+    plugin_type = field(type, mandatory=True)
+
+    def __invariant__(self):
+        for builtin in self.builtin_plugins:
+            if not isinstance(builtin, self.plugin_type):
+                return (
+                    False,
+                    "Builtin plugins must be of `{plugin_type.__name__}`, not "
+                    "`{actual_type.__name__}`.".format(
+                        plugin_type=self.plugin_type,
+                        actual_type=type(builtin),
+                    )
+                )
+
+        return (True, "")
+
+    def get(self, plugin_name):
+        """
+        Find the plugin in ``builtin_plugins`` that matches the one named by
+        ``plugin_name``. If not found then an attempt is made to load it as
+        module describing a plugin.
+
+        :param plugin_name: The name of the backend.
+        :param backends: Collection of `BackendDescription`` instances.
+
+        :raise PluginNotFound: If ``plugin_name`` doesn't match any
+            known plugin.
+        :raise InvalidPlugin: If ``plugin_name`` names a module that
+            doesn't satisfy the plugin interface.
+        :return: The matching ``plugin_type`` instance.
+        """
+        for builtin in self.builtin_plugins:
+            if builtin.name == plugin_name:
+                return builtin
+
+        try:
+            plugin_module = namedAny(plugin_name)
+        except (AttributeError, ValueError):
+            raise PluginNotFound(plugin_name=plugin_name)
+
+        try:
+            plugin = getattr(plugin_module, self.module_attribute)
+        except AttributeError:
+            raise MissingPluginAttribute(
+                plugin_name=plugin_name,
+                module_attribute=self.module_attribute,
+            )
+
+        if not isinstance(plugin, self.plugin_type):
+            raise InvalidPluginType(
+                plugin_name=plugin_name,
+                plugin_type=self.plugin_type,
+                actual_type=type(plugin),
+                module_attribute=self.module_attribute,
+            )
+
+        return plugin

flocker/common/test/test_plugin.py

@@ -0,0 +1,119 @@
+# Copyright ClusterHQ Inc.  See LICENSE file for details.
+
+"""
+Tests for ``flocker.common.plugin``
+"""
+
+from pyrsistent import PClass, field
+
+from flocker.testtools import TestCase
+
+from ..plugin import (
+    PluginLoader,
+    PluginNotFound,
+    MissingPluginAttribute,
+    InvalidPluginType,
+)
+
+
+class DummyDescription(PClass):
+    """
+    Dummy plugin type."
+    """
+    name = field(unicode, mandatory=True)
+
+
+# The following test examples use classes instead of modules for
+# namespacing. Real plugins should use modules.
+class DummyPlugin(object):
+    """
+    A plugin."
+    """
+    FLOCKER_PLUGIN = DummyDescription(
+        name=u"dummyplugin",
+    )
+
+
+class DummyPluginMissingAttribute(object):
+    """
+    A purported plugin that is missing the expected attribute.
+    """
+
+
+class DummyPluginWrongType(object):
+    """
+    A purported plugin that has the wrong type of description.
+    """
+    FLOCKER_PLUGIN = object()
+
+
+DUMMY_LOADER = PluginLoader(
+    builtin_plugins=[],
+    module_attribute="FLOCKER_PLUGIN",
+    plugin_type=DummyDescription,
+)
+
+
+class PluginLoaderTests(TestCase):
+    """
+    Tests for ``PluginLoader``.
+    """
+
+    def test_builtin_backend(self):
+        """
+        If the plugin name is that of a pre-configured plugin, the
+        corresponding builtin plugin is returned.
+        """
+        loader = DUMMY_LOADER.set(
+            "builtin_plugins", [
+                DummyDescription(name=u"other-builtin"),
+                DummyDescription(name=u"builtin"),
+            ]
+        )
+        plugin = loader.get("builtin")
+        self.assertEqual(plugin, DummyDescription(name=u"builtin"))
+
+    def test_3rd_party_plugin(self):
+        """
+        If the plugin name is not that of a pre-configured plugin, the
+        plugin name is treated as a Python import path, and the
+        specified attribute of that is used as the plugin.
+        """
+        plugin = DUMMY_LOADER.get(
+            "flocker.common.test.test_plugin.DummyPlugin"
+        )
+        self.assertEqual(plugin, DummyDescription(name=u"dummyplugin"))
+
+    def test_wrong_package_3rd_party_backend(self):
+        """
+        If the plugin name refers to an unimportable package,
+        ``PluginNotFound`` is raised.
+        """
+        self.assertRaises(
+            PluginNotFound,
+            DUMMY_LOADER.get,
+            "notarealmoduleireallyhope",
+        )
+
+    def test_missing_attribute_3rd_party_backend(self):
+        """
+        If the plugin name refers to an object that doesn't have the
+        specified attribute, ``MissingPluginAttribute`` is raised.
+        """
+        self.assertRaises(
+            MissingPluginAttribute,
+            DUMMY_LOADER.get,
+            "flocker.common.test.test_plugin.DummyPluginMissingAttribute"
+        )
+
+    def test_wrong_attribute_type_3rd_party_backend(self):
+        """
+        If the plugin name refers to an object whose specified
+        attribute isn't of the right type, ``InvalidPluginType`` is
+        raised.
+        """
+        self.assertRaises(
+            InvalidPluginType,
+            DUMMY_LOADER.get,
+            "flocker.common.test.test_plugin.DummyPluginWrongType"
+        )

flocker/node/__init__.py

@@ -16,7 +16,8 @@
 from ._container import ApplicationNodeDeployer
 from ._p2p import P2PManifestationDeployer
 
-from .script import BackendDescription, DeployerType
+from .backends import BackendDescription
+from .script import DeployerType
 
 from ._docker import dockerpy_client