New discovery feature (#18)

Closes #16.

Signed-off-by: Sylvain Hellegouarch <sh@defuze.org>

CHANGELOG.md

@@ -2,7 +2,22 @@
 
 ## [Unreleased][]
 
-[Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/0.9.4...HEAD
+[Unreleased]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/0.10.0...HEAD
+
+## [0.10.0][] - 2018-01-16
+
+[0.9.4]: https://github.com/chaostoolkit/chaostoolkit-lib/compare/0.9.4...0.10.0
+
+### Added
+
+- New discovery of extension capabilities [#16][16]
+
+### Changed
+
+- Pinning ply dependency version to 3.4 to avoid random install failures [#8][8]
+
+[8]: https://github.com/chaostoolkit/chaostoolkit-lib/issues/8
+[16]: https://github.com/chaostoolkit/chaostoolkit-lib/issues/16
 
 ## [0.9.4][] - 2018-01-10
 

chaoslib/__init__.py

@@ -6,7 +6,7 @@
 from chaoslib.types import Configuration, Secrets
 
 __all__ = ["__version__", "substitute"]
-__version__ = '0.9.4'
+__version__ = '0.10.0'
 
 
 def substitute(data: Union[str, Dict[str, Any]], configuration: Configuration,

chaoslib/discovery/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+from chaoslib.discovery.discover import discover, \
+    discover_actions, discover_probes, initialize_discovery_result

chaoslib/discovery/discover.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+from datetime import datetime
+import importlib
+import inspect
+import platform
+import uuid
+
+from logzero import logger
+
+from chaoslib import __version__
+from chaoslib.discovery.package import get_discover_function, install,\
+    load_package
+from chaoslib.types import Discovery, DiscoveredActivities
+
+
+__all__ = ["discover", "discover_activities", "discover_actions",
+           "discover_probes", "initialize_discovery_result"]
+
+
+def discover(package_name: str, discover_system: bool = True,
+             download_and_install: bool = True) -> Discovery:
+    """
+    Discover the capabilities of an extension as well as the system it targets.
+
+    Then apply any post discovery hook that are declared in the chaostoolkit
+    settings under the `discovery/post-hook` section.
+    """
+    if download_and_install:
+        install(package_name)
+    package = load_package(package_name)
+    discover_func = get_discover_function(package)
+
+    return discover_func(discover_system=discover_system)
+
+
+def initialize_discovery_result(extension_name: str, extension_version: str,
+                                discovery_type: str) -> Discovery:
+    """
+    Intialize the discovery result payload to fill with activities and system
+    discovery.
+    """
+    plt = platform.uname()
+    return {
+        "chaoslib_version": __version__,
+        "id": str(uuid.uuid4()),
+        "type": discovery_type,
+        "date": "{d}Z".format(d=datetime.utcnow().isoformat()),
+        "platform": {
+            "system": plt.system,
+            "node": plt.node,
+            "release": plt.release,
+            "version": plt.version,
+            "machine": plt.machine,
+            "proc": plt.processor,
+            "python": platform.python_version()
+        },
+        "extension": {
+            "name": extension_name,
+            "version": extension_version,
+        },
+        "activities": [],
+        "system": None
+    }
+
+
+def discover_actions(extension_mod_name: str) -> DiscoveredActivities:
+    """
+    Discover actions from the given extension named `extension_mod_name`.
+    """
+    logger.info("Searching for actions")
+    return discover_activities(extension_mod_name, "action")
+
+
+def discover_probes(extension_mod_name: str) -> DiscoveredActivities:
+    """
+    Discover probes from the given extension named `extension_mod_name`.
+    """
+    logger.info("Searching for probes")
+    return discover_activities(extension_mod_name, "probe")
+
+
+def discover_activities(extension_mod_name: str,
+                        activity_type: str) -> DiscoveredActivities:
+    """
+    Discover exported activities from the given extension module name.
+    """
+    try:
+        mod = importlib.import_module(extension_mod_name)
+    except ImportError:
+        raise DiscoveryFailed(
+            "could not import Python module '{m}'".format(
+                m=extension_mod_name))
+
+    activities = []
+    exported = getattr(mod, "__all__")
+    funcs = inspect.getmembers(mod, inspect.isfunction)
+    for (name, func) in funcs:
+        if exported and name not in exported:
+            # do not return "private" functions
+            continue
+
+        sig = inspect.signature(func)
+        activity = {
+            "type": activity_type,
+            "name": name,
+            "mod": mod.__name__,
+            "doc": inspect.getdoc(func),
+            "arguments": []
+        }
+
+        # if sig.return_annotation is not inspect.Signature.empty:
+        #     activity["return_type"] = sig.return_annotation
+
+        for param in sig.parameters.values():
+            arg = {
+                "name": param.name,
+            }
+
+            if param.default is not inspect.Parameter.empty:
+                arg["default"] = param.default
+        #    if param.annotation is not inspect.Parameter.empty:
+        #         arg["type"] = param.annotation
+            activity["arguments"].append(arg)
+
+        activities.append(activity)
+
+    return activities

chaoslib/discovery/package.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import importlib
+import inspect
+import operator
+import subprocess
+
+from logzero import logger
+import pkg_resources
+
+from chaoslib.exceptions import DiscoveryFailed
+
+__all__ = ["get_discover_function", "install", "load_package"]
+
+
+def install(package_name: str):
+    """
+    Use pip to download and install the `package_name` to the current Python
+    environment. Pip can detect it is already installed.
+    """
+    logger.info("Attempting to download and install package '{p}'".format(
+        p=package_name))
+
+    process = subprocess.run(
+        ["pip", "install", "-U", package_name], stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+
+    stdout = process.stdout.decode('utf-8')
+    stderr = process.stderr.decode('utf-8')
+    logger.debug(stdout)
+
+    if process.returncode != 0:
+        msg = "failed to install `{p}`".format(p=package_name)
+        logger.debug(
+            msg + "\n=================\n{o}\n=================\n{e}\n".format(
+                o=stdout, e=stderr))
+        raise DiscoveryFailed(msg)
+
+    logger.info("Package downloaded and installed in current environment")
+
+
+def load_package(package_name: str) -> object:
+    """
+    Import the module into the current process state.
+    """
+    name = get_importname_from_package(package_name)
+    try:
+        package = importlib.import_module(name)
+    except ImportError:
+        raise DiscoveryFailed(
+            "could not load Python module '{name}'".format(name=name))
+
+    return package
+
+
+def get_discover_function(package: object):
+    """
+    Lookup the `discover` function from the given imported package.
+    """
+    funcs = inspect.getmembers(package, inspect.isfunction)
+    for (name, value) in funcs:
+        if name == 'discover':
+            return value
+
+    raise DiscoveryFailed(
+            "package '{name}' does not export a `discover` function".format(
+                name=package.__name__))
+
+
+###############################################################################
+# Private functions
+###############################################################################
+def get_importname_from_package(package_name: str) -> str:
+    """
+    Try to fetch the name of the top-level import name for the given
+    package. For some reason, this isn't straightforward.
+    """
+    reqs = list(pkg_resources.parse_requirements(package_name))
+    if not reqs:
+        raise DiscoveryFailed(
+            "no requirements met for package '{p}'".format(p=package_name))
+
+    req = reqs[0]
+    dist = pkg_resources.get_distribution(req)
+    try:
+        name = dist.get_metadata('top_level.txt').split("\n)", 1)[0]
+    except FileNotFoundError as err:
+        raise DiscoveryFailed(
+            "failed to load package '{p}' metadata. "
+            "Was the package installed properly?".format(p=package_name))
+
+    return name.strip()

chaoslib/exceptions.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 __all__ = ["ChaosException", "InvalidExperiment", "InvalidActivity",
-           "FailedActivity"]
+           "FailedActivity", "DiscoveryFailed"]
 
 
 class ChaosException(Exception):
@@ -18,3 +18,7 @@ class InvalidExperiment(ChaosException):
 
 class FailedActivity(ChaosException):
     pass
+
+
+class DiscoveryFailed(ChaosException):
+    pass

chaoslib/types.py

@@ -3,7 +3,8 @@
 
 __all__ = ["MicroservicesStatus", "Probe", "Action", "Experiment", "Layer",
            "TargetLayers", "Activity", "Journal", "Run", "Secrets", "Step",
-           "Configuration"]
+           "Configuration", "Discovery", "DiscoveredActivities",
+           "DiscoveredSystemInfo"]
 
 
 Action = Dict[str, Any]
@@ -22,3 +23,7 @@
 
 Secrets = Dict[str, Dict[str, str]]
 Configuration = Dict[str, Dict[str, str]]
+
+Discovery = Dict[str, Any]
+DiscoveredActivities = Dict[str, Any]
+DiscoveredSystemInfo = Dict[str, Any]

requirements.txt

@@ -1,5 +1,6 @@
 logzero
 pycodestyle
 hvac
-ply
-pyhcl>=0.2.1,<0.3.0
+ply==3.4
+pyhcl>=0.2.1,<0.3.0
+pyyaml

setup.py

@@ -36,6 +36,7 @@
 license = 'Apache License Version 2.0'
 packages = [
     'chaoslib',
+    'chaoslib.discovery',
     'chaoslib.provider'
 ]
 

tests/test_discover.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+import types
+
+import pytest
+
+from chaoslib.exceptions import DiscoveryFailed
+from chaoslib.discovery import discover, initialize_discovery_result
+from chaoslib.types import Discovery, DiscoveredActivities, \
+    DiscoveredSystemInfo