Skip to content

Commit

Permalink
Add experimental --parallel functionality
Browse files Browse the repository at this point in the history
Working towards closing ansible#1702.
  • Loading branch information
decentral1se committed Jun 28, 2019
1 parent ddc8b6a commit d8d3d2a
Show file tree
Hide file tree
Showing 18 changed files with 205 additions and 31 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ History
Unreleased
==========

* Add the `--parallel` flag to experimentally allow molecule to be run in parallel.
* `dependency` step is now run by default before any playbook sequence step, including
`create` and `destroy`. This allows the use of roles in all sequence step playbooks.
* Removed validation regex for docker registry passwords, all ``string`` values are now valid.
Expand Down
45 changes: 45 additions & 0 deletions docs/examples.rst
Original file line number Diff line number Diff line change
Expand Up @@ -279,3 +279,48 @@ lives in a shared location and ``molecule.yml`` is points to the shared tests.
directory: ../resources/tests/
lint:
name: flake8
.. _parallel-usage-example:

Running Molecule processes in parallel mode
===========================================

.. important::

This functionality should be considered experimental. It is part of ongoing
work towards enabling parallelizable functionality across all moving parts
in the execution of the Molecule feature set.

.. note::

Only the following sequences support parallelizable functionality:

* ``check_sequence``: ``molecule check --parallel``
* ``destroy_sequence``: ``molecule destroy --parallel``
* ``test_sequence``: ``molecule test --parallel``

It is possible to run Molecule processes in parallel using another tool to
orchestrate the parallelization (such as `GNU Parallel`_ or `Pytest`_).

When Molecule receives the ``--parallel`` flag it will generate a `UUID`_ for
the duration of the testing sequence and will use that unique identifier to
cache the run-time state for that process. The parallel Molecule processes
cached state and created instances will therefore not interfere with each
other.

Molecule uses a new and separate caching folder for this in the
``$HOME/.cache/molecule_parallel`` location.

Two new variables (``molecule_parallel`` and ``molecule_run_uuid``) and a new
filter (``molecule_parallelize``) are introduced into the playbook context from
the Molecule internals. The API and usage of these variables and filters
**cannot** be relied on. They are experimental and may be purely for internal
use in the coming iterations.

If you are writing custom playbooks for your ``create.yml`` and ``destroy.yml``
then it is your responsibility to adapt your playbooks to allow for
``--parallel`` based functionality.

.. _GNU Parallel: https://www.gnu.org/software/parallel/
.. _Pytest: https://docs.pytest.org/en/latest/
.. _UUID: https://en.wikipedia.org/wiki/Universally_unique_identifier
6 changes: 6 additions & 0 deletions docs/faq.rst
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,9 @@ Are there similar tools to Molecule?
.. _`ansible-test`: https://github.com/nylas/ansible-test
.. _`abandoned`: https://github.com/nylas/ansible-test/issues/14
.. _`RoleSpec`: https://github.com/nickjj/rolespec


Can I run Molecule processes in parallel?
=========================================

Please see :ref:`parallel-usage-example` for usage.
5 changes: 5 additions & 0 deletions molecule/command/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ def execute_cmdline_scenarios(scenario_name,
execute_subcommand(scenario.config, 'destroy')
# always prune ephemeral dir if destroying on failure
scenario.prune()
if scenario.config.is_parallel:
scenario._remove_scenario_state_directory()
util.sysexit()
else:
raise
Expand Down Expand Up @@ -144,6 +146,9 @@ def execute_scenario(scenario):
if 'destroy' in scenario.sequence:
scenario.prune()

if scenario.config.is_parallel:
scenario._remove_scenario_state_directory()


def get_configs(args, command_args, ansible_args=()):
"""
Expand Down
17 changes: 16 additions & 1 deletion molecule/command/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

from molecule import logger
from molecule.command import base
from molecule import util

LOG = logger.get_logger(__name__)

Expand Down Expand Up @@ -58,6 +59,12 @@ class Check(base.Base):
Load an env file to read variables from when rendering
molecule.yml.
.. program:: molecule --parallel check
.. option:: molecule --parallel check
Run in parallelizable mode.
"""

def execute(self):
Expand All @@ -79,15 +86,23 @@ def execute(self):
default=base.MOLECULE_DEFAULT_SCENARIO_NAME,
help='Name of the scenario to target. ({})'.format(
base.MOLECULE_DEFAULT_SCENARIO_NAME))
def check(ctx, scenario_name): # pragma: no cover
@click.option(
'--parallel/--no-parallel',
default=False,
help='Enable or disable parallel mode. Default is disabled.')
def check(ctx, scenario_name, parallel): # pragma: no cover
"""
Use the provisioner to perform a Dry-Run (destroy, dependency, create,
prepare, converge).
"""
args = ctx.obj.get('args')
subcommand = base._get_subcommand(__name__)
command_args = {
'parallel': parallel,
'subcommand': subcommand,
}

if parallel:
util.validate_parallel_cmd_args(command_args)

base.execute_cmdline_scenarios(scenario_name, args, command_args)
18 changes: 17 additions & 1 deletion molecule/command/destroy.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from molecule import config
from molecule import logger
from molecule.command import base
from molecule import util

LOG = logger.get_logger(__name__)

Expand Down Expand Up @@ -71,6 +72,12 @@ class Destroy(base.Base):
Load an env file to read variables from when rendering
molecule.yml.
.. program:: molecule --parallel destroy
.. option:: molecule --parallel destroy
Run in parallelizable mode.
"""

def execute(self):
Expand Down Expand Up @@ -114,16 +121,25 @@ def execute(self):
'__all',
default=False,
help='Destroy all scenarios. Default is False.')
def destroy(ctx, scenario_name, driver_name, __all): # pragma: no cover
@click.option(
'--parallel/--no-parallel',
default=False,
help='Enable or disable parallel mode. Default is disabled.')
def destroy(ctx, scenario_name, driver_name, __all,
parallel): # pragma: no cover
""" Use the provisioner to destroy the instances. """
args = ctx.obj.get('args')
subcommand = base._get_subcommand(__name__)
command_args = {
'parallel': parallel,
'subcommand': subcommand,
'driver_name': driver_name,
}

if __all:
scenario_name = None

if parallel:
util.validate_parallel_cmd_args(command_args)

base.execute_cmdline_scenarios(scenario_name, args, command_args)
18 changes: 17 additions & 1 deletion molecule/command/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from molecule import config
from molecule import logger
from molecule.command import base
from molecule import util

LOG = logger.get_logger(__name__)

Expand Down Expand Up @@ -71,6 +72,12 @@ class Test(base.Base):
Load an env file to read variables from when rendering
molecule.yml.
.. program:: molecule --parallel test
.. option:: molecule --parallel test
Run in parallelizable mode.
"""

def execute(self):
Expand Down Expand Up @@ -106,7 +113,12 @@ def execute(self):
default='always',
help=('The destroy strategy used at the conclusion of a '
'Molecule run (always).'))
def test(ctx, scenario_name, driver_name, __all, destroy): # pragma: no cover
@click.option(
'--parallel/--no-parallel',
default=False,
help='Enable or disable parallel mode. Default is disabled.')
def test(ctx, scenario_name, driver_name, __all, destroy,
parallel): # pragma: no cover
"""
Test (lint, cleanup, destroy, dependency, syntax, create, prepare,
converge, idempotence, side_effect, verify, cleanup, destroy).
Expand All @@ -115,6 +127,7 @@ def test(ctx, scenario_name, driver_name, __all, destroy): # pragma: no cover
args = ctx.obj.get('args')
subcommand = base._get_subcommand(__name__)
command_args = {
'parallel': parallel,
'destroy': destroy,
'subcommand': subcommand,
'driver_name': driver_name,
Expand All @@ -123,4 +136,7 @@ def test(ctx, scenario_name, driver_name, __all, destroy): # pragma: no cover
if __all:
scenario_name = None

if parallel:
util.validate_parallel_cmd_args(command_args)

base.execute_cmdline_scenarios(scenario_name, args, command_args)
13 changes: 12 additions & 1 deletion molecule/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from uuid import uuid4
import os

import anyconfig
Expand Down Expand Up @@ -109,11 +110,16 @@ def __init__(self,
self.ansible_args = ansible_args
self.config = self._get_config()
self._action = None
self._run_uuid = str(uuid4())

def after_init(self):
self.config = self._reget_config()
self._validate()

@property
def is_parallel(self):
return self.command_args.get('parallel', False)

@property
def debug(self):
return self.args.get('debug', MOLECULE_DEBUG)
Expand All @@ -138,6 +144,10 @@ def action(self, value):
def project_directory(self):
return os.getcwd()

@property
def cache_directory(self):
return 'molecule_parallel' if self.is_parallel else 'molecule'

@property
def molecule_directory(self):
return molecule_directory(self.project_directory)
Expand Down Expand Up @@ -224,7 +234,8 @@ def lint(self):
@property
@util.memoize
def platforms(self):
return platforms.Platforms(self)
return platforms.Platforms(
self, parallelize_platforms=self.is_parallel)

@property
@util.memoize
Expand Down
21 changes: 20 additions & 1 deletion molecule/platforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,34 @@ class Platforms(object):
- child_group1
"""

def __init__(self, config):
def __init__(self, config, parallelize_platforms=False):
"""
Initialize a new platform class and returns None.
:param config: An instance of a Molecule config.
:return: None
"""
if parallelize_platforms:
config.config['platforms'] = self._parallelize_platforms(config)
self._config = config

@property
def instances(self):
return self._config.config['platforms']

def _parallelize_platforms(self, config):
"""
Internally parallize the platform instance names.
:param config: An instance of a Molecule config.
:return: list
"""

def parallelize(platform):
platform['name'] = '{}-{}'.format(platform['name'],
config._run_uuid)
return platform

return [
parallelize(platform) for platform in config.config['platforms']
]
6 changes: 5 additions & 1 deletion molecule/provisioner/ansible.py
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,11 @@ def inventory(self):
"{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}",
'molecule_no_log':
"{{ lookup('env', 'MOLECULE_NO_LOG') or not "
"molecule_yml.provisioner.log|default(False) | bool }}"
"molecule_yml.provisioner.log|default(False) | bool }}",
'molecule_parallel':
self._config.is_parallel,
'molecule_run_uuid':
self._config._run_uuid,
}

# All group
Expand Down
2 changes: 1 addition & 1 deletion molecule/provisioner/ansible/playbooks/docker/create.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@

- name: Create molecule instance(s)
docker_container:
name: "{{ item.name }}"
name: "{{ item.name | molecule_parallelize(molecule_parallel, molecule_run_uuid) }}"
docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
Expand Down
2 changes: 1 addition & 1 deletion molecule/provisioner/ansible/playbooks/docker/destroy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
tasks:
- name: Destroy molecule instance(s)
docker_container:
name: "{{ item.name }}"
name: "{{ item.name | molecule_parallelize(molecule_parallel, molecule_run_uuid) }}"
docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}"
Expand Down
7 changes: 7 additions & 0 deletions molecule/provisioner/ansible/plugins/filters/molecule_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ def get_docker_networks(data):
return network_list


def parallelize(item, is_parallel, uuid):
if is_parallel:
return '{}-{}'.format(item, uuid)
return item


class FilterModule(object):
""" Core Molecule filter plugins. """

Expand All @@ -74,4 +80,5 @@ def filters(self):
'molecule_to_yaml': to_yaml,
'molecule_header': header,
'molecule_get_docker_networks': get_docker_networks,
'molecule_parallelize': parallelize,
}
Loading

0 comments on commit d8d3d2a

Please sign in to comment.