Add experimental --parallel functionality (#2137)

Working towards closing #1702.

Author: decentral1se

CHANGELOG.rst

@@ -5,6 +5,7 @@ History
 Unreleased
 ==========
 
+* Add the `--parallel` flag to experimentally allow molecule to be run in parallel.
 * `dependency` step is now run by default before any playbook sequence step, including
   `create` and `destroy`. This allows the use of roles in all sequence step playbooks.
 * Removed validation regex for docker registry passwords, all ``string`` values are now valid.

docs/examples.rst

@@ -279,3 +279,41 @@ lives in a shared location and ``molecule.yml`` is points to the shared tests.
     directory: ../resources/tests/
     lint:
       name: flake8
+
+.. _parallel-usage-example:
+
+Running Molecule processes in parallel mode
+===========================================
+
+.. important::
+
+    This functionality should be considered experimental. It is part of ongoing
+    work towards enabling parallelizable functionality across all moving parts
+    in the execution of the Molecule feature set.
+
+.. note::
+
+    Only the following sequences support parallelizable functionality:
+
+      * ``check_sequence``: ``molecule check --parallel``
+      * ``destroy_sequence``: ``molecule destroy --parallel``
+      * ``test_sequence``: ``molecule test --parallel``
+
+    It is currently only available for use with the Docker driver.
+
+It is possible to run Molecule processes in parallel using another tool to
+orchestrate the parallelization (such as `GNU Parallel`_ or `Pytest`_).
+
+When Molecule receives the ``--parallel`` flag it will generate a `UUID`_ for
+the duration of the testing sequence and will use that unique identifier to
+cache the run-time state for that process. The parallel Molecule processes
+cached state and created instances will therefore not interfere with each
+other.
+
+Molecule uses a new and separate caching folder for this in the
+``$HOME/.cache/molecule_parallel`` location. Molecule exposes a new environment
+variable ``MOLECULE_PARALLEL`` which can enable this functionality.
+
+.. _GNU Parallel: https://www.gnu.org/software/parallel/
+.. _Pytest: https://docs.pytest.org/en/latest/
+.. _UUID: https://en.wikipedia.org/wiki/Universally_unique_identifier

docs/faq.rst

@@ -78,3 +78,9 @@ Are there similar tools to Molecule?
 .. _`ansible-test`: https://github.com/nylas/ansible-test
 .. _`abandoned`: https://github.com/nylas/ansible-test/issues/14
 .. _`RoleSpec`: https://github.com/nickjj/rolespec
+
+
+Can I run Molecule processes in parallel?
+=========================================
+
+Please see :ref:`parallel-usage-example` for usage.

molecule/command/base.py

@@ -110,6 +110,8 @@ def execute_cmdline_scenarios(scenario_name,
                 execute_subcommand(scenario.config, 'destroy')
                 # always prune ephemeral dir if destroying on failure
                 scenario.prune()
+                if scenario.config.is_parallel:
+                    scenario._remove_scenario_state_directory()
                 util.sysexit()
             else:
                 raise
@@ -144,6 +146,9 @@ def execute_scenario(scenario):
     if 'destroy' in scenario.sequence:
         scenario.prune()
 
+        if scenario.config.is_parallel:
+            scenario._remove_scenario_state_directory()
+
 
 def get_configs(args, command_args, ansible_args=()):
     """

molecule/command/check.py

@@ -18,12 +18,15 @@
 #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 #  DEALINGS IN THE SOFTWARE.
 
+import os
 import click
 
 from molecule import logger
 from molecule.command import base
+from molecule import util
 
 LOG = logger.get_logger(__name__)
+MOLECULE_PARALLEL = os.environ.get('MOLECULE_PARALLEL', False)
 
 
 class Check(base.Base):
@@ -58,6 +61,12 @@ class Check(base.Base):
 
         Load an env file to read variables from when rendering
         molecule.yml.
+
+    .. program:: molecule --parallel check
+
+    .. option:: molecule --parallel check
+
+       Run in parallelizable mode.
     """
 
     def execute(self):
@@ -79,15 +88,23 @@ def execute(self):
     default=base.MOLECULE_DEFAULT_SCENARIO_NAME,
     help='Name of the scenario to target. ({})'.format(
         base.MOLECULE_DEFAULT_SCENARIO_NAME))
-def check(ctx, scenario_name):  # pragma: no cover
+@click.option(
+    '--parallel/--no-parallel',
+    default=MOLECULE_PARALLEL,
+    help='Enable or disable parallel mode. Default is disabled.')
+def check(ctx, scenario_name, parallel):  # pragma: no cover
     """
     Use the provisioner to perform a Dry-Run (destroy, dependency, create,
     prepare, converge).
     """
     args = ctx.obj.get('args')
     subcommand = base._get_subcommand(__name__)
     command_args = {
+        'parallel': parallel,
         'subcommand': subcommand,
     }
 
+    if parallel:
+        util.validate_parallel_cmd_args(command_args)
+
     base.execute_cmdline_scenarios(scenario_name, args, command_args)

molecule/command/destroy.py

@@ -18,13 +18,16 @@
 #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 #  DEALINGS IN THE SOFTWARE.
 
+import os
 import click
 
 from molecule import config
 from molecule import logger
 from molecule.command import base
+from molecule import util
 
 LOG = logger.get_logger(__name__)
+MOLECULE_PARALLEL = os.environ.get('MOLECULE_PARALLEL', False)
 
 
 class Destroy(base.Base):
@@ -71,6 +74,12 @@ class Destroy(base.Base):
 
         Load an env file to read variables from when rendering
         molecule.yml.
+
+    .. program:: molecule --parallel destroy
+
+    .. option:: molecule --parallel destroy
+
+       Run in parallelizable mode.
     """
 
     def execute(self):
@@ -112,18 +121,27 @@ def execute(self):
 @click.option(
     '--all/--no-all',
     '__all',
-    default=False,
+    default=MOLECULE_PARALLEL,
     help='Destroy all scenarios. Default is False.')
-def destroy(ctx, scenario_name, driver_name, __all):  # pragma: no cover
+@click.option(
+    '--parallel/--no-parallel',
+    default=False,
+    help='Enable or disable parallel mode. Default is disabled.')
+def destroy(ctx, scenario_name, driver_name, __all,
+            parallel):  # pragma: no cover
     """ Use the provisioner to destroy the instances. """
     args = ctx.obj.get('args')
     subcommand = base._get_subcommand(__name__)
     command_args = {
+        'parallel': parallel,
         'subcommand': subcommand,
         'driver_name': driver_name,
     }
 
     if __all:
         scenario_name = None
 
+    if parallel:
+        util.validate_parallel_cmd_args(command_args)
+
     base.execute_cmdline_scenarios(scenario_name, args, command_args)

molecule/command/test.py

@@ -18,13 +18,16 @@
 #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 #  DEALINGS IN THE SOFTWARE.
 
+import os
 import click
 
 from molecule import config
 from molecule import logger
 from molecule.command import base
+from molecule import util
 
 LOG = logger.get_logger(__name__)
+MOLECULE_PARALLEL = os.environ.get('MOLECULE_PARALLEL', False)
 
 
 class Test(base.Base):
@@ -71,6 +74,12 @@ class Test(base.Base):
 
         Load an env file to read variables from when rendering
         molecule.yml.
+
+    .. program:: molecule --parallel test
+
+    .. option:: molecule --parallel test
+
+       Run in parallelizable mode.
     """
 
     def execute(self):
@@ -106,7 +115,12 @@ def execute(self):
     default='always',
     help=('The destroy strategy used at the conclusion of a '
           'Molecule run (always).'))
-def test(ctx, scenario_name, driver_name, __all, destroy):  # pragma: no cover
+@click.option(
+    '--parallel/--no-parallel',
+    default=MOLECULE_PARALLEL,
+    help='Enable or disable parallel mode. Default is disabled.')
+def test(ctx, scenario_name, driver_name, __all, destroy,
+         parallel):  # pragma: no cover
     """
     Test (lint, cleanup, destroy, dependency, syntax, create, prepare,
           converge, idempotence, side_effect, verify, cleanup, destroy).
@@ -115,6 +129,7 @@ def test(ctx, scenario_name, driver_name, __all, destroy):  # pragma: no cover
     args = ctx.obj.get('args')
     subcommand = base._get_subcommand(__name__)
     command_args = {
+        'parallel': parallel,
         'destroy': destroy,
         'subcommand': subcommand,
         'driver_name': driver_name,
@@ -123,4 +138,7 @@ def test(ctx, scenario_name, driver_name, __all, destroy):  # pragma: no cover
     if __all:
         scenario_name = None
 
+    if parallel:
+        util.validate_parallel_cmd_args(command_args)
+
     base.execute_cmdline_scenarios(scenario_name, args, command_args)

molecule/config.py

@@ -18,6 +18,7 @@
 #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 #  DEALINGS IN THE SOFTWARE.
 
+from uuid import uuid4
 import os
 
 import anyconfig
@@ -109,11 +110,16 @@ def __init__(self,
         self.ansible_args = ansible_args
         self.config = self._get_config()
         self._action = None
+        self._run_uuid = str(uuid4())
 
     def after_init(self):
         self.config = self._reget_config()
         self._validate()
 
+    @property
+    def is_parallel(self):
+        return self.command_args.get('parallel', False)
+
     @property
     def debug(self):
         return self.args.get('debug', MOLECULE_DEBUG)
@@ -138,6 +144,10 @@ def action(self, value):
     def project_directory(self):
         return os.getcwd()
 
+    @property
+    def cache_directory(self):
+        return 'molecule_parallel' if self.is_parallel else 'molecule'
+
     @property
     def molecule_directory(self):
         return molecule_directory(self.project_directory)
@@ -198,6 +208,7 @@ def env(self):
             'MOLECULE_DEBUG': str(self.debug),
             'MOLECULE_FILE': self.molecule_file,
             'MOLECULE_ENV_FILE': self.env_file,
+            'MOLECULE_STATE_FILE': self.state.state_file,
             'MOLECULE_INVENTORY_FILE': self.provisioner.inventory_file,
             'MOLECULE_EPHEMERAL_DIRECTORY': self.scenario.ephemeral_directory,
             'MOLECULE_SCENARIO_DIRECTORY': self.scenario.directory,
@@ -224,7 +235,8 @@ def lint(self):
     @property
     @util.memoize
     def platforms(self):
-        return platforms.Platforms(self)
+        return platforms.Platforms(
+            self, parallelize_platforms=self.is_parallel)
 
     @property
     @util.memoize

molecule/platforms.py

@@ -19,6 +19,7 @@
 #  DEALINGS IN THE SOFTWARE.
 
 from molecule import logger
+from molecule import util
 
 LOG = logger.get_logger(__name__)
 
@@ -65,13 +66,16 @@ class Platforms(object):
               - child_group1
     """
 
-    def __init__(self, config):
+    def __init__(self, config, parallelize_platforms=False):
         """
         Initialize a new platform class and returns None.
 
         :param config: An instance of a Molecule config.
         :return: None
         """
+        if parallelize_platforms:
+            config.config['platforms'] = util._parallelize_platforms(
+                config.config, config._run_uuid)
         self._config = config
 
     @property

molecule/provisioner/ansible/plugins/filters/molecule_core.py

@@ -43,7 +43,9 @@ def from_yaml(data):
     i = interpolation.Interpolator(interpolation.TemplateWithDefaults, env)
     interpolated_data = i.interpolate(data)
 
-    return util.safe_load(interpolated_data)
+    loaded_data = util.safe_load(interpolated_data)
+    loaded_data = _parallelize_config(loaded_data)
+    return loaded_data
 
 
 def to_yaml(data):
@@ -65,6 +67,14 @@ def get_docker_networks(data):
     return network_list
 
 
+def _parallelize_config(data):
+    state = util.safe_load_file(os.environ['MOLECULE_STATE_FILE'])
+    if state['is_parallel']:
+        data['platforms'] = util._parallelize_platforms(
+            data, state['run_uuid'])
+    return data
+
+
 class FilterModule(object):
     """ Core Molecule filter plugins. """
 

molecule/scenario.py

@@ -18,6 +18,7 @@
 #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 #  DEALINGS IN THE SOFTWARE.
 
+import shutil
 import os
 import fnmatch
 try:
@@ -98,6 +99,14 @@ def __init__(self, config):
         self.config = config
         self._setup()
 
+    def _remove_scenario_state_directory(self):
+        """Remove scenario cached disk stored state.
+
+        :return: None
+        """
+        LOG.info('Removing scenario state directory from cache')
+        shutil.rmtree(Path(self.ephemeral_directory).parent)
+
     def prune(self):
         """
         Prune the scenario ephemeral directory files and returns None.
@@ -137,9 +146,14 @@ def directory(self):
     @property
     def ephemeral_directory(self):
         project_directory = os.path.basename(self.config.project_directory)
-        scenario_name = self.name
-        project_scenario_directory = os.path.join(
-            'molecule', project_directory, scenario_name)
+
+        if self.config.is_parallel:
+            project_directory = '{}-{}'.format(project_directory,
+                                               self.config._run_uuid)
+
+        project_scenario_directory = os.path.join(self.config.cache_directory,
+                                                  project_directory, self.name)
+
         path = ephemeral_directory(project_scenario_directory)
 
         return ephemeral_directory(path)