New asyncio-based execution engine (PR#196 1/3)

.travis.before_install.bash

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+  echo "AOK"
+  #if [ "$PYTHON" == "/usr/bin/python3.4" ]; then
+    #sudo add-apt-repository ppa:fkrull/deadsnakes -y
+    #sudo apt-get update
+    #sudo apt-get install python3.4 python3-dev
+  #fi
+elif [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  if [ "$PYTHON" == "/usr/local/bin/python3" ]; then
+    brew install python3
+  fi
+  sudo pip install virtualenv
+  virtualenv -p $PYTHON venv
+  source venv/bin/activate
+fi

.travis.before_script.bash

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+  sudo apt-get install cmake libgtest-dev build-essential python-setuptools python-pip
+  pip install trollius
+elif [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  # noop?
+  pip install setuptools
+  pip install trollius
+fi

.travis.yml

@@ -1,12 +1,35 @@
-language: python
-python:
-  - "2.7"
-  - "3.3"
+# Travis auto-virtualenv isn't supported on OS X
+#language: python
+#python:
+  #- "2.7"
+  #- "3.3"
+language: generic
+matrix:
+  include:
+    - python: 2.7
+      language: python
+      python: "2.7"
+      os: linux
+      env: PYTHON=/usr/bin/python2.7
+    - python: 3.4
+      language: python
+      python: "3.4"
+      os: linux
+      env: PYTHON=/usr/bin/python3.4
+    - python: 2
+      os: osx
+      env: PYTHON=/usr/bin/python
+    - python: 3
+      os: osx
+      env: PYTHON=/usr/local/bin/python3
+before_install:
+  - source .travis.before_install.bash
 install:
   - pip install argparse catkin-pkg distribute PyYAML psutil
-  - pip install nose coverage flake8 --upgrade
+  - pip install nose coverage flake8 mock --upgrade
 before_script:
-  - sudo apt-get install cmake python-setuptools libgtest-dev build-essential
+  - ./.travis.before_script.bash
+  - pip install git+https://github.com/osrf/osrf_pycommon.git
   - pip install empy --upgrade
   - git clone https://github.com/ros/catkin.git /tmp/catkin_source
   - mkdir /tmp/catkin_source/build

catkin_tools/argument_parsing.py

@@ -14,6 +14,7 @@
 
 from __future__ import print_function
 
+import argparse
 import os
 import re
 import sys
@@ -22,9 +23,7 @@
 
 from catkin_tools.common import wide_log
 
-from catkin_tools.make_jobserver import initialize_jobserver
-from catkin_tools.make_jobserver import jobserver_arguments
-from catkin_tools.make_jobserver import jobserver_supported
+import catkin_tools.execution.job_server as job_server
 
 
 def add_context_args(parser):
@@ -60,10 +59,12 @@ def add_cmake_and_make_and_catkin_make_args(parser):
     """
 
     add = parser.add_argument
-    add('-p', '--parallel-packages', '--parallel-jobs', '--parallel', dest='parallel_jobs', default=None,
-        help='Maximum number of packages which could be built in parallel (default is cpu count)')
     add('-j', '--jobs', default=None,
-        help='Limit parallel job count through the internal GNU make job server. default is cpu count')
+        help='Maximum number of build jobs to be distributed across active packages. (default is cpu count)')
+    add('-p', '--parallel-packages', metavar='PACKAGE_JOBS', dest='parallel_jobs', default=None,
+        help='Maximum number of packages allowed to be built in parallel (default is cpu count)')
+    # Deprecated flags kept for compatibility
+    add('--parallel-jobs', '--parallel', action='store_true', dest='parallel_jobs', help=argparse.SUPPRESS)
 
     add = parser.add_mutually_exclusive_group().add_argument
     add('--jobserver', dest='use_internal_make_jobserver', default=None, action='store_true',
@@ -247,6 +248,10 @@ def handle_make_arguments(
 
     # Get the values for the jobs flags which may be in the make args
     jobs_dict = extract_jobs_flags_values(' '.join(make_args))
+    jobs_args = extract_jobs_flags(' '.join(make_args))
+    if len(jobs_args) > 0:
+        # Remove jobs flags from cli args if they're present
+        make_args = re.sub(' '.join(jobs_args), '', ' '.join(make_args)).split()
 
     if force_single_threaded_when_running_tests:
         # force single threaded execution when running test since rostest does not support multiple parallel runs
@@ -255,8 +260,8 @@ def handle_make_arguments(
             wide_log('Forcing "-j1" for running unit tests.')
             jobs_dict['jobs'] = 1
 
-    if len(jobs_dict) == 0:
-        make_args.extend(jobserver_arguments())
+    if job_server.gnu_make_enabled():
+        make_args.extend(job_server.gnu_make_args())
     else:
         if 'jobs' in jobs_dict:
             make_args.append('-j{0}'.format(jobs_dict['jobs']))
@@ -282,7 +287,7 @@ def configure_make_args(make_args, use_internal_make_jobserver):
         n_cpus = cpu_count()
         jobs_flags = {
             'jobs': n_cpus,
-            'load-average': n_cpus}
+            'load-average': n_cpus + 1}
     except NotImplementedError:
         # If the number of cores cannot be determined, limit to one job
         jobs_flags = {
@@ -292,29 +297,31 @@ def configure_make_args(make_args, use_internal_make_jobserver):
     # Get MAKEFLAGS from environment
     makeflags_jobs_flags = extract_jobs_flags(os.environ.get('MAKEFLAGS', ''))
     using_makeflags_jobs_flags = len(makeflags_jobs_flags) > 0
-    jobs_flags.update(extract_jobs_flags_values(' '.join(makeflags_jobs_flags)))
+    if using_makeflags_jobs_flags:
+        makeflags_jobs_flags_dict = extract_jobs_flags_values(' '.join(makeflags_jobs_flags))
+        jobs_flags.update(makeflags_jobs_flags_dict)
 
     # Extract make jobs flags (these override MAKEFLAGS)
     cli_jobs_flags = extract_jobs_flags(' '.join(make_args))
     using_cli_flags = len(cli_jobs_flags) > 0
-    jobs_flags.update(extract_jobs_flags_values(' '.join(cli_jobs_flags)))
     if cli_jobs_flags:
+        jobs_flags.update(extract_jobs_flags_values(' '.join(cli_jobs_flags)))
         # Remove jobs flags from cli args if they're present
         make_args = re.sub(' '.join(cli_jobs_flags), '', ' '.join(make_args)).split()
 
-    # Instantiate a jobserver
-    if use_internal_make_jobserver:
-        initialize_jobserver(
-            num_jobs=jobs_flags.get('jobs', None),
-            max_load=jobs_flags.get('load-average', None))
+    # Instantiate the jobserver
+    job_server.initialize(
+        max_jobs=jobs_flags.get('jobs', None),
+        max_load=jobs_flags.get('load-average', None),
+        gnu_make_enabled=use_internal_make_jobserver)
 
     # If the jobserver is supported
-    if jobserver_supported():
+    if job_server.gnu_make_enabled():
         jobs_args = []
     else:
         jobs_args = cli_jobs_flags
 
-    return make_args + jobs_args, using_makeflags_jobs_flags, using_cli_flags, jobserver_supported()
+    return make_args + jobs_args, using_makeflags_jobs_flags, using_cli_flags, job_server.gnu_make_enabled()
 
 
 def argument_preprocessor(args):
@@ -337,13 +344,14 @@ def argument_preprocessor(args):
 
     # Extract make jobs flags (these override MAKEFLAGS later on)
     jobs_args = extract_jobs_flags(' '.join(args))
-    if jobs_args:
+    if len(jobs_args) > 0:
         # Remove jobs flags from cli args if they're present
         args = re.sub(' '.join(jobs_args), '', ' '.join(args)).split()
 
     extras = {
         'cmake_args': cmake_args,
-        'make_args': (make_args or []) + (jobs_args or []),
+        'make_args': (make_args or []) + jobs_args,
         'catkin_make_args': catkin_make_args,
     }
+
     return args, extras

catkin_tools/commands/catkin.py

@@ -81,15 +81,66 @@ def create_subparsers(parser, verbs):
     return argument_preprocessors
 
 
-def main(sysargs=None):
+def expand_one_verb_alias(sysargs, verb_aliases, used_aliases):
+    """Iterate through sysargs looking for expandable verb aliases.
+
+    When a verb alias is found, sysargs is modified to effectively expand the alias.
+    The alias is removed from verb_aliases and added to used_aliases.
+    After finding and expanding an alias, this function returns True.
+    If no alias is found to be expanded, this function returns False.
+    """
+    cmd = os.path.basename(sys.argv[0])
+    for index, arg in enumerate(sysargs):
+        if arg.startswith('-'):
+            # Not a verb, continue through the arguments
+            continue
+        if arg in used_aliases:
+            print(fmt(
+                "@!@{gf}==>@| Expanding alias '@!@{yf}" + arg +
+                "@|' was previously expanded, ignoring this time to prevent infinite recursion."
+            ))
+        if arg in verb_aliases:
+            before = [] if index == 0 else sysargs[:index - 1]
+            after = [] if index == len(sysargs) else sysargs[index + 1:]
+            sysargs[:] = before + verb_aliases[arg].split() + after
+            print(fmt(
+                "@!@{gf}==>@| Expanding alias "
+                "'@!@{yf}{alias}@|' "
+                "from '@{yf}{before} @!{alias}@{boldoff}{after}@|' "
+                "to '@{yf}{before} @!{expansion}@{boldoff}{after}@|'"
+            ).format(
+                alias=arg,
+                expansion=verb_aliases[arg],
+                before=' '.join([cmd] + before),
+                after=(' '.join([''] + after) if after else '')
+            ))
+            # Prevent the alias from being used again, to prevent infinite recursion
+            used_aliases.append(arg)
+            del verb_aliases[arg]
+            # Return True since one has been found
+            return True
+        # Return False since no verb alias was found
+        return False
+
+
+def expand_verb_aliases(sysargs, verb_aliases):
+    """Expands aliases in sysargs which are found in verb_aliases until none are found."""
+    used_aliases = []
+    while expand_one_verb_alias(sysargs, verb_aliases, used_aliases):
+        pass
+    return sysargs
+
+
+def catkin_main(sysargs):
     # Initialize config
     try:
         initialize_config()
     except RuntimeError as exc:
         sys.exit("Failed to initialize config: {0}".format(exc))
 
     # Create a top level parser
-    parser = argparse.ArgumentParser(description="catkin command", formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser = argparse.ArgumentParser(
+        description="catkin command", formatter_class=argparse.RawDescriptionHelpFormatter)
     add = parser.add_argument
     add('-a', '--list-aliases', action="store_true", default=False,
         help="Lists the current verb aliases and then quits, all other arguments are ignored")
@@ -113,7 +164,6 @@ def main(sysargs=None):
 
     # Setup sysargs
     sysargs = sys.argv[1:] if sysargs is None else sysargs
-    cmd = os.path.basename(sys.argv[0])
 
     # Get colors config
     no_color = False
@@ -144,39 +194,8 @@ def main(sysargs=None):
         if not arg.startswith('-'):
             break
 
-    # Do alias expansion
-    expanding_verb_aliases = True
-    used_aliases = []
-    while expanding_verb_aliases:
-        expanding_verb_aliases = False
-        for index, arg in enumerate(sysargs):
-            if not arg.startswith('-'):
-                if arg in used_aliases:
-                    print(fmt(
-                        "@!@{gf}==>@| Expanding alias '@!@{yf}" +
-                        arg +
-                        "@|' was previously expanded, ignoring this time to prevent infinite recursion."
-                    ))
-                if arg in verb_aliases:
-                    before = [] if index == 0 else sysargs[:index - 1]
-                    after = [] if index == len(sysargs) else sysargs[index + 1:]
-                    sysargs = before + verb_aliases[arg].split() + after
-                    print(fmt(
-                        "@!@{gf}==>@| Expanding alias "
-                        "'@!@{yf}{alias}@|' "
-                        "from '@{yf}{before} @!{alias}@{boldoff}{after}@|' "
-                        "to '@{yf}{before} @!{expansion}@{boldoff}{after}@|'"
-                    ).format(
-                        alias=arg,
-                        expansion=verb_aliases[arg],
-                        before=' '.join([cmd] + before),
-                        after=(' '.join([''] + after) if after else '')
-                    ))
-                    expanding_verb_aliases = True
-                    # Prevent the alias from being used again, to prevent infinite recursion
-                    used_aliases.append(arg)
-                    del verb_aliases[arg]
-                break
+    # Do verb alias expansion
+    sysargs = expand_verb_aliases(sysargs, verb_aliases)
 
     # Determine the verb, splitting arguments into pre and post verb
     verb = None
@@ -217,3 +236,10 @@ def main(sysargs=None):
     # Finally call the subparser's main function with the processed args
     # and the extras which the preprocessor may have returned
     sys.exit(args.main(args) or 0)
+
+
+def main(sysargs=None):
+    try:
+        catkin_main(sysargs)
+    except KeyboardInterrupt:
+        print('Interrupted by user!')