[pantsd] Launch the daemon via the thin client.

src/python/pants/backend/jvm/tasks/nailgun_task.py

@@ -9,11 +9,11 @@
 
 from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
 from pants.base.exceptions import TaskError
+from pants.init.subprocess import Subprocess
 from pants.java import util
 from pants.java.executor import SubprocessExecutor
 from pants.java.jar.jar_dependency import JarDependency
 from pants.java.nailgun_executor import NailgunExecutor, NailgunProcessGroup
-from pants.pantsd.subsystem.subprocess import Subprocess
 from pants.task.task import Task, TaskBase
 
 

src/python/pants/bin/BUILD

@@ -34,10 +34,12 @@ python_library(
     'src/python/pants/init',
     'src/python/pants/option',
     'src/python/pants/reporting',
+    'src/python/pants/pantsd:pants_daemon_launcher',
     'src/python/pants/scm/subsystems:changed',
     'src/python/pants/subsystem',
     'src/python/pants/task',
     'src/python/pants/util:contextutil',
+    'src/python/pants/util:collections',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:filtering',
     'src/python/pants/util:memo',

src/python/pants/bin/goal_runner.py

@@ -23,7 +23,7 @@
 from pants.goal.goal import Goal
 from pants.goal.run_tracker import RunTracker
 from pants.help.help_printer import HelpPrinter
-from pants.init.pants_daemon_launcher import PantsDaemonLauncher
+from pants.init.subprocess import Subprocess
 from pants.init.target_roots import TargetRoots
 from pants.java.nailgun_executor import NailgunProcessGroup
 from pants.reporting.reporting import Reporting
@@ -152,14 +152,7 @@ def generate_targets(specs):
 
     return list(generate_targets(specs))
 
-  def _maybe_launch_pantsd(self, pantsd_launcher):
-    """Launches pantsd if configured to do so."""
-    if self._global_options.enable_pantsd:
-      # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
-      with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]):
-        pantsd_launcher.maybe_launch()
-
-  def _setup_context(self, pantsd_launcher):
+  def _setup_context(self):
     with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
       self._build_graph, self._address_mapper, spec_roots = self._init_graph(
         self._global_options.enable_v2_engine,
@@ -189,15 +182,12 @@ def _setup_context(self, pantsd_launcher):
                         build_graph=self._build_graph,
                         build_file_parser=self._build_file_parser,
                         address_mapper=self._address_mapper,
-                        invalidation_report=invalidation_report,
-                        pantsd_launcher=pantsd_launcher)
+                        invalidation_report=invalidation_report)
       return goals, context
 
   def setup(self):
-    pantsd_launcher = PantsDaemonLauncher.Factory.global_instance().create(EngineInitializer)
-    self._maybe_launch_pantsd(pantsd_launcher)
     self._handle_help(self._help_request)
-    goals, context = self._setup_context(pantsd_launcher)
+    goals, context = self._setup_context()
     return GoalRunner(context=context,
                       goals=goals,
                       run_tracker=self._run_tracker,
@@ -234,7 +224,7 @@ def subsystems(cls):
       RunTracker,
       Changed.Factory,
       BinaryUtil.Factory,
-      PantsDaemonLauncher.Factory,
+      Subprocess.Factory
     }
 
   def _execute_engine(self):

src/python/pants/bin/pants_runner.py

@@ -29,29 +29,24 @@ def __init__(self, exiter, args=None, env=None):
     self._args = args or sys.argv
     self._env = env or os.environ
 
-  def _run(self, is_remote, exiter, args, env, process_metadata_dir=None, options_bootstrapper=None):
-    if is_remote:
+  def run(self):
+    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
+    bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
+
+    if bootstrap_options.enable_pantsd:
       try:
-        return RemotePantsRunner(exiter, args, env, process_metadata_dir).run()
-      except RemotePantsRunner.RECOVERABLE_EXCEPTIONS as e:
-        # N.B. RemotePantsRunner will raise one of RECOVERABLE_EXCEPTIONS in the event we
-        # encounter a failure while discovering or initially connecting to the pailgun. In
-        # this case, we fall back to LocalPantsRunner which seamlessly executes the requested
-        # run and bootstraps pantsd for use in subsequent runs.
-        logger.debug('caught client exception: {!r}, falling back to LocalPantsRunner'.format(e))
+        return RemotePantsRunner(self._exiter,
+                                 self._args,
+                                 self._env,
+                                 bootstrap_options.pants_subprocessdir,
+                                 bootstrap_options).run()
+      except RemotePantsRunner.Fallback as e:
+        logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e))
 
     # N.B. Inlining this import speeds up the python thin client run by about 100ms.
     from pants.bin.local_pants_runner import LocalPantsRunner
 
-    return LocalPantsRunner(exiter, args, env, options_bootstrapper=options_bootstrapper).run()
-
-  def run(self):
-    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
-    global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
-
-    return self._run(is_remote=global_bootstrap_options.enable_pantsd,
-                     exiter=self._exiter,
-                     args=self._args,
-                     env=self._env,
-                     process_metadata_dir=global_bootstrap_options.pants_subprocessdir,
-                     options_bootstrapper=options_bootstrapper)
+    return LocalPantsRunner(self._exiter,
+                            self._args,
+                            self._env,
+                            options_bootstrapper=options_bootstrapper).run()

src/python/pants/bin/remote_pants_runner.py

@@ -5,30 +5,41 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import logging
 import signal
 import sys
 from contextlib import contextmanager
 
+from pants.bin.engine_initializer import EngineInitializer
 from pants.java.nailgun_client import NailgunClient
 from pants.java.nailgun_protocol import NailgunProtocol
-from pants.pantsd.process_manager import ProcessMetadataManager
+from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher
+from pants.util.collections import combined_dict
+
+
+logger = logging.getLogger(__name__)
 
 
 class RemotePantsRunner(object):
   """A thin client variant of PantsRunner."""
 
-  class PortNotFound(Exception): pass
+  class Fallback(Exception):
+    """Raised when fallback to an alternate execution mode is requested."""
+
+  class PortNotFound(Exception):
+    """Raised when the pailgun port can't be found."""
 
   PANTS_COMMAND = 'pants'
   RECOVERABLE_EXCEPTIONS = (PortNotFound, NailgunClient.NailgunConnectionError)
 
-  def __init__(self, exiter, args, env, process_metadata_dir=None,
-               stdin=None, stdout=None, stderr=None):
+  def __init__(self, exiter, args, env, process_metadata_dir,
+               bootstrap_options, stdin=None, stdout=None, stderr=None):
     """
     :param Exiter exiter: The Exiter instance to use for this run.
     :param list args: The arguments (e.g. sys.argv) for this run.
     :param dict env: The environment (e.g. os.environ) for this run.
     :param str process_metadata_dir: The directory in which process metadata is kept.
+    :param Options bootstrap_options: The Options bag containing the bootstrap options.
     :param file stdin: The stream representing stdin.
     :param file stdout: The stream representing stdout.
     :param file stderr: The stream representing stderr.
@@ -37,17 +48,11 @@ def __init__(self, exiter, args, env, process_metadata_dir=None,
     self._args = args
     self._env = env
     self._process_metadata_dir = process_metadata_dir
+    self._bootstrap_options = bootstrap_options
     self._stdin = stdin or sys.stdin
     self._stdout = stdout or sys.stdout
     self._stderr = stderr or sys.stderr
-    self._port = self._retrieve_pailgun_port()
-    if not self._port:
-      raise self.PortNotFound('unable to locate pailgun port!')
-
-  @staticmethod
-  def _combine_dicts(*dicts):
-    """Combine one or more dicts into a new, unified dict (dicts to the right take precedence)."""
-    return {k: v for d in dicts for k, v in d.items()}
+    self._launcher = PantsDaemonLauncher(self._bootstrap_options, EngineInitializer)
 
   @contextmanager
   def _trapped_control_c(self, client):
@@ -62,17 +67,36 @@ def handle_control_c(signum, frame):
     finally:
       signal.signal(signal.SIGINT, existing_sigint_handler)
 
-  def _retrieve_pailgun_port(self):
-    return ProcessMetadataManager(
-      self._process_metadata_dir).read_metadata_by_name('pantsd', 'socket_pailgun', int)
+  def _setup_logging(self):
+    """Sets up basic stdio logging for the thin client."""
+    log_level = logging.getLevelName(self._bootstrap_options.level.upper())
 
-  def run(self, args=None):
+    formatter = logging.Formatter('%(levelname)s] %(message)s')
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(log_level)
+    handler.setFormatter(formatter)
+
+    root = logging.getLogger()
+    root.setLevel(log_level)
+    root.addHandler(handler)
+
+  def _find_or_launch_pantsd(self):
+    """Launches pantsd if configured to do so.
+
+    :returns: The port pantsd can be found on.
+    :rtype: int
+    """
+    return self._launcher.maybe_launch()
+
+  def _connect_and_execute(self, port):
     # Merge the nailgun TTY capability environment variables with the passed environment dict.
     ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr)
-    modified_env = self._combine_dicts(self._env, ng_env)
+    modified_env = combined_dict(self._env, ng_env)
+
+    assert isinstance(port, int), 'port {} is not an integer!'.format(port)
 
     # Instantiate a NailgunClient.
-    client = NailgunClient(port=self._port,
+    client = NailgunClient(port=port,
                            ins=self._stdin,
                            out=self._stdout,
                            err=self._stderr,
@@ -84,3 +108,13 @@ def run(self, args=None):
 
     # Exit.
     self._exiter.exit(result)
+
+  def run(self, args=None):
+    self._setup_logging()
+    port = self._find_or_launch_pantsd()
+
+    logger.debug('connecting to pailgun on port {}'.format(port))
+    try:
+      self._connect_and_execute(port)
+    except self.RECOVERABLE_EXCEPTIONS as e:
+      raise self.Fallback(e)

src/python/pants/core_tasks/pantsd_kill.py

@@ -6,6 +6,7 @@
                         unicode_literals, with_statement)
 
 from pants.base.exceptions import TaskError
+from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher
 from pants.pantsd.process_manager import ProcessManager
 from pants.task.task import Task
 
@@ -15,6 +16,6 @@ class PantsDaemonKill(Task):
 
   def execute(self):
     try:
-      self.context.pantsd_launcher.terminate()
+      PantsDaemonLauncher(self.get_options()).terminate()
     except ProcessManager.NonResponsiveProcess as e:
       raise TaskError('failure while terminating pantsd: {}'.format(e))

src/python/pants/core_tasks/register.py

@@ -59,7 +59,9 @@ def register_goals():
   # Pantsd.
   kill_pantsd = task(name='kill-pantsd', action=PantsDaemonKill)
   kill_pantsd.install()
-  kill_pantsd.install('clean-all')
+  # Kill pantsd/watchman first, so that they're not using any files
+  # in .pants.d at the time of removal.
+  kill_pantsd.install('clean-all', first=True)
 
   # Reporting server.
   # TODO: The reporting server should be subsumed into pantsd, and not run via a task.

src/python/pants/engine/native.py

@@ -561,9 +561,9 @@ class Native(object):
   """Encapsulates fetching a platform specific version of the native portion of the engine."""
 
   @staticmethod
-  def create(options):
+  def create(bootstrap_options):
     """:param options: Any object that provides access to bootstrap option values."""
-    return Native(options.native_engine_visualize_to)
+    return Native(bootstrap_options.native_engine_visualize_to)
 
   def __init__(self, visualize_to_dir):
     """

src/python/pants/goal/context.py

@@ -61,7 +61,7 @@ def fatal(self, *msg_elements):
   def __init__(self, options, run_tracker, target_roots,
                requested_goals=None, target_base=None, build_graph=None,
                build_file_parser=None, address_mapper=None, console_outstream=None, scm=None,
-               workspace=None, invalidation_report=None, pantsd_launcher=None):
+               workspace=None, invalidation_report=None):
     self._options = options
     self.build_graph = build_graph
     self.build_file_parser = build_file_parser
@@ -80,7 +80,6 @@ def __init__(self, options, run_tracker, target_roots,
     self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
     self._replace_targets(target_roots)
     self._invalidation_report = invalidation_report
-    self._pantsd_launcher = pantsd_launcher
 
   @property
   def options(self):
@@ -151,10 +150,6 @@ def workspace(self):
   def invalidation_report(self):
     return self._invalidation_report
 
-  @property
-  def pantsd_launcher(self):
-    return self._pantsd_launcher
-
   def __str__(self):
     ident = Target.identify(self.targets())
     return 'Context(id:{}, targets:{})'.format(ident, self.targets())

src/python/pants/init/BUILD

@@ -1,7 +1,6 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-
 python_library(
   dependencies=[
     ':plugins',
@@ -21,12 +20,6 @@ python_library(
     'src/python/pants/goal:run_tracker',
     'src/python/pants/logging',
     'src/python/pants/option',
-    'src/python/pants/pantsd:pants_daemon',
-    'src/python/pants/pantsd/service:fs_event_service',
-    'src/python/pants/pantsd/service:pailgun_service',
-    'src/python/pants/pantsd/service:scheduler_service',
-    'src/python/pants/pantsd/subsystem:subprocess',
-    'src/python/pants/pantsd/subsystem:watchman_launcher',
     'src/python/pants/process',
     'src/python/pants/python',
     'src/python/pants/subsystem',