split NailgunProtocol and onwards into PailgunProtocol versions of ev…

…erything

we were using something that kinda supported either without making it clear

src/python/pants/bin/daemon_pants_runner.py

@@ -21,7 +21,7 @@
 from pants.bin.local_pants_runner import LocalPantsRunner
 from pants.init.util import clean_global_runtime_state
 from pants.java.nailgun_io import NailgunStreamStdinReader, NailgunStreamWriter
-from pants.java.nailgun_protocol import ChunkType, NailgunProtocol
+from pants.java.nailgun_protocol import PailgunChunkType, PailgunProtocol
 from pants.pantsd.process_manager import ProcessManager
 from pants.rules.core.exceptions import GracefulTerminationException
 from pants.util.contextutil import hermetic_environment_as, stdio_as
@@ -50,7 +50,7 @@ def exit(self, result=0, msg=None, *args, **kwargs):
         self._finalizer()
       except Exception as e:
         try:
-          NailgunProtocol.send_stderr(
+          PailgunProtocol.send_stderr(
             self._socket,
             '\nUnexpected exception in finalizer: {!r}\n'.format(e)
           )
@@ -60,10 +60,10 @@ def exit(self, result=0, msg=None, *args, **kwargs):
     try:
       # Write a final message to stderr if present.
       if msg:
-        NailgunProtocol.send_stderr(self._socket, msg)
+        PailgunProtocol.send_stderr(self._socket, msg)
 
       # Send an Exit chunk with the result.
-      NailgunProtocol.send_exit_with_code(self._socket, result)
+      PailgunProtocol.send_exit_with_code(self._socket, result)
 
       # Shutdown the connected socket.
       teardown_socket(self._socket)
@@ -157,7 +157,7 @@ def _tty_stdio(cls, env):
     # character device for output redirection - eliminating the need to directly marshall any
     # interactive stdio back/forth across the socket and permitting full, correct tty control with
     # no middle-man.
-    stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env(env)
+    stdin_ttyname, stdout_ttyname, stderr_ttyname = PailgunProtocol.ttynames_from_env(env)
     assert stdin_ttyname == stdout_ttyname == stderr_ttyname, (
       'expected all stdio ttys to be the same, but instead got: {}\n'
       'please file a bug at http://github.com/pantsbuild/pants'
@@ -175,8 +175,8 @@ def finalizer():
   def _pipe_stdio(cls, sock, stdin_isatty, stdout_isatty, stderr_isatty, handle_stdin):
     """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys."""
     stdio_writers = (
-      (ChunkType.STDOUT, stdout_isatty),
-      (ChunkType.STDERR, stderr_isatty)
+      (PailgunChunkType.STDOUT, stdout_isatty),
+      (PailgunChunkType.STDERR, stderr_isatty)
     )
     types, ttys = zip(*(stdio_writers))
 
@@ -215,7 +215,7 @@ def finalizer():
   def nailgunned_stdio(cls, sock, env, handle_stdin=True):
     """Redirects stdio to the connected socket speaking the nailgun protocol."""
     # Determine output tty capabilities from the environment.
-    stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env(env)
+    stdin_isatty, stdout_isatty, stderr_isatty = PailgunProtocol.isatty_from_env(env)
     is_tty_capable = all((stdin_isatty, stdout_isatty, stderr_isatty))
 
     if is_tty_capable:
@@ -294,8 +294,8 @@ def post_fork_child(self):
 
     # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
     # they can send signals (e.g. SIGINT) to all processes in the runners process group.
-    NailgunProtocol.send_pid(self._socket, os.getpid())
-    NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
+    PailgunProtocol.send_pid(self._socket, os.getpid())
+    PailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1)
 
     # Stop the services that were paused pre-fork.
     for service in self._services.services:
@@ -307,6 +307,8 @@ def post_fork_child(self):
       try:
         # Setup the Exiter's finalizer.
         self._exiter.set_finalizer(finalizer)
+        # TODO: ???
+        # self._exiter.set_except_hook(sys.stderr)
 
         # Clean global state.
         clean_global_runtime_state(reset_subsystem=True)

src/python/pants/bin/local_pants_runner.py

@@ -154,16 +154,34 @@ def __init__(self, build_root, exiter, options, options_bootstrapper, build_conf
     self._profile_path = profile_path
 
     self._run_start_time = None
+    self._run_tracker = None
+    self._reporting = None
+    self._repro = None
+
     self._global_options = options.for_global_scope()
 
   def set_start_time(self, start_time):
+    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
+    run_tracker = RunTracker.global_instance()
+    reporting = Reporting.global_instance()
+    reporting.initialize(run_tracker, start_time)
+
+    # Capture a repro of the 'before' state for this build, if needed.
+    repro = Reproducer.global_instance().create_repro()
+    if repro:
+      repro.capture(run_tracker.run_info.get_as_dict())
+
+    # Set all these fields at once for exception-safety.
     self._run_start_time = start_time
+    self._run_tracker = run_tracker
+    self._reporting = reporting
+    self._repro = repro
 
   def run(self):
     with maybe_profiled(self._profile_path):
       self._run()
 
-  def _maybe_run_v1(self, run_tracker, reporting):
+  def _maybe_run_v1(self):
     if not self._global_options.v1:
       return 0
 
@@ -172,8 +190,8 @@ def _maybe_run_v1(self, run_tracker, reporting):
       self._build_root,
       self._options,
       self._build_config,
-      run_tracker,
-      reporting,
+      self._run_tracker,
+      self._reporting,
       self._graph_session,
       self._target_roots,
       self._exiter
@@ -217,25 +235,15 @@ def _compute_final_exit_code(*codes):
     return max_code
 
   def _run(self):
-    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
-    run_tracker = RunTracker.global_instance()
-    reporting = Reporting.global_instance()
-    reporting.initialize(run_tracker, self._run_start_time)
-
     try:
-      # Capture a repro of the 'before' state for this build, if needed.
-      repro = Reproducer.global_instance().create_repro()
-      if repro:
-        repro.capture(run_tracker.run_info.get_as_dict())
-
       engine_result = self._maybe_run_v2()
-      goal_runner_result = self._maybe_run_v1(run_tracker, reporting)
+      goal_runner_result = self._maybe_run_v1()
 
-      if repro:
+      if self._repro:
         # TODO: Have Repro capture the 'after' state (as a diff) as well?
-        repro.log_location_of_repro_file()
+        self._repro.log_location_of_repro_file()
     finally:
-      run_tracker_result = run_tracker.end()
+      run_tracker_result = self._run_tracker.end()
 
     final_exit_code = self._compute_final_exit_code(
       engine_result,

src/python/pants/bin/remote_pants_runner.py

@@ -11,15 +11,16 @@
 from builtins import object, str
 from contextlib import contextmanager
 
-from future.utils import raise_with_traceback
+from future.utils import binary_type, raise_with_traceback
 
 from pants.base.exception_sink import ExceptionSink
 from pants.console.stty_utils import STTYSettings
-from pants.java.nailgun_client import NailgunClient
-from pants.java.nailgun_protocol import NailgunProtocol
+from pants.java.nailgun_client import NailgunClient, NailgunClientSession, PailgunClient
+from pants.java.nailgun_protocol import PailgunProtocol
 from pants.pantsd.pants_daemon import PantsDaemon
 from pants.util.collections import combined_dict
 from pants.util.dirutil import maybe_read_file
+from pants.util.osutil import safe_kill
 
 
 logger = logging.getLogger(__name__)
@@ -37,7 +38,7 @@ class Terminated(Exception):
   PANTS_COMMAND = 'pants'
   RECOVERABLE_EXCEPTIONS = (
     NailgunClient.NailgunConnectionError,
-    NailgunClient.NailgunExecutionError
+    NailgunClient.NailgunExecutionError,
   )
 
   def __init__(self, exiter, args, env, options_bootstrapper, stdin=None, stdout=None, stderr=None):
@@ -61,20 +62,25 @@ def __init__(self, exiter, args, env, options_bootstrapper, stdin=None, stdout=N
     self._stderr = stderr or sys.stderr
 
   @contextmanager
-  def _trapped_signals(self, client):
+  def _trapped_signals(self, client_handle):
     """A contextmanager that overrides the SIGINT (control-c) and SIGQUIT (control-\\) handlers
     and handles them remotely."""
+    remote_info = client_handle.remote_process_info
+    pid = remote_info.pid
+    pgrp = remote_info.pgrp
+
     def handle_control_c(signum, frame):
-      client.maybe_send_signal(signum, include_pgrp=True)
+      safe_kill(pgrp, signum)
+      safe_kill(pid, signum)
 
     existing_sigint_handler = signal.signal(signal.SIGINT, handle_control_c)
     # N.B. SIGQUIT will abruptly kill the pantsd-runner, which will shut down the other end
     # of the Pailgun connection - so we send a gentler SIGINT here instead.
     existing_sigquit_handler = signal.signal(signal.SIGQUIT, handle_control_c)
 
     # Retry interrupted system calls.
-    signal.siginterrupt(signal.SIGINT, False)
-    signal.siginterrupt(signal.SIGQUIT, False)
+    signal.siginterrupt(signal.SIGINT, True)
+    signal.siginterrupt(signal.SIGQUIT, True)
     try:
       yield
     finally:
@@ -136,23 +142,35 @@ def _run_pants_with_retry(self, pantsd_handle, retries=3):
 
   def _connect_and_execute(self, port):
     # Merge the nailgun TTY capability environment variables with the passed environment dict.
-    ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr)
+    ng_env = PailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr)
     modified_env = combined_dict(self._env, ng_env)
     modified_env['PANTSD_RUNTRACKER_CLIENT_START_TIME'] = str(self._start_time)
 
     assert isinstance(port, int), 'port {} is not an integer!'.format(port)
 
-    # Instantiate a NailgunClient.
-    client = NailgunClient(port=port,
-                           ins=self._stdin,
-                           out=self._stdout,
-                           err=self._stderr,
-                           exit_on_broken_pipe=True,
-                           expects_pid=True)
-
-    with self._trapped_signals(client), STTYSettings.preserved():
-      # Execute the command on the pailgun.
-      result = client.execute(self.PANTS_COMMAND, *self._args, **modified_env)
+    # Instantiate a PailgunClient.
+    req = NailgunClient.NailgunClientRequest(
+      port=port,
+      ins=self._stdin,
+      out=self._stdout,
+      err=self._stderr,
+      exit_on_broken_pipe=True)
+    client = PailgunClient(NailgunClient(req))
+
+    exe_request = NailgunClientSession.NailgunClientSessionExecutionRequest(
+      main_class=binary_type(self.PANTS_COMMAND),
+      cwd=None,
+      arguments=tuple(self._args),
+      environment=dict(**modified_env))
+
+    with client.remote_pants_session(exe_request) as handle,\
+         self._trapped_signals(handle),\
+         STTYSettings.preserved():
+      try:
+        # Execute the command on the pailgun.
+        result = client.execute(self.PANTS_COMMAND, *self._args, **modified_env)
+      finally:
+        client.send_terminate()
 
     # Exit.
     self._exiter.exit(result)

src/python/pants/java/BUILD

@@ -23,7 +23,8 @@ python_library(
     '3rdparty/python:future',
     ':nailgun_io',
     ':nailgun_protocol',
-    'src/python/pants/util:socket'
+    'src/python/pants/util:objects',
+    'src/python/pants/util:socket',
   ]
 )
 
@@ -43,6 +44,7 @@ python_library(
   dependencies = [
     '3rdparty/python:future',
     '3rdparty/python:six',
+    'src/python/pants/util:osutil',
   ]
 )