Skip to content

Commit

Permalink
[pantsd] Launch the daemon via the thin client.
Browse files Browse the repository at this point in the history
  • Loading branch information
kwlzn committed Oct 9, 2017
1 parent 89feae2 commit 354bc37
Show file tree
Hide file tree
Showing 39 changed files with 531 additions and 487 deletions.
2 changes: 1 addition & 1 deletion src/python/pants/backend/jvm/tasks/nailgun_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@

from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
from pants.base.exceptions import TaskError
from pants.init.subprocess import Subprocess
from pants.java import util
from pants.java.executor import SubprocessExecutor
from pants.java.jar.jar_dependency import JarDependency
from pants.java.nailgun_executor import NailgunExecutor, NailgunProcessGroup
from pants.pantsd.subsystem.subprocess import Subprocess
from pants.task.task import Task, TaskBase


Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/bin/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@ python_library(
'src/python/pants/init',
'src/python/pants/option',
'src/python/pants/reporting',
'src/python/pants/pantsd:pants_daemon_launcher',
'src/python/pants/scm/subsystems:changed',
'src/python/pants/subsystem',
'src/python/pants/task',
'src/python/pants/util:contextutil',
'src/python/pants/util:collections',
'src/python/pants/util:dirutil',
'src/python/pants/util:filtering',
'src/python/pants/util:memo',
Expand Down
20 changes: 5 additions & 15 deletions src/python/pants/bin/goal_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from pants.goal.goal import Goal
from pants.goal.run_tracker import RunTracker
from pants.help.help_printer import HelpPrinter
from pants.init.pants_daemon_launcher import PantsDaemonLauncher
from pants.init.subprocess import Subprocess
from pants.init.target_roots import TargetRoots
from pants.java.nailgun_executor import NailgunProcessGroup
from pants.reporting.reporting import Reporting
Expand Down Expand Up @@ -152,14 +152,7 @@ def generate_targets(specs):

return list(generate_targets(specs))

def _maybe_launch_pantsd(self, pantsd_launcher):
"""Launches pantsd if configured to do so."""
if self._global_options.enable_pantsd:
# Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]):
pantsd_launcher.maybe_launch()

def _setup_context(self, pantsd_launcher):
def _setup_context(self):
with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
self._build_graph, self._address_mapper, spec_roots = self._init_graph(
self._global_options.enable_v2_engine,
Expand Down Expand Up @@ -189,15 +182,12 @@ def _setup_context(self, pantsd_launcher):
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
invalidation_report=invalidation_report,
pantsd_launcher=pantsd_launcher)
invalidation_report=invalidation_report)
return goals, context

def setup(self):
pantsd_launcher = PantsDaemonLauncher.Factory.global_instance().create(EngineInitializer)
self._maybe_launch_pantsd(pantsd_launcher)
self._handle_help(self._help_request)
goals, context = self._setup_context(pantsd_launcher)
goals, context = self._setup_context()
return GoalRunner(context=context,
goals=goals,
run_tracker=self._run_tracker,
Expand Down Expand Up @@ -234,7 +224,7 @@ def subsystems(cls):
RunTracker,
Changed.Factory,
BinaryUtil.Factory,
PantsDaemonLauncher.Factory,
Subprocess.Factory
}

def _execute_engine(self):
Expand Down
37 changes: 16 additions & 21 deletions src/python/pants/bin/pants_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,29 +29,24 @@ def __init__(self, exiter, args=None, env=None):
self._args = args or sys.argv
self._env = env or os.environ

def _run(self, is_remote, exiter, args, env, process_metadata_dir=None, options_bootstrapper=None):
if is_remote:
def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()

if bootstrap_options.enable_pantsd:
try:
return RemotePantsRunner(exiter, args, env, process_metadata_dir).run()
except RemotePantsRunner.RECOVERABLE_EXCEPTIONS as e:
# N.B. RemotePantsRunner will raise one of RECOVERABLE_EXCEPTIONS in the event we
# encounter a failure while discovering or initially connecting to the pailgun. In
# this case, we fall back to LocalPantsRunner which seamlessly executes the requested
# run and bootstraps pantsd for use in subsequent runs.
logger.debug('caught client exception: {!r}, falling back to LocalPantsRunner'.format(e))
return RemotePantsRunner(self._exiter,
self._args,
self._env,
bootstrap_options.pants_subprocessdir,
bootstrap_options).run()
except RemotePantsRunner.Fallback as e:
logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e))

# N.B. Inlining this import speeds up the python thin client run by about 100ms.
from pants.bin.local_pants_runner import LocalPantsRunner

return LocalPantsRunner(exiter, args, env, options_bootstrapper=options_bootstrapper).run()

def run(self):
options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()

return self._run(is_remote=global_bootstrap_options.enable_pantsd,
exiter=self._exiter,
args=self._args,
env=self._env,
process_metadata_dir=global_bootstrap_options.pants_subprocessdir,
options_bootstrapper=options_bootstrapper)
return LocalPantsRunner(self._exiter,
self._args,
self._env,
options_bootstrapper=options_bootstrapper).run()
70 changes: 52 additions & 18 deletions src/python/pants/bin/remote_pants_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,30 +5,41 @@
from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

import logging
import signal
import sys
from contextlib import contextmanager

from pants.bin.engine_initializer import EngineInitializer
from pants.java.nailgun_client import NailgunClient
from pants.java.nailgun_protocol import NailgunProtocol
from pants.pantsd.process_manager import ProcessMetadataManager
from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher
from pants.util.collections import combined_dict


logger = logging.getLogger(__name__)


class RemotePantsRunner(object):
"""A thin client variant of PantsRunner."""

class PortNotFound(Exception): pass
class Fallback(Exception):
"""Raised when fallback to an alternate execution mode is requested."""

class PortNotFound(Exception):
"""Raised when the pailgun port can't be found."""

PANTS_COMMAND = 'pants'
RECOVERABLE_EXCEPTIONS = (PortNotFound, NailgunClient.NailgunConnectionError)

def __init__(self, exiter, args, env, process_metadata_dir=None,
stdin=None, stdout=None, stderr=None):
def __init__(self, exiter, args, env, process_metadata_dir,
bootstrap_options, stdin=None, stdout=None, stderr=None):
"""
:param Exiter exiter: The Exiter instance to use for this run.
:param list args: The arguments (e.g. sys.argv) for this run.
:param dict env: The environment (e.g. os.environ) for this run.
:param str process_metadata_dir: The directory in which process metadata is kept.
:param Options bootstrap_options: The Options bag containing the bootstrap options.
:param file stdin: The stream representing stdin.
:param file stdout: The stream representing stdout.
:param file stderr: The stream representing stderr.
Expand All @@ -37,17 +48,11 @@ def __init__(self, exiter, args, env, process_metadata_dir=None,
self._args = args
self._env = env
self._process_metadata_dir = process_metadata_dir
self._bootstrap_options = bootstrap_options
self._stdin = stdin or sys.stdin
self._stdout = stdout or sys.stdout
self._stderr = stderr or sys.stderr
self._port = self._retrieve_pailgun_port()
if not self._port:
raise self.PortNotFound('unable to locate pailgun port!')

@staticmethod
def _combine_dicts(*dicts):
"""Combine one or more dicts into a new, unified dict (dicts to the right take precedence)."""
return {k: v for d in dicts for k, v in d.items()}
self._launcher = PantsDaemonLauncher(self._bootstrap_options, EngineInitializer)

@contextmanager
def _trapped_control_c(self, client):
Expand All @@ -62,17 +67,36 @@ def handle_control_c(signum, frame):
finally:
signal.signal(signal.SIGINT, existing_sigint_handler)

def _retrieve_pailgun_port(self):
return ProcessMetadataManager(
self._process_metadata_dir).read_metadata_by_name('pantsd', 'socket_pailgun', int)
def _setup_logging(self):
"""Sets up basic stdio logging for the thin client."""
log_level = logging.getLevelName(self._bootstrap_options.level.upper())

def run(self, args=None):
formatter = logging.Formatter('%(levelname)s] %(message)s')
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(log_level)
handler.setFormatter(formatter)

root = logging.getLogger()
root.setLevel(log_level)
root.addHandler(handler)

def _find_or_launch_pantsd(self):
"""Launches pantsd if configured to do so.
:returns: The port pantsd can be found on.
:rtype: int
"""
return self._launcher.maybe_launch()

def _connect_and_execute(self, port):
# Merge the nailgun TTY capability environment variables with the passed environment dict.
ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr)
modified_env = self._combine_dicts(self._env, ng_env)
modified_env = combined_dict(self._env, ng_env)

assert isinstance(port, int), 'port {} is not an integer!'.format(port)

# Instantiate a NailgunClient.
client = NailgunClient(port=self._port,
client = NailgunClient(port=port,
ins=self._stdin,
out=self._stdout,
err=self._stderr,
Expand All @@ -84,3 +108,13 @@ def run(self, args=None):

# Exit.
self._exiter.exit(result)

def run(self, args=None):
self._setup_logging()
port = self._find_or_launch_pantsd()

logger.debug('connecting to pailgun on port {}'.format(port))
try:
self._connect_and_execute(port)
except self.RECOVERABLE_EXCEPTIONS as e:
raise self.Fallback(e)
3 changes: 2 additions & 1 deletion src/python/pants/core_tasks/pantsd_kill.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
unicode_literals, with_statement)

from pants.base.exceptions import TaskError
from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher
from pants.pantsd.process_manager import ProcessManager
from pants.task.task import Task

Expand All @@ -15,6 +16,6 @@ class PantsDaemonKill(Task):

def execute(self):
try:
self.context.pantsd_launcher.terminate()
PantsDaemonLauncher(self.get_options()).terminate()
except ProcessManager.NonResponsiveProcess as e:
raise TaskError('failure while terminating pantsd: {}'.format(e))
2 changes: 1 addition & 1 deletion src/python/pants/core_tasks/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def register_goals():
# Pantsd.
kill_pantsd = task(name='kill-pantsd', action=PantsDaemonKill)
kill_pantsd.install()
kill_pantsd.install('clean-all')
kill_pantsd.install('clean-all', first=True)

# Reporting server.
# TODO: The reporting server should be subsumed into pantsd, and not run via a task.
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/engine/native.py
Original file line number Diff line number Diff line change
Expand Up @@ -561,9 +561,9 @@ class Native(object):
"""Encapsulates fetching a platform specific version of the native portion of the engine."""

@staticmethod
def create(options):
def create(bootstrap_options):
""":param options: Any object that provides access to bootstrap option values."""
return Native(options.native_engine_visualize_to)
return Native(bootstrap_options.native_engine_visualize_to)

def __init__(self, visualize_to_dir):
"""
Expand Down
7 changes: 1 addition & 6 deletions src/python/pants/goal/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def fatal(self, *msg_elements):
def __init__(self, options, run_tracker, target_roots,
requested_goals=None, target_base=None, build_graph=None,
build_file_parser=None, address_mapper=None, console_outstream=None, scm=None,
workspace=None, invalidation_report=None, pantsd_launcher=None):
workspace=None, invalidation_report=None):
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
Expand All @@ -80,7 +80,6 @@ def __init__(self, options, run_tracker, target_roots,
self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
self._pantsd_launcher = pantsd_launcher

@property
def options(self):
Expand Down Expand Up @@ -151,10 +150,6 @@ def workspace(self):
def invalidation_report(self):
return self._invalidation_report

@property
def pantsd_launcher(self):
return self._pantsd_launcher

def __str__(self):
ident = Target.identify(self.targets())
return 'Context(id:{}, targets:{})'.format(ident, self.targets())
Expand Down
7 changes: 0 additions & 7 deletions src/python/pants/init/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).


python_library(
dependencies=[
':plugins',
Expand All @@ -21,12 +20,6 @@ python_library(
'src/python/pants/goal:run_tracker',
'src/python/pants/logging',
'src/python/pants/option',
'src/python/pants/pantsd:pants_daemon',
'src/python/pants/pantsd/service:fs_event_service',
'src/python/pants/pantsd/service:pailgun_service',
'src/python/pants/pantsd/service:scheduler_service',
'src/python/pants/pantsd/subsystem:subprocess',
'src/python/pants/pantsd/subsystem:watchman_launcher',
'src/python/pants/process',
'src/python/pants/python',
'src/python/pants/subsystem',
Expand Down
Loading

0 comments on commit 354bc37

Please sign in to comment.