diff --git a/src/python/pants/backend/jvm/tasks/nailgun_task.py b/src/python/pants/backend/jvm/tasks/nailgun_task.py index 7d0fd4a134b..8160b8489fc 100644 --- a/src/python/pants/backend/jvm/tasks/nailgun_task.py +++ b/src/python/pants/backend/jvm/tasks/nailgun_task.py @@ -9,11 +9,11 @@ from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin from pants.base.exceptions import TaskError +from pants.init.subprocess import Subprocess from pants.java import util from pants.java.executor import SubprocessExecutor from pants.java.jar.jar_dependency import JarDependency from pants.java.nailgun_executor import NailgunExecutor, NailgunProcessGroup -from pants.pantsd.subsystem.subprocess import Subprocess from pants.task.task import Task, TaskBase diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 3aba4107c20..2cba8ac8670 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -34,10 +34,12 @@ python_library( 'src/python/pants/init', 'src/python/pants/option', 'src/python/pants/reporting', + 'src/python/pants/pantsd:pants_daemon_launcher', 'src/python/pants/scm/subsystems:changed', 'src/python/pants/subsystem', 'src/python/pants/task', 'src/python/pants/util:contextutil', + 'src/python/pants/util:collections', 'src/python/pants/util:dirutil', 'src/python/pants/util:filtering', 'src/python/pants/util:memo', diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py index 09eb28f8922..b7c36deee44 100644 --- a/src/python/pants/bin/goal_runner.py +++ b/src/python/pants/bin/goal_runner.py @@ -23,7 +23,7 @@ from pants.goal.goal import Goal from pants.goal.run_tracker import RunTracker from pants.help.help_printer import HelpPrinter -from pants.init.pants_daemon_launcher import PantsDaemonLauncher +from pants.init.subprocess import Subprocess from pants.init.target_roots import TargetRoots from pants.java.nailgun_executor import NailgunProcessGroup from pants.reporting.reporting import Reporting @@ -152,14 +152,7 @@ def generate_targets(specs): return list(generate_targets(specs)) - def _maybe_launch_pantsd(self, pantsd_launcher): - """Launches pantsd if configured to do so.""" - if self._global_options.enable_pantsd: - # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on. - with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]): - pantsd_launcher.maybe_launch() - - def _setup_context(self, pantsd_launcher): + def _setup_context(self): with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._build_graph, self._address_mapper, spec_roots = self._init_graph( self._global_options.enable_v2_engine, @@ -189,15 +182,12 @@ def _setup_context(self, pantsd_launcher): build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, - invalidation_report=invalidation_report, - pantsd_launcher=pantsd_launcher) + invalidation_report=invalidation_report) return goals, context def setup(self): - pantsd_launcher = PantsDaemonLauncher.Factory.global_instance().create(EngineInitializer) - self._maybe_launch_pantsd(pantsd_launcher) self._handle_help(self._help_request) - goals, context = self._setup_context(pantsd_launcher) + goals, context = self._setup_context() return GoalRunner(context=context, goals=goals, run_tracker=self._run_tracker, @@ -234,7 +224,7 @@ def subsystems(cls): RunTracker, Changed.Factory, BinaryUtil.Factory, - PantsDaemonLauncher.Factory, + Subprocess.Factory } def _execute_engine(self): diff --git a/src/python/pants/bin/pants_runner.py b/src/python/pants/bin/pants_runner.py index 83f09ac171f..895d371b19c 100644 --- a/src/python/pants/bin/pants_runner.py +++ b/src/python/pants/bin/pants_runner.py @@ -29,29 +29,24 @@ def __init__(self, exiter, args=None, env=None): self._args = args or sys.argv self._env = env or os.environ - def _run(self, is_remote, exiter, args, env, process_metadata_dir=None, options_bootstrapper=None): - if is_remote: + def run(self): + options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) + bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() + + if bootstrap_options.enable_pantsd: try: - return RemotePantsRunner(exiter, args, env, process_metadata_dir).run() - except RemotePantsRunner.RECOVERABLE_EXCEPTIONS as e: - # N.B. RemotePantsRunner will raise one of RECOVERABLE_EXCEPTIONS in the event we - # encounter a failure while discovering or initially connecting to the pailgun. In - # this case, we fall back to LocalPantsRunner which seamlessly executes the requested - # run and bootstraps pantsd for use in subsequent runs. - logger.debug('caught client exception: {!r}, falling back to LocalPantsRunner'.format(e)) + return RemotePantsRunner(self._exiter, + self._args, + self._env, + bootstrap_options.pants_subprocessdir, + bootstrap_options).run() + except RemotePantsRunner.Fallback as e: + logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e)) # N.B. Inlining this import speeds up the python thin client run by about 100ms. from pants.bin.local_pants_runner import LocalPantsRunner - return LocalPantsRunner(exiter, args, env, options_bootstrapper=options_bootstrapper).run() - - def run(self): - options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args) - global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope() - - return self._run(is_remote=global_bootstrap_options.enable_pantsd, - exiter=self._exiter, - args=self._args, - env=self._env, - process_metadata_dir=global_bootstrap_options.pants_subprocessdir, - options_bootstrapper=options_bootstrapper) + return LocalPantsRunner(self._exiter, + self._args, + self._env, + options_bootstrapper=options_bootstrapper).run() diff --git a/src/python/pants/bin/remote_pants_runner.py b/src/python/pants/bin/remote_pants_runner.py index f0c9c7816eb..47e07fe99b0 100644 --- a/src/python/pants/bin/remote_pants_runner.py +++ b/src/python/pants/bin/remote_pants_runner.py @@ -5,30 +5,41 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import logging import signal import sys from contextlib import contextmanager +from pants.bin.engine_initializer import EngineInitializer from pants.java.nailgun_client import NailgunClient from pants.java.nailgun_protocol import NailgunProtocol -from pants.pantsd.process_manager import ProcessMetadataManager +from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher +from pants.util.collections import combined_dict + + +logger = logging.getLogger(__name__) class RemotePantsRunner(object): """A thin client variant of PantsRunner.""" - class PortNotFound(Exception): pass + class Fallback(Exception): + """Raised when fallback to an alternate execution mode is requested.""" + + class PortNotFound(Exception): + """Raised when the pailgun port can't be found.""" PANTS_COMMAND = 'pants' RECOVERABLE_EXCEPTIONS = (PortNotFound, NailgunClient.NailgunConnectionError) - def __init__(self, exiter, args, env, process_metadata_dir=None, - stdin=None, stdout=None, stderr=None): + def __init__(self, exiter, args, env, process_metadata_dir, + bootstrap_options, stdin=None, stdout=None, stderr=None): """ :param Exiter exiter: The Exiter instance to use for this run. :param list args: The arguments (e.g. sys.argv) for this run. :param dict env: The environment (e.g. os.environ) for this run. :param str process_metadata_dir: The directory in which process metadata is kept. + :param Options bootstrap_options: The Options bag containing the bootstrap options. :param file stdin: The stream representing stdin. :param file stdout: The stream representing stdout. :param file stderr: The stream representing stderr. @@ -37,17 +48,11 @@ def __init__(self, exiter, args, env, process_metadata_dir=None, self._args = args self._env = env self._process_metadata_dir = process_metadata_dir + self._bootstrap_options = bootstrap_options self._stdin = stdin or sys.stdin self._stdout = stdout or sys.stdout self._stderr = stderr or sys.stderr - self._port = self._retrieve_pailgun_port() - if not self._port: - raise self.PortNotFound('unable to locate pailgun port!') - - @staticmethod - def _combine_dicts(*dicts): - """Combine one or more dicts into a new, unified dict (dicts to the right take precedence).""" - return {k: v for d in dicts for k, v in d.items()} + self._launcher = PantsDaemonLauncher(self._bootstrap_options, EngineInitializer) @contextmanager def _trapped_control_c(self, client): @@ -62,17 +67,36 @@ def handle_control_c(signum, frame): finally: signal.signal(signal.SIGINT, existing_sigint_handler) - def _retrieve_pailgun_port(self): - return ProcessMetadataManager( - self._process_metadata_dir).read_metadata_by_name('pantsd', 'socket_pailgun', int) + def _setup_logging(self): + """Sets up basic stdio logging for the thin client.""" + log_level = logging.getLevelName(self._bootstrap_options.level.upper()) - def run(self, args=None): + formatter = logging.Formatter('%(levelname)s] %(message)s') + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(log_level) + handler.setFormatter(formatter) + + root = logging.getLogger() + root.setLevel(log_level) + root.addHandler(handler) + + def _find_or_launch_pantsd(self): + """Launches pantsd if configured to do so. + + :returns: The port pantsd can be found on. + :rtype: int + """ + return self._launcher.maybe_launch() + + def _connect_and_execute(self, port): # Merge the nailgun TTY capability environment variables with the passed environment dict. ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr) - modified_env = self._combine_dicts(self._env, ng_env) + modified_env = combined_dict(self._env, ng_env) + + assert isinstance(port, int), 'port {} is not an integer!'.format(port) # Instantiate a NailgunClient. - client = NailgunClient(port=self._port, + client = NailgunClient(port=port, ins=self._stdin, out=self._stdout, err=self._stderr, @@ -84,3 +108,13 @@ def run(self, args=None): # Exit. self._exiter.exit(result) + + def run(self, args=None): + self._setup_logging() + port = self._find_or_launch_pantsd() + + logger.debug('connecting to pailgun on port {}'.format(port)) + try: + self._connect_and_execute(port) + except self.RECOVERABLE_EXCEPTIONS as e: + raise self.Fallback(e) diff --git a/src/python/pants/core_tasks/pantsd_kill.py b/src/python/pants/core_tasks/pantsd_kill.py index 187a803b903..bccf2dd4c1f 100644 --- a/src/python/pants/core_tasks/pantsd_kill.py +++ b/src/python/pants/core_tasks/pantsd_kill.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) from pants.base.exceptions import TaskError +from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher from pants.pantsd.process_manager import ProcessManager from pants.task.task import Task @@ -15,6 +16,6 @@ class PantsDaemonKill(Task): def execute(self): try: - self.context.pantsd_launcher.terminate() + PantsDaemonLauncher(self.get_options()).terminate() except ProcessManager.NonResponsiveProcess as e: raise TaskError('failure while terminating pantsd: {}'.format(e)) diff --git a/src/python/pants/core_tasks/register.py b/src/python/pants/core_tasks/register.py index bf8bd3dfcb7..48c68f68cf5 100644 --- a/src/python/pants/core_tasks/register.py +++ b/src/python/pants/core_tasks/register.py @@ -59,7 +59,9 @@ def register_goals(): # Pantsd. kill_pantsd = task(name='kill-pantsd', action=PantsDaemonKill) kill_pantsd.install() - kill_pantsd.install('clean-all') + # Kill pantsd/watchman first, so that they're not using any files + # in .pants.d at the time of removal. + kill_pantsd.install('clean-all', first=True) # Reporting server. # TODO: The reporting server should be subsumed into pantsd, and not run via a task. diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py index 9a3468d9796..afa4077f814 100644 --- a/src/python/pants/engine/native.py +++ b/src/python/pants/engine/native.py @@ -561,9 +561,9 @@ class Native(object): """Encapsulates fetching a platform specific version of the native portion of the engine.""" @staticmethod - def create(options): + def create(bootstrap_options): """:param options: Any object that provides access to bootstrap option values.""" - return Native(options.native_engine_visualize_to) + return Native(bootstrap_options.native_engine_visualize_to) def __init__(self, visualize_to_dir): """ diff --git a/src/python/pants/goal/context.py b/src/python/pants/goal/context.py index 1abc11e0071..bb6143ae968 100644 --- a/src/python/pants/goal/context.py +++ b/src/python/pants/goal/context.py @@ -61,7 +61,7 @@ def fatal(self, *msg_elements): def __init__(self, options, run_tracker, target_roots, requested_goals=None, target_base=None, build_graph=None, build_file_parser=None, address_mapper=None, console_outstream=None, scm=None, - workspace=None, invalidation_report=None, pantsd_launcher=None): + workspace=None, invalidation_report=None): self._options = options self.build_graph = build_graph self.build_file_parser = build_file_parser @@ -80,7 +80,6 @@ def __init__(self, options, run_tracker, target_roots, self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None) self._replace_targets(target_roots) self._invalidation_report = invalidation_report - self._pantsd_launcher = pantsd_launcher @property def options(self): @@ -151,10 +150,6 @@ def workspace(self): def invalidation_report(self): return self._invalidation_report - @property - def pantsd_launcher(self): - return self._pantsd_launcher - def __str__(self): ident = Target.identify(self.targets()) return 'Context(id:{}, targets:{})'.format(ident, self.targets()) diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index 9b0bee315dd..4b11180e6c3 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -1,7 +1,6 @@ # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - python_library( dependencies=[ ':plugins', @@ -21,12 +20,6 @@ python_library( 'src/python/pants/goal:run_tracker', 'src/python/pants/logging', 'src/python/pants/option', - 'src/python/pants/pantsd:pants_daemon', - 'src/python/pants/pantsd/service:fs_event_service', - 'src/python/pants/pantsd/service:pailgun_service', - 'src/python/pants/pantsd/service:scheduler_service', - 'src/python/pants/pantsd/subsystem:subprocess', - 'src/python/pants/pantsd/subsystem:watchman_launcher', 'src/python/pants/process', 'src/python/pants/python', 'src/python/pants/subsystem', diff --git a/src/python/pants/init/pants_daemon_launcher.py b/src/python/pants/init/pants_daemon_launcher.py deleted file mode 100644 index dc675386c83..00000000000 --- a/src/python/pants/init/pants_daemon_launcher.py +++ /dev/null @@ -1,179 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import logging -import os - -from pants.base.build_environment import get_buildroot -from pants.binaries.binary_util import BinaryUtil -from pants.engine.native import Native -from pants.init.target_roots import TargetRoots -from pants.init.util import clean_global_runtime_state -from pants.pantsd.pants_daemon import PantsDaemon -from pants.pantsd.service.fs_event_service import FSEventService -from pants.pantsd.service.pailgun_service import PailgunService -from pants.pantsd.service.scheduler_service import SchedulerService -from pants.pantsd.subsystem.subprocess import Subprocess -from pants.pantsd.subsystem.watchman_launcher import WatchmanLauncher -from pants.process.lock import OwnerPrintingInterProcessFileLock -from pants.subsystem.subsystem import Subsystem -from pants.util.memo import testable_memoized_property - - -class PantsDaemonLauncher(object): - """A subsystem that manages the configuration and launching of pantsd.""" - - class Factory(Subsystem): - options_scope = 'pantsd' - - @classmethod - def register_options(cls, register): - register('--pailgun-host', advanced=True, default='127.0.0.1', - help='The host to bind the pants nailgun server to.') - register('--pailgun-port', advanced=True, type=int, default=0, - help='The port to bind the pants nailgun server to. Defaults to a random port.') - register('--log-dir', advanced=True, default=None, - help='The directory to log pantsd output to.') - register('--fs-event-detection', advanced=True, type=bool, - removal_version='1.5.0.dev0', - removal_hint='This option is now implied by `--enable-pantsd`.', - help='Whether or not to use filesystem event detection.') - register('--fs-event-workers', advanced=True, type=int, default=4, - help='The number of workers to use for the filesystem event service executor pool.') - - @classmethod - def subsystem_dependencies(cls): - return super(PantsDaemonLauncher.Factory, - cls).subsystem_dependencies() + (WatchmanLauncher.Factory, Subprocess.Factory, - BinaryUtil.Factory) - - def create(self, engine_initializer=None): - """ - :param class engine_initializer: The class representing the EngineInitializer. Only necessary - for startup. - """ - build_root = get_buildroot() - options = self.global_instance().get_options() - return PantsDaemonLauncher(build_root=build_root, - engine_initializer=engine_initializer, - options=options) - - def __init__(self, - build_root, - engine_initializer, - options): - """ - :param str build_root: The path of the build root. - :param class engine_initializer: The class representing the EngineInitializer. - """ - self._build_root = build_root - self._engine_initializer = engine_initializer - - # The options we register directly. - self._pailgun_host = options.pailgun_host - self._pailgun_port = options.pailgun_port - self._log_dir = options.log_dir - self._fs_event_workers = options.fs_event_workers - - # Values derived from global options (which our scoped options inherit). - self._pants_workdir = options.pants_workdir - self._log_level = options.level.upper() - self._pants_ignore_patterns = options.pants_ignore - self._build_ignore_patterns = options.build_ignore - self._exclude_target_regexp = options.exclude_target_regexp - self._subproject_roots = options.subproject_roots - # Native.create() reads global options, which, thanks to inheritance, it can - # read them via our scoped options. - self._native = Native.create(options) - # TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration. - - lock_location = os.path.join(self._build_root, '.pantsd.startup') - self._lock = OwnerPrintingInterProcessFileLock(lock_location) - self._logger = logging.getLogger(__name__) - - @testable_memoized_property - def pantsd(self): - return PantsDaemon( - self._build_root, - self._pants_workdir, - self._log_level, - self._native, - self._log_dir, - reset_func=clean_global_runtime_state - ) - - @testable_memoized_property - def watchman_launcher(self): - return WatchmanLauncher.Factory.global_instance().create() - - def _setup_services(self, watchman): - """Initialize pantsd services. - - :returns: A tuple of (`tuple` service_instances, `dict` port_map). - """ - # N.B. This inline import is currently necessary to avoid a circular reference in the import - # of LocalPantsRunner for use by DaemonPantsRunner. This is because LocalPantsRunner must - # ultimately import the pantsd services in order to itself launch pantsd. - from pants.bin.daemon_pants_runner import DaemonExiter, DaemonPantsRunner - - legacy_graph_helper = self._engine_initializer.setup_legacy_graph( - self._pants_ignore_patterns, - self._pants_workdir, - native=self._native, - build_ignore_patterns=self._build_ignore_patterns, - exclude_target_regexps=self._exclude_target_regexp, - subproject_roots=self._subproject_roots, - ) - - fs_event_service = FSEventService(watchman, self._build_root, self._fs_event_workers) - scheduler_service = SchedulerService(fs_event_service, legacy_graph_helper) - pailgun_service = PailgunService( - bind_addr=(self._pailgun_host, self._pailgun_port), - exiter_class=DaemonExiter, - runner_class=DaemonPantsRunner, - target_roots_class=TargetRoots, - scheduler_service=scheduler_service - ) - - return ( - # Use the schedulers reentrant lock as the daemon's global lock. - legacy_graph_helper.scheduler.lock, - # Services. - (fs_event_service, scheduler_service, pailgun_service), - # Port map. - dict(pailgun=pailgun_service.pailgun_port) - ) - - def _launch_pantsd(self): - # Launch Watchman (if so configured). - watchman = self.watchman_launcher.maybe_launch() - - # Initialize pantsd services. - lock, services, port_map = self._setup_services(watchman) - - # Setup and fork pantsd. - self.pantsd.set_lock(lock) - self.pantsd.set_services(services) - self.pantsd.set_socket_map(port_map) - self.pantsd.daemonize() - - # Wait up to 10 seconds for pantsd to write its pidfile so we can display the pid to the user. - self.pantsd.await_pid(10) - - def maybe_launch(self): - self._logger.debug('acquiring lock: {}'.format(self._lock)) - with self._lock: - if not self.pantsd.is_alive(): - self._logger.debug('launching pantsd') - self._launch_pantsd() - self._logger.debug('released lock: {}'.format(self._lock)) - - self._logger.debug('pantsd is running at pid {}'.format(self.pantsd.pid)) - - def terminate(self): - self.pantsd.terminate() - self.watchman_launcher.terminate() diff --git a/src/python/pants/pantsd/subsystem/subprocess.py b/src/python/pants/init/subprocess.py similarity index 100% rename from src/python/pants/pantsd/subsystem/subprocess.py rename to src/python/pants/init/subprocess.py diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 4bc3bdfded1..f5e9cdddca4 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -36,6 +36,9 @@ def register_bootstrap_options(cls, register): status as "bootstrap options" is only pertinent during option registration. """ buildroot = get_buildroot() + default_distdir_name = 'dist' + default_distdir = os.path.join(buildroot, default_distdir_name) + default_rel_distdir = '/{}/'.format(default_distdir_name) # Although logging supports the WARN level, its not documented and could conceivably be yanked. # Since pants has supported 'warn' since inception, leave the 'warn' choice as-is but explicitly @@ -89,8 +92,10 @@ def register_bootstrap_options(cls, register): default=os.path.join(buildroot, 'build-support'), help='Use support files from this dir.') register('--pants-distdir', advanced=True, metavar='', - default=os.path.join(buildroot, 'dist'), - help='Write end-product artifacts to this dir.') + default=default_distdir, + help='Write end-product artifacts to this dir. If you modify this path, you ' + 'should also update --build-ignore and --pants-ignore to include the ' + 'custom dist dir path as well.') register('--pants-subprocessdir', advanced=True, default=os.path.join(buildroot, '.pids'), help='The directory to use for tracking subprocess metadata, if any. This should ' 'live outside of the dir used by `--pants-workdir` to allow for tracking ' @@ -115,6 +120,25 @@ def register_bootstrap_options(cls, register): help='Read additional specs from this file, one per line') register('--verify-config', type=bool, default=True, help='Verify that all config file values correspond to known options.') + register('--build-ignore', advanced=True, type=list, fromfile=True, + default=['.*/', default_rel_distdir, 'bower_components/', + 'node_modules/', '*.egg-info/'], + help='Paths to ignore when identifying BUILD files. ' + 'This does not affect any other filesystem operations. ' + 'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).') + register('--pants-ignore', advanced=True, type=list, fromfile=True, + default=['.*/', default_rel_distdir], + help='Paths to ignore for all filesystem operations performed by pants ' + '(e.g. BUILD file scanning, glob matching, etc). ' + 'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). ' + 'This currently only affects the v2 engine. ' + 'To experiment with v2 engine, try --enable-v2-engine option.') + register('--exclude-target-regexp', advanced=True, type=list, default=[], + metavar='', + help='Exclude target roots that match these regexes.') + register('--subproject-roots', type=list, advanced=True, fromfile=True, default=[], + help='Paths that correspond with build roots for any subproject that this ' + 'project depends on.') # These logging options are registered in the bootstrap phase so that plugins can log during # registration and not so that their values can be interpolated in configs. @@ -158,6 +182,34 @@ def register_bootstrap_options(cls, register): help=('Maps output of uname for a machine to a binary search path. e.g. ' '{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux", "arm32"]}')) + # Pants Daemon options. + register('--pantsd-pailgun-host', advanced=True, default='127.0.0.1', + help='The host to bind the pants nailgun server to.') + register('--pantsd-pailgun-port', advanced=True, type=int, default=0, + help='The port to bind the pants nailgun server to. Defaults to a random port.') + register('--pantsd-log-dir', advanced=True, default=None, + help='The directory to log pantsd output to.') + register('--pantsd-fs-event-detection', advanced=True, type=bool, + removal_version='1.5.0.dev0', + removal_hint='This option is now implied by `--enable-pantsd`.', + help='Whether or not to use filesystem event detection.') + register('--pantsd-fs-event-workers', advanced=True, type=int, default=4, + help='The number of workers to use for the filesystem event service executor pool.') + + # Watchman options. + register('--watchman-version', advanced=True, default='4.5.0', help='Watchman version.') + register('--watchman-supportdir', advanced=True, default='bin/watchman', + help='Find watchman binaries under this dir. Used as part of the path to lookup ' + 'the binary with --binary-util-baseurls and --pants-bootstrapdir.') + register('--watchman-startup-timeout', type=float, advanced=True, default=30.0, + help='The watchman socket timeout (in seconds) for the initial `watch-project` command. ' + 'This may need to be set higher for larger repos due to watchman startup cost.') + register('--watchman-socket-timeout', type=float, advanced=True, default=5.0, + help='The watchman client socket timeout in seconds.') + register('--watchman-socket-path', type=str, advanced=True, default=None, + help='The path to the watchman UNIX socket. This can be overridden if the default ' + 'absolute path length exceeds the maximum allowed by the OS.') + @classmethod def register_options(cls, register): """Register options not tied to any particular task or subsystem.""" @@ -187,23 +239,6 @@ def register_options(cls, register): help="Constrain what Python interpreters to use. Uses Requirement format from " "pkg_resources, e.g. 'CPython>=2.7,<3' or 'PyPy'. By default, no constraints " "are used. Multiple constraints may be added. They will be ORed together.") - register('--exclude-target-regexp', advanced=True, type=list, default=[], - metavar='', - help='Exclude target roots that match these regexes.') - # Relative pants_distdir to buildroot. Requires --pants-distdir to be bootstrapped above first. - # e.g. '/dist/' - rel_distdir = '/{}/'.format(os.path.relpath(register.bootstrap.pants_distdir, get_buildroot())) - register('--build-ignore', advanced=True, type=list, fromfile=True, - default=['.*/', rel_distdir, 'bower_components/', 'node_modules/', '*.egg-info/'], - help='Paths to ignore when identifying BUILD files. ' - 'This does not affect any other filesystem operations. ' - 'Patterns use the gitignore pattern syntax (https://git-scm.com/docs/gitignore).') - register('--pants-ignore', advanced=True, type=list, fromfile=True, default=['.*/', rel_distdir], - help='Paths to ignore for all filesystem operations performed by pants ' - '(e.g. BUILD file scanning, glob matching, etc). ' - 'Patterns use the gitignore syntax (https://git-scm.com/docs/gitignore). ' - 'This currently only affects the v2 engine. ' - 'To experiment with v2 engine, try --enable-v2-engine option.') register('--fail-fast', advanced=True, type=bool, recursive=True, help='Exit as quickly as possible on error, rather than attempting to continue ' 'to process the non-erroneous subset of the input.') @@ -224,6 +259,3 @@ def register_options(cls, register): register('--lock', advanced=True, type=bool, default=True, help='Use a global lock to exclude other versions of pants from running during ' 'critical operations.') - register('--subproject-roots', type=list, advanced=True, fromfile=True, default=[], - help='Paths that correspond with build roots for any subproject that this ' - 'project depends on.') diff --git a/src/python/pants/pantsd/BUILD b/src/python/pants/pantsd/BUILD index 5499547678c..1c69d93ac51 100644 --- a/src/python/pants/pantsd/BUILD +++ b/src/python/pants/pantsd/BUILD @@ -7,7 +7,6 @@ python_library( dependencies = [ '3rdparty/python:psutil', 'src/python/pants/base:build_environment', - 'src/python/pants/pantsd/subsystem:subprocess', 'src/python/pants/util:dirutil', 'src/python/pants/util:process_handler', ] @@ -35,6 +34,16 @@ python_library( ] ) +python_library( + name = 'watchman_launcher', + sources = ['watchman_launcher.py'], + dependencies = [ + ':watchman', + 'src/python/pants/binaries:binary_util', + 'src/python/pants/util:memo', + ] +) + python_library( name = 'watchman_client', sources = ['watchman_client.py'], @@ -54,3 +63,21 @@ python_library( ':process_manager', ] ) + +python_library( + name = 'pants_daemon_launcher', + sources = ['pants_daemon_launcher.py'], + dependencies = [ + ':pants_daemon', + ':watchman_launcher', + 'src/python/pants/pantsd/service:fs_event_service', + 'src/python/pants/pantsd/service:pailgun_service', + 'src/python/pants/pantsd/service:scheduler_service', + 'src/python/pants/binaries:binary_util', + 'src/python/pants/base:build_environment', + 'src/python/pants/engine:native', + 'src/python/pants/init', + 'src/python/pants/process', + 'src/python/pants/util:memo' + ] +) diff --git a/src/python/pants/pantsd/pailgun_server.py b/src/python/pants/pantsd/pailgun_server.py index 6f81d10e30b..add43f2acaf 100644 --- a/src/python/pants/pantsd/pailgun_server.py +++ b/src/python/pants/pantsd/pailgun_server.py @@ -132,8 +132,7 @@ def process_request(self, request, client_address): try: # Attempt to handle a request with the handler under the context_lock. - with self._context_lock(): - handler.handle_request() + handler.handle_request() except Exception as e: # If that fails, (synchronously) handle the error with the error handler sans-fork. try: diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py index d3e5fb0a17a..9666cb285af 100644 --- a/src/python/pants/pantsd/pants_daemon.py +++ b/src/python/pants/pantsd/pants_daemon.py @@ -97,12 +97,13 @@ def set_socket_map(self, socket_map): def shutdown(self, service_thread_map): """Gracefully terminate all services and kill the main PantsDaemon loop.""" - for service, service_thread in service_thread_map.items(): - self._logger.info('terminating pantsd service: {}'.format(service)) - service.terminate() - service_thread.join() - self._logger.info('terminating pantsd') - self._kill_switch.set() + with self._lock: + for service, service_thread in service_thread_map.items(): + self._logger.info('terminating pantsd service: {}'.format(service)) + service.terminate() + service_thread.join() + self._logger.info('terminating pantsd') + self._kill_switch.set() @staticmethod def _close_fds(): @@ -164,6 +165,9 @@ def _run_services(self, services): self.shutdown(service_thread_map) raise self.StartupFailure('service {} failed to start, shutting down!'.format(service)) + # Once all services are started, write our pid. + self.write_pid() + # Monitor services. while not self.is_killed: for service, service_thread in service_thread_map.items(): diff --git a/src/python/pants/pantsd/pants_daemon_launcher.py b/src/python/pants/pantsd/pants_daemon_launcher.py new file mode 100644 index 00000000000..d3f4fa10c13 --- /dev/null +++ b/src/python/pants/pantsd/pants_daemon_launcher.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import logging +import os + +from pants.base.build_environment import get_buildroot +from pants.bin.daemon_pants_runner import DaemonExiter, DaemonPantsRunner +from pants.engine.native import Native +from pants.init.target_roots import TargetRoots +from pants.init.util import clean_global_runtime_state +from pants.pantsd.pants_daemon import PantsDaemon +from pants.pantsd.service.fs_event_service import FSEventService +from pants.pantsd.service.pailgun_service import PailgunService +from pants.pantsd.service.scheduler_service import SchedulerService +from pants.pantsd.watchman_launcher import WatchmanLauncher +from pants.process.lock import OwnerPrintingInterProcessFileLock +from pants.util.memo import testable_memoized_property + + +class PantsDaemonLauncher(object): + """An object that manages the configuration and lifecycle of pantsd.""" + + def __init__(self, bootstrap_options, engine_initializer=None): + """ + :param Options bootstrap_options: An Options object containing the bootstrap options. + :param class engine_initializer: The class representing the EngineInitializer. Only required + for startup. + """ + self._bootstrap_options = bootstrap_options + self._engine_initializer = engine_initializer + + self._pailgun_host = bootstrap_options.pantsd_pailgun_host + self._pailgun_port = bootstrap_options.pantsd_pailgun_port + self._log_dir = bootstrap_options.pantsd_log_dir + self._fs_event_workers = bootstrap_options.pantsd_fs_event_workers + self._pants_workdir = bootstrap_options.pants_workdir + self._log_level = bootstrap_options.level.upper() + self._pants_ignore_patterns = bootstrap_options.pants_ignore + self._build_ignore_patterns = bootstrap_options.build_ignore + self._exclude_target_regexp = bootstrap_options.exclude_target_regexp + self._subproject_roots = bootstrap_options.subproject_roots + self._metadata_base_dir = bootstrap_options.pants_subprocessdir + + # TODO: https://github.com/pantsbuild/pants/issues/3479 + self._build_root = get_buildroot() + self._native = Native.create(bootstrap_options) + self._logger = logging.getLogger(__name__) + self._lock = OwnerPrintingInterProcessFileLock( + os.path.join(self._build_root, '.pantsd.startup') + ) + + @testable_memoized_property + def pantsd(self): + return PantsDaemon( + self._build_root, + self._pants_workdir, + self._log_level, + self._native, + self._log_dir, + reset_func=clean_global_runtime_state, + metadata_base_dir=self._metadata_base_dir + ) + + @testable_memoized_property + def watchman_launcher(self): + return WatchmanLauncher.create(self._bootstrap_options) + + def _setup_services(self, watchman): + """Initialize pantsd services. + + :returns: A tuple of (`tuple` service_instances, `dict` port_map). + """ + legacy_graph_helper = self._engine_initializer.setup_legacy_graph( + self._pants_ignore_patterns, + self._pants_workdir, + native=self._native, + build_ignore_patterns=self._build_ignore_patterns, + exclude_target_regexps=self._exclude_target_regexp, + subproject_roots=self._subproject_roots, + ) + + fs_event_service = FSEventService(watchman, self._build_root, self._fs_event_workers) + scheduler_service = SchedulerService(fs_event_service, legacy_graph_helper) + pailgun_service = PailgunService( + bind_addr=(self._pailgun_host, self._pailgun_port), + exiter_class=DaemonExiter, + runner_class=DaemonPantsRunner, + target_roots_class=TargetRoots, + scheduler_service=scheduler_service + ) + + return ( + # Use the schedulers reentrant lock as the daemon's global lock. + legacy_graph_helper.scheduler.lock, + # Services. + (fs_event_service, scheduler_service, pailgun_service), + # Port map. + dict(pailgun=pailgun_service.pailgun_port) + ) + + def _launch_pantsd(self): + # Launch Watchman (if so configured). + watchman = self.watchman_launcher.maybe_launch() + + # Initialize pantsd services. + lock, services, port_map = self._setup_services(watchman) + + # Setup and fork pantsd. + self.pantsd.set_lock(lock) + self.pantsd.set_services(services) + self.pantsd.set_socket_map(port_map) + # Defer pid writing until the daemon has fully spawned. + self.pantsd.daemonize(write_pid=False) + + # Wait up to 10 seconds for pantsd to write its pidfile so we can display the pid to the user. + self.pantsd.await_pid(10) + + def maybe_launch(self): + """Launches pantsd if not already running. + + :returns: The port that pantsd is listening on. + :rtype: int + """ + self._logger.debug('acquiring lock: {}'.format(self._lock)) + with self._lock: + if not self.pantsd.is_alive(): + self._logger.debug('launching pantsd') + self._launch_pantsd() + listening_port = self.pantsd.read_named_socket('pailgun', int) + pantsd_pid = self.pantsd.pid + self._logger.debug('released lock: {}'.format(self._lock)) + self._logger.debug('pantsd is running at pid {}, pailgun port is {}' + .format(pantsd_pid, listening_port)) + return listening_port + + def terminate(self): + self.pantsd.terminate() + self.watchman_launcher.terminate() diff --git a/src/python/pants/pantsd/process_manager.py b/src/python/pants/pantsd/process_manager.py index e49593db944..a545691b756 100644 --- a/src/python/pants/pantsd/process_manager.py +++ b/src/python/pants/pantsd/process_manager.py @@ -15,7 +15,7 @@ import psutil from pants.base.build_environment import get_buildroot -from pants.pantsd.subsystem.subprocess import Subprocess +from pants.init.subprocess import Subprocess from pants.util.dirutil import read_file, rm_rf, safe_file_dump, safe_mkdir from pants.util.process_handler import subprocess @@ -301,8 +301,9 @@ def await_socket(self, timeout): """Wait up to a given timeout for a process to write socket info.""" return self.await_metadata_by_name(self._name, 'socket', timeout, self._socket_type) - def write_pid(self, pid): + def write_pid(self, pid=None): """Write the current processes PID to the pidfile location""" + pid = pid or os.getpid() self.write_metadata_by_name(self._name, 'pid', str(pid)) def write_socket(self, socket_info): @@ -313,6 +314,10 @@ def write_named_socket(self, socket_name, socket_info): """A multi-tenant, named alternative to ProcessManager.write_socket().""" self.write_metadata_by_name(self._name, 'socket_{}'.format(socket_name), str(socket_info)) + def read_named_socket(self, socket_name, socket_type): + """A multi-tenant, named alternative to ProcessManager.socket.""" + return self.read_metadata_by_name(self._name, 'socket_{}'.format(socket_name), socket_type) + def _as_process(self): """Returns a psutil `Process` object wrapping our pid. diff --git a/src/python/pants/pantsd/service/scheduler_service.py b/src/python/pants/pantsd/service/scheduler_service.py index a5af2162296..cacf4bc1c7a 100644 --- a/src/python/pants/pantsd/service/scheduler_service.py +++ b/src/python/pants/pantsd/service/scheduler_service.py @@ -80,13 +80,11 @@ def _process_event_queue(self): self._logger.debug('processing {} files for subscription {} (first_event={})' .format(len(files), subscription, is_initial_event)) - if is_initial_event: - # Once we've seen the initial watchman event, set the internal `Event`. - self._ready.set() - else: - # The first watchman event is a listing of all files - ignore it. + # The first watchman event is a listing of all files - ignore it. + if not is_initial_event: self._handle_batch_event(files) + if not self._ready.is_set(): self._ready.set() self._event_queue.task_done() def warm_product_graph(self, spec_roots): diff --git a/src/python/pants/pantsd/subsystem/BUILD b/src/python/pants/pantsd/subsystem/BUILD deleted file mode 100644 index 3a69b9c073d..00000000000 --- a/src/python/pants/pantsd/subsystem/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -python_library( - name = 'subprocess', - sources = ['subprocess.py'], - dependencies = [ - 'src/python/pants/option', - 'src/python/pants/subsystem', - ] -) - -python_library( - name = 'watchman_launcher', - sources = ['watchman_launcher.py'], - dependencies = [ - ':subprocess', - 'src/python/pants/binaries:binary_util', - 'src/python/pants/pantsd:watchman', - 'src/python/pants/subsystem:subsystem', - 'src/python/pants/util:memo', - ] -) diff --git a/src/python/pants/pantsd/subsystem/__init__.py b/src/python/pants/pantsd/subsystem/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/python/pants/pantsd/watchman.py b/src/python/pants/pantsd/watchman.py index 368639c17ff..bed189ba9a4 100644 --- a/src/python/pants/pantsd/watchman.py +++ b/src/python/pants/pantsd/watchman.py @@ -20,6 +20,9 @@ class Watchman(ProcessManager): """Watchman process manager and helper class.""" + class WatchmanCrash(Exception): + """Raised when Watchman crashes.""" + STARTUP_TIMEOUT_SECONDS = 30.0 SOCKET_TIMEOUT_SECONDS = 5.0 @@ -77,7 +80,6 @@ def _normalize_watchman_path(self, watchman_path): return os.path.abspath(watchman_path) def _maybe_init_metadata(self): - self._logger.debug('ensuring creation of directory: {}'.format(self._watchman_work_dir)) safe_mkdir(self._watchman_work_dir) # Initialize watchman with an empty, but valid statefile so it doesn't complain on startup. safe_file_dump(self._state_file, '{}') @@ -138,6 +140,15 @@ def launch(self): self.write_pid(pid) self.write_socket(self._sock_file) + def _attempt_set_timeout(self, timeout): + """Sets a timeout on the inner watchman client's socket.""" + try: + self.client.setTimeout(timeout) + except Exception: + self._logger.debug('failed to set post-startup watchman timeout to %s', self._timeout) + else: + self._logger.debug('set post-startup watchman timeout to %s', self._timeout) + def watch_project(self, path): """Issues the watch-project command to watchman to begin watching the buildroot. @@ -147,8 +158,7 @@ def watch_project(self, path): try: return self.client.query('watch-project', os.path.realpath(path)) finally: - self.client.setTimeout(self._timeout) - self._logger.debug('setting post-startup watchman timeout to %s', self._timeout) + self._attempt_set_timeout(self._timeout) def subscribed(self, build_root, handlers): """Bulk subscribe generator for StreamableWatchmanClient. @@ -164,13 +174,16 @@ def subscribed(self, build_root, handlers): self._logger.debug('watchman command_list is: {}'.format(command_list)) - for event in self.client.stream_query(command_list): - if event is None: - yield None, None - elif 'subscribe' in event: - self._logger.info('confirmed watchman subscription: {}'.format(event)) - yield None, None - elif 'subscription' in event: - yield event.get('subscription'), event - else: - self._logger.warning('encountered non-subscription event: {}'.format(event)) + try: + for event in self.client.stream_query(command_list): + if event is None: + yield None, None + elif 'subscribe' in event: + self._logger.info('confirmed watchman subscription: {}'.format(event)) + yield None, None + elif 'subscription' in event: + yield event.get('subscription'), event + else: + self._logger.warning('encountered non-subscription event: {}'.format(event)) + except self.client.WatchmanError as e: + raise self.WatchmanCrash(e) diff --git a/src/python/pants/pantsd/watchman_client.py b/src/python/pants/pantsd/watchman_client.py index 81c0e6671b4..1ccd546083d 100644 --- a/src/python/pants/pantsd/watchman_client.py +++ b/src/python/pants/pantsd/watchman_client.py @@ -14,6 +14,7 @@ class StreamableWatchmanClient(pywatchman.client): """A watchman client subclass that provides for interruptable unilateral queries.""" + WatchmanError = pywatchman.WatchmanError SocketTimeout = pywatchman.SocketTimeout def stream_query(self, commands): diff --git a/src/python/pants/pantsd/subsystem/watchman_launcher.py b/src/python/pants/pantsd/watchman_launcher.py similarity index 56% rename from src/python/pants/pantsd/subsystem/watchman_launcher.py rename to src/python/pants/pantsd/watchman_launcher.py index bc15cae8a49..3417a6df704 100644 --- a/src/python/pants/pantsd/subsystem/watchman_launcher.py +++ b/src/python/pants/pantsd/watchman_launcher.py @@ -8,52 +8,39 @@ import logging from pants.binaries.binary_util import BinaryUtil -from pants.pantsd.subsystem.subprocess import Subprocess from pants.pantsd.watchman import Watchman -from pants.subsystem.subsystem import Subsystem from pants.util.memo import testable_memoized_property class WatchmanLauncher(object): - """A subsystem that encapsulates access to Watchman.""" + """An object that manages the configuration and lifecycle of Watchman.""" - class Factory(Subsystem): - options_scope = 'watchman' - - @classmethod - def subsystem_dependencies(cls): - return (BinaryUtil.Factory, Subprocess.Factory) - - @classmethod - def register_options(cls, register): - register('--version', advanced=True, default='4.9.0', - help='Watchman version.') - register('--supportdir', advanced=True, default='bin/watchman', - help='Find watchman binaries under this dir. Used as part of the path to lookup ' - 'the binary with --binary-util-baseurls and --pants-bootstrapdir.') - register('--startup-timeout', type=float, advanced=True, default=Watchman.STARTUP_TIMEOUT_SECONDS, - help='The watchman socket timeout (in seconds) for the initial `watch-project` command. ' - 'This may need to be set higher for larger repos due to watchman startup cost.') - register('--socket-timeout', type=float, advanced=True, default=Watchman.SOCKET_TIMEOUT_SECONDS, - help='The watchman client socket timeout (in seconds).') - register('--socket-path', type=str, advanced=True, default=None, - help='The path to the watchman UNIX socket. This can be overridden if the default ' - 'absolute path length exceeds the maximum allowed by the OS.') + @classmethod + def create(cls, bootstrap_options): + """ + :param Options bootstrap_options: The bootstrap options bag. + """ + binary_util = BinaryUtil( + bootstrap_options.binaries_baseurls, + bootstrap_options.binaries_fetch_timeout_secs, + bootstrap_options.pants_bootstrapdir, + bootstrap_options.binaries_path_by_id + ) - def create(self): - binary_util = BinaryUtil.Factory.create() - options = self.get_options() - return WatchmanLauncher(binary_util, - options.pants_workdir, - options.level, - options.version, - options.supportdir, - options.startup_timeout, - options.socket_timeout, - options.socket_path) + return WatchmanLauncher( + binary_util, + bootstrap_options.pants_workdir, + bootstrap_options.level, + bootstrap_options.watchman_version, + bootstrap_options.watchman_supportdir, + bootstrap_options.watchman_startup_timeout, + bootstrap_options.watchman_socket_timeout, + bootstrap_options.watchman_socket_path, + bootstrap_options.pants_subprocessdir + ) def __init__(self, binary_util, workdir, log_level, watchman_version, watchman_supportdir, - startup_timeout, socket_timeout, socket_path_override=None): + startup_timeout, socket_timeout, socket_path_override=None, metadata_base_dir=None): """ :param binary_util: The BinaryUtil subsystem instance for binary retrieval. :param workdir: The current pants workdir. @@ -62,6 +49,7 @@ def __init__(self, binary_util, workdir, log_level, watchman_version, watchman_s :param watchman_supportdir: The supportdir for BinaryUtil. :param socket_timeout: The watchman client socket timeout (in seconds). :param socket_path_override: The overridden target path of the watchman socket, if any. + :param metadata_base_dir: The ProcessManager metadata base directory. """ self._binary_util = binary_util self._workdir = workdir @@ -72,7 +60,7 @@ def __init__(self, binary_util, workdir, log_level, watchman_version, watchman_s self._socket_path_override = socket_path_override self._log_level = log_level self._logger = logging.getLogger(__name__) - self._watchman = None + self._metadata_base_dir = metadata_base_dir @staticmethod def _convert_log_level(level): @@ -88,12 +76,15 @@ def watchman(self): watchman_binary = self._binary_util.select_binary(self._watchman_supportdir, self._watchman_version, 'watchman') - return Watchman(watchman_binary, - self._workdir, - self._convert_log_level(self._log_level), - self._startup_timeout, - self._socket_timeout, - self._socket_path_override) + return Watchman( + watchman_binary, + self._workdir, + self._convert_log_level(self._log_level), + self._startup_timeout, + self._socket_timeout, + self._socket_path_override, + metadata_base_dir=self._metadata_base_dir + ) def maybe_launch(self): if not self.watchman.is_alive(): @@ -106,6 +97,7 @@ def maybe_launch(self): self._logger.debug('watchman is running, pid={pid} socket={socket}' .format(pid=self.watchman.pid, socket=self.watchman.socket)) + return self.watchman def terminate(self): diff --git a/src/python/pants/util/BUILD b/src/python/pants/util/BUILD index 29f30a16302..697b297c0c9 100644 --- a/src/python/pants/util/BUILD +++ b/src/python/pants/util/BUILD @@ -17,6 +17,11 @@ python_library( ], ) +python_library( + name = 'collections', + sources = ['collections.py'], +) + python_library( name = 'desktop', sources = ['desktop.py'], diff --git a/src/python/pants/util/collections.py b/src/python/pants/util/collections.py new file mode 100644 index 00000000000..c2e45161cd9 --- /dev/null +++ b/src/python/pants/util/collections.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + + +def combined_dict(*dicts): + """Combine one or more dicts into a new, unified dict (dicts to the right take precedence).""" + return {k: v for d in dicts for k, v in d.items()} diff --git a/tests/python/pants_test/base_test.py b/tests/python/pants_test/base_test.py index 8361400d1ab..b398ce27ff2 100644 --- a/tests/python/pants_test/base_test.py +++ b/tests/python/pants_test/base_test.py @@ -25,6 +25,7 @@ from pants.build_graph.mutable_build_graph import MutableBuildGraph from pants.build_graph.target import Target from pants.init.util import clean_global_runtime_state +from pants.option.options_bootstrapper import OptionsBootstrapper from pants.source.source_root import SourceRootConfig from pants.subsystem.subsystem import Subsystem from pants.util.dirutil import safe_mkdir, safe_open, safe_rmtree @@ -435,3 +436,12 @@ def assertPrefixEqual(self, expected, actual_iter): :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected)))) + + def get_bootstrap_options(self, cli_options=()): + """Retrieves bootstrap options. + + :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. + """ + # Can't parse any options without a pants.ini. + self.create_file('pants.ini') + return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope() diff --git a/tests/python/pants_test/pantsd/subsystem/test_subprocess.py b/tests/python/pants_test/init/test_subprocess.py similarity index 92% rename from tests/python/pants_test/pantsd/subsystem/test_subprocess.py rename to tests/python/pants_test/init/test_subprocess.py index 745e04ceabb..19a0f043c0b 100644 --- a/tests/python/pants_test/pantsd/subsystem/test_subprocess.py +++ b/tests/python/pants_test/init/test_subprocess.py @@ -5,7 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -from pants.pantsd.subsystem.subprocess import Subprocess +from pants.init.subprocess import Subprocess from pants_test.base_test import BaseTest from pants_test.subsystem.subsystem_util import global_subsystem_instance diff --git a/tests/python/pants_test/option/test_options_integration.py b/tests/python/pants_test/option/test_options_integration.py index 15da2bc1653..ff59ae7da81 100644 --- a/tests/python/pants_test/option/test_options_integration.py +++ b/tests/python/pants_test/option/test_options_integration.py @@ -291,21 +291,3 @@ def test_pants_ignore_option(self): self.assertIn("pants_ignore = ['.*/', '/dist/', 'some/random/dir'] (from CONFIG in {})" .format(config_path), pants_run.stdout_data) - - @ensure_engine - def test_pants_ignore_option_non_default_dist_dir(self): - with temporary_dir(root_dir=os.path.abspath('.')) as tempdir: - config_path = os.path.relpath(os.path.join(tempdir, 'config.ini')) - with open(config_path, 'w+') as f: - f.write(dedent(""" - [GLOBAL] - pants_ignore: +['some/random/dir'] - pants_distdir: some/other/dist/dir - """)) - pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), - '--no-colors', - 'options']) - self.assert_success(pants_run) - self.assertIn("pants_ignore = ['.*/', '/some/other/dist/dir/', 'some/random/dir'] " - "(from CONFIG in {})".format(config_path), - pants_run.stdout_data) diff --git a/tests/python/pants_test/pantsd/BUILD b/tests/python/pants_test/pantsd/BUILD index 3a8cfa65bd2..150e86f38cc 100644 --- a/tests/python/pants_test/pantsd/BUILD +++ b/tests/python/pants_test/pantsd/BUILD @@ -73,3 +73,25 @@ python_tests( tags = {'integration'}, timeout = 120 ) + +python_tests( + name = 'pants_daemon_launcher', + sources = ['test_pants_daemon_launcher.py'], + coverage = ['pants.pantsd.pants_daemon_launcher'], + dependencies = [ + 'tests/python/pants_test/pantsd:test_deps', + 'tests/python/pants_test/subsystem:subsystem_utils', + 'src/python/pants/pantsd:pants_daemon_launcher' + ] +) + +python_tests( + name = 'watchman_launcher', + sources = ['test_watchman_launcher.py'], + coverage = ['pants.pantsd.watchman_launcher'], + dependencies = [ + 'tests/python/pants_test/pantsd:test_deps', + 'tests/python/pants_test/subsystem:subsystem_utils', + 'src/python/pants/pantsd:watchman_launcher' + ] +) diff --git a/tests/python/pants_test/pantsd/subsystem/BUILD b/tests/python/pants_test/pantsd/subsystem/BUILD deleted file mode 100644 index 1adc01a42e0..00000000000 --- a/tests/python/pants_test/pantsd/subsystem/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -python_tests( - name = 'subprocess', - sources = ['test_subprocess.py'], - coverage = ['pants.pantsd.subsystem.subprocess'], - dependencies = [ - 'src/python/pants/pantsd/subsystem:subprocess', - 'tests/python/pants_test/subsystem:subsystem_utils', - 'tests/python/pants_test:base_test' - ] -) - -python_tests( - name = 'watchman_launcher', - sources = ['test_watchman_launcher.py'], - coverage = ['pants.pantsd.subsystem.watchman_launcher'], - dependencies = [ - 'tests/python/pants_test/pantsd:test_deps', - 'tests/python/pants_test/subsystem:subsystem_utils', - 'src/python/pants/pantsd/subsystem:watchman_launcher' - ] -) diff --git a/tests/python/pants_test/pantsd/subsystem/__init__.py b/tests/python/pants_test/pantsd/subsystem/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/python/pants_test/pantsd/test_pants_daemon.py b/tests/python/pants_test/pantsd/test_pants_daemon.py index 254c88a10ab..a0843e4b80d 100644 --- a/tests/python/pants_test/pantsd/test_pants_daemon.py +++ b/tests/python/pants_test/pantsd/test_pants_daemon.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import logging +import threading import mock @@ -44,6 +45,7 @@ def test_flush(self): class PantsDaemonTest(BaseTest): def setUp(self): super(PantsDaemonTest, self).setUp() + lock = threading.RLock() self.pantsd = PantsDaemon('test_buildroot', 'test_work_dir', logging.INFO, @@ -52,6 +54,7 @@ def setUp(self): metadata_base_dir=self.subprocess_dir) self.pantsd.set_services([]) self.pantsd.set_socket_map({}) + self.pantsd.set_lock(lock) self.mock_killswitch = mock.Mock() self.pantsd._kill_switch = self.mock_killswitch diff --git a/tests/python/pants_test/init/test_pants_daemon_launcher.py b/tests/python/pants_test/pantsd/test_pants_daemon_launcher.py similarity index 79% rename from tests/python/pants_test/init/test_pants_daemon_launcher.py rename to tests/python/pants_test/pantsd/test_pants_daemon_launcher.py index 06bdb57602e..203148c0aaf 100644 --- a/tests/python/pants_test/init/test_pants_daemon_launcher.py +++ b/tests/python/pants_test/pantsd/test_pants_daemon_launcher.py @@ -7,19 +7,18 @@ import mock -from pants.init.pants_daemon_launcher import PantsDaemonLauncher from pants.pantsd.pants_daemon import PantsDaemon -from pants.pantsd.subsystem.watchman_launcher import WatchmanLauncher +from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher +from pants.pantsd.watchman_launcher import WatchmanLauncher from pants_test.base_test import BaseTest -from pants_test.subsystem.subsystem_util import global_subsystem_instance class PantsDaemonLauncherTest(BaseTest): PDL_PATCH_OPTS = dict(autospec=True, spec_set=True, return_value=(None, None, None)) - def pants_daemon_launcher(self): - factory = global_subsystem_instance(PantsDaemonLauncher.Factory) - pdl = factory.create(None) + def pants_daemon_launcher(self, cli_options=()): + bootstrap_options = self.get_bootstrap_options(cli_options) + pdl = PantsDaemonLauncher(bootstrap_options) pdl.pantsd = self.mock_pantsd pdl.watchman_launcher = self.mock_watchman_launcher return pdl @@ -43,9 +42,8 @@ def test_maybe_launch(self, mock_setup_services): @mock.patch.object(PantsDaemonLauncher, '_setup_services', **PDL_PATCH_OPTS) def test_maybe_launch_already_alive(self, mock_setup_services): self.mock_pantsd.is_alive.return_value = True - #options = {'default': {'pantsd_enabled': 'true'}} - pdl = self.pants_daemon_launcher() + pdl = self.pants_daemon_launcher(['--pantsd-enabled']) pdl.maybe_launch() self.assertEqual(mock_setup_services.call_count, 0) diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py index b620a0a643c..17d4cc8912a 100644 --- a/tests/python/pants_test/pantsd/test_pantsd_integration.py +++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py @@ -23,12 +23,10 @@ def await_pantsd(self, timeout=10): self._pid = self.await_pid(timeout) self.assert_running() - def assert_running(self, sleep=0): - time.sleep(sleep) + def assert_running(self): assert self._pid is not None and self.is_alive(), 'pantsd should be running!' - def assert_stopped(self, sleep=0): - time.sleep(sleep) + def assert_stopped(self): assert self._pid is not None and self.is_dead(), 'pantsd should be stopped!' @@ -41,19 +39,25 @@ def read_pantsd_log(workdir): class TestPantsDaemonIntegration(PantsRunIntegrationTest): @contextmanager - def pantsd_test_context(self, config=None): + def pantsd_test_context(self, log_level='info'): with self.temporary_workdir() as workdir_base: pid_dir = os.path.join(workdir_base, '.pids') workdir = os.path.join(workdir_base, '.workdir.pants.d') - pantsd_config = config or {} - pantsd_config.setdefault('GLOBAL', {'enable_pantsd': True, - 'level': 'debug', - 'pants_subprocessdir': pid_dir}) + pantsd_config = { + 'GLOBAL': { + 'enable_pantsd': True, + # The absolute paths in CI can exceed the UNIX socket path limitation + # (>104-108 characters), so we override that here with a shorter path. + 'watchman_socket_path': '/tmp/watchman.{}.sock'.format(os.getpid()), + 'level': log_level, + 'pants_subprocessdir': pid_dir + } + } checker = PantsDaemonMonitor(pid_dir) yield workdir, pantsd_config, checker - def test_pantsd_run(self): - with self.pantsd_test_context() as (workdir, pantsd_config, checker): + def test_pantsd_compile(self): + with self.pantsd_test_context('debug') as (workdir, pantsd_config, checker): # Explicitly kill any running pantsd instances for the current buildroot. self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) try: @@ -61,10 +65,6 @@ def test_pantsd_run(self): self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) checker.await_pantsd() - # This run should execute via pantsd testing the end to end client/server. - self.assert_success(self.run_pants_with_workdir(['help-advanced'], workdir, pantsd_config)) - checker.assert_running() - # This tests a deeper pantsd-based run by actually invoking a full compile. self.assert_success( self.run_pants_with_workdir( @@ -74,20 +74,16 @@ def test_pantsd_run(self): ) checker.assert_running() finally: - # Explicitly kill pantsd (from a pantsd-launched runner). - self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) - - checker.assert_stopped() - - for line in read_pantsd_log(workdir): - print(line) + try: + for line in read_pantsd_log(workdir): + print(line) + finally: + # Explicitly kill pantsd (from a pantsd-launched runner). + self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + checker.assert_stopped() - def test_pantsd_run_with_watchman(self): - # The absolute paths in CI can exceed the UNIX socket path limitation - # (>104-108 characters), so we override that here with a shorter path. - config = {'watchman': {'socket_path': '/tmp/watchman.{}.sock'.format(os.getpid())}} - - with self.pantsd_test_context(config) as (workdir, pantsd_config, checker): + def test_pantsd_run(self): + with self.pantsd_test_context('debug') as (workdir, pantsd_config, checker): print('log: {}/pantsd/pantsd.log'.format(workdir)) # Explicitly kill any running pantsd instances for the current buildroot. print('\nkill-pantsd') @@ -98,43 +94,39 @@ def test_pantsd_run_with_watchman(self): self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) checker.await_pantsd() - # This run should execute via pantsd testing the end to end client/server. - print('list') + print('list 3rdparty:') self.assert_success(self.run_pants_with_workdir(['list', '3rdparty:'], workdir, pantsd_config)) - checker.assert_running(3) + checker.assert_running() - print('list') + print('list :') self.assert_success(self.run_pants_with_workdir(['list', ':'], workdir, pantsd_config)) - checker.assert_running(3) + checker.assert_running() - print('list') - self.assert_success(self.run_pants_with_workdir(['list', ':'], + print('list ::') + self.assert_success(self.run_pants_with_workdir(['list', '::'], workdir, pantsd_config)) - checker.assert_running(3) + checker.assert_running() # And again using the cached BuildGraph. - print('cached list') + print('list ::') self.assert_success(self.run_pants_with_workdir(['list', '::'], workdir, pantsd_config)) - checker.assert_running(3) + checker.assert_running() finally: - for line in read_pantsd_log(workdir): - print(line) - - # Explicitly kill pantsd (from a pantsd-launched runner). - print('kill-pantsd') - self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) - - checker.assert_stopped() - - for line in read_pantsd_log(workdir): - print(line) + try: + for line in read_pantsd_log(workdir): + print(line) + finally: + # Explicitly kill pantsd (from a pantsd-launched runner). + print('kill-pantsd') + self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + checker.assert_stopped() # Assert there were no warnings or errors thrown in the pantsd log. for line in read_pantsd_log(workdir): @@ -159,22 +151,19 @@ def test_pantsd_broken_pipe(self): finally: # Explicitly kill pantsd (from a pantsd-launched runner). self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) - - checker.assert_stopped() + checker.assert_stopped() def test_pantsd_stacktrace_dump(self): - config = {'watchman': {'socket_path': '/tmp/watchman.{}.sock'.format(os.getpid())}} - with self.pantsd_test_context(config) as (workdir, pantsd_config, checker): + with self.pantsd_test_context() as (workdir, pantsd_config, checker): print('log: {}/pantsd/pantsd.log'.format(workdir)) # Explicitly kill any running pantsd instances for the current buildroot. self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) try: # Start pantsd implicitly via a throwaway invocation. self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) - checker.await_pantsd(3) + checker.await_pantsd() os.kill(checker.pid, signal.SIGUSR2) - checker.assert_running() # Wait for log flush. time.sleep(2) @@ -183,10 +172,9 @@ def test_pantsd_stacktrace_dump(self): finally: # Explicitly kill pantsd (from a pantsd-launched runner). self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + checker.assert_stopped() - checker.assert_stopped() - - def test_pantsd_runner_dies_after_failed_run(self): + def test_pantsd_runner_doesnt_die_after_failed_run(self): with self.pantsd_test_context() as (workdir, pantsd_config, checker): self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) try: @@ -201,15 +189,15 @@ def test_pantsd_runner_dies_after_failed_run(self): workdir, pantsd_config) ) + checker.assert_running() # Check for no stray pantsd-runner prcesses. self.assertFalse(check_process_exists_by_command('pantsd-runner')) - + # Assert pantsd is in a good functional state. self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) - + checker.assert_running() finally: # Explicitly kill pantsd (from a pantsd-launched runner). self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) - - checker.assert_stopped() + checker.assert_stopped() diff --git a/tests/python/pants_test/pantsd/subsystem/test_watchman_launcher.py b/tests/python/pants_test/pantsd/test_watchman_launcher.py similarity index 83% rename from tests/python/pants_test/pantsd/subsystem/test_watchman_launcher.py rename to tests/python/pants_test/pantsd/test_watchman_launcher.py index 7e162367acd..8b80088ca91 100644 --- a/tests/python/pants_test/pantsd/subsystem/test_watchman_launcher.py +++ b/tests/python/pants_test/pantsd/test_watchman_launcher.py @@ -7,16 +7,15 @@ import mock -from pants.pantsd.subsystem.watchman_launcher import WatchmanLauncher from pants.pantsd.watchman import Watchman +from pants.pantsd.watchman_launcher import WatchmanLauncher from pants_test.base_test import BaseTest -from pants_test.subsystem.subsystem_util import global_subsystem_instance class TestWatchmanLauncher(BaseTest): - def watchman_launcher(self, options=None): - options = options or {} - return global_subsystem_instance(WatchmanLauncher.Factory, options=options).create() + def watchman_launcher(self, cli_options=()): + bootstrap_options = self.get_bootstrap_options(cli_options) + return WatchmanLauncher.create(bootstrap_options) def create_mock_watchman(self, is_alive): mock_watchman = mock.create_autospec(Watchman, spec_set=False) @@ -62,6 +61,6 @@ def test_watchman_property(self): def test_watchman_socket_path(self): expected_path = '/a/shorter/path' - options = {WatchmanLauncher.Factory.options_scope: {'socket_path': expected_path}} + options = ['--watchman-socket-path={}'.format(expected_path)] wl = self.watchman_launcher(options) self.assertEquals(wl.watchman._sock_file, expected_path) diff --git a/tests/python/pants_test/util/BUILD b/tests/python/pants_test/util/BUILD index a8672058cc8..c113e60128b 100644 --- a/tests/python/pants_test/util/BUILD +++ b/tests/python/pants_test/util/BUILD @@ -10,6 +10,15 @@ python_tests( ], ) +python_tests( + name = 'collections', + sources = ['test_collections.py'], + coverage = ['pants.util.collections'], + dependencies = [ + 'src/python/pants/util:collections', + ] +) + python_tests( name = 'contextutil', sources = ['test_contextutil.py'], diff --git a/tests/python/pants_test/util/test_collections.py b/tests/python/pants_test/util/test_collections.py new file mode 100644 index 00000000000..559a5e0ad00 --- /dev/null +++ b/tests/python/pants_test/util/test_collections.py @@ -0,0 +1,22 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import unittest + +from pants.util.collections import combined_dict + + +class TestCollections(unittest.TestCase): + def test_combined_dict(self): + self.assertEqual( + combined_dict( + {'a': 1, 'b': 1, 'c': 1}, + {'b': 2, 'c': 2}, + {'c': 3}, + ), + {'a': 1, 'b': 2, 'c': 3} + )