Initial @rules for Options computation via the engine. (#5889)

Closes #5831

src/python/pants/backend/python/pex_util.py

@@ -32,4 +32,4 @@ def get_local_platform():
   # TODO(John Sirois): Kill some or all usages when https://github.com/pantsbuild/pex/issues/511
   # is fixed.
   current_platform = Platform.current()
-  return current_platform.platform
+  return current_platform.platform

src/python/pants/bin/goal_runner.py

@@ -10,8 +10,6 @@
 
 from pants.base.cmd_line_spec_parser import CmdLineSpecParser
 from pants.base.workunit import WorkUnit, WorkUnitLabel
-from pants.bin.repro import Reproducer
-from pants.binaries.binary_util import BinaryUtil
 from pants.build_graph.build_file_parser import BuildFileParser
 from pants.engine.native import Native
 from pants.engine.round_engine import RoundEngine
@@ -20,13 +18,9 @@
 from pants.goal.run_tracker import RunTracker
 from pants.help.help_printer import HelpPrinter
 from pants.init.engine_initializer import EngineInitializer
-from pants.init.subprocess import Subprocess
 from pants.init.target_roots_calculator import TargetRootsCalculator
 from pants.java.nailgun_executor import NailgunProcessGroup
 from pants.option.ranked_value import RankedValue
-from pants.reporting.reporting import Reporting
-from pants.scm.subsystems.changed import Changed
-from pants.source.source_root import SourceRootConfig
 from pants.task.task import QuietTaskMixin
 from pants.util.filtering import create_filters, wrap_filters
 
@@ -200,19 +194,6 @@ def __init__(self, context, goals, run_tracker, kill_nailguns, exiter=sys.exit):
     self._kill_nailguns = kill_nailguns
     self._exiter = exiter
 
-  @classmethod
-  def subsystems(cls):
-    """Subsystems used outside of any task."""
-    return {
-      SourceRootConfig,
-      Reporting,
-      Reproducer,
-      RunTracker,
-      Changed,
-      BinaryUtil.Factory,
-      Subprocess.Factory
-    }
-
   def _execute_engine(self):
     workdir = self._context.options.for_global_scope().pants_workdir
     if not workdir.endswith('.pants.d'):

src/python/pants/bin/local_pants_runner.py

@@ -7,9 +7,10 @@
 
 from pants.base.build_environment import get_buildroot
 from pants.bin.goal_runner import GoalRunner
-from pants.bin.repro import Reproducer
 from pants.goal.run_tracker import RunTracker
-from pants.init.options_initializer import OptionsInitializer
+from pants.init.logging import setup_logging_from_options
+from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer
+from pants.init.repro import Reproducer
 from pants.option.options_bootstrapper import OptionsBootstrapper
 from pants.reporting.reporting import Reporting
 from pants.util.contextutil import hard_exit_handler, maybe_profiled
@@ -48,7 +49,10 @@ def _run(self):
     # Bootstrap options and logging.
     options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
                                                                              args=self._args)
-    options, build_config = OptionsInitializer(options_bootstrapper, exiter=self._exiter).setup()
+    bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
+    setup_logging_from_options(bootstrap_options)
+    build_config = BuildConfigInitializer.get(options_bootstrapper)
+    options = OptionsInitializer.create(options_bootstrapper, build_config)
     global_options = options.for_global_scope()
 
     # Apply exiter options.

src/python/pants/build_graph/build_configuration.py

@@ -50,7 +50,8 @@ def registered_aliases(self):
     return BuildFileAliases(
         targets=target_factories_by_alias,
         objects=self._exposed_object_by_alias.copy(),
-        context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy())
+        context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy()
+    )
 
   def register_aliases(self, aliases):
     """Registers the given aliases to be exposed in parsed BUILD files.

src/python/pants/engine/legacy/BUILD

@@ -31,6 +31,17 @@ python_library(
   ],
 )
 
+python_library(
+  name='options_parsing',
+  sources=['options_parsing.py'],
+  dependencies=[
+    'src/python/pants/build_graph',
+    'src/python/pants/engine:fs',
+    'src/python/pants/engine:objects',
+    'src/python/pants/option',
+  ],
+)
+
 python_library(
   name='structs',
   sources=['structs.py'],

src/python/pants/engine/legacy/options_parsing.py

@@ -0,0 +1,44 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.build_graph.build_configuration import BuildConfiguration
+from pants.engine.rules import RootRule, rule
+from pants.engine.selectors import Select
+from pants.init.options_initializer import OptionsInitializer
+from pants.option.options_bootstrapper import OptionsBootstrapper
+from pants.util.objects import datatype
+
+
+class OptionsParseRequest(datatype(['args', 'env'])):
+  """Represents a request for Options computation."""
+
+  @classmethod
+  def create(cls, args, env):
+    assert isinstance(args, (list, tuple))
+    return cls(
+      tuple(args),
+      tuple(sorted(env.items() if isinstance(env, dict) else env))
+    )
+
+
+class Options(datatype(['options', 'build_config'])):
+  """Represents the result of an Options computation."""
+
+
+# TODO: Accommodate file_option, dir_option, etc.
+@rule(Options, [Select(OptionsBootstrapper), Select(BuildConfiguration)])
+def parse_options(options_bootstrapper, build_config):
+  options = OptionsInitializer.create(options_bootstrapper, build_config)
+  options.freeze()
+  return Options(options, build_config)
+
+
+def create_options_parsing_rules():
+  return [
+    parse_options,
+    RootRule(OptionsBootstrapper),
+  ]

src/python/pants/engine/rules.py

@@ -124,6 +124,10 @@ def __str__(self):
 class SingletonRule(datatype(['output_constraint', 'value']), Rule):
   """A default rule for a product, which is thus a singleton for that product."""
 
+  @classmethod
+  def from_instance(cls, obj):
+    return cls(type(obj), obj)
+
   def __new__(cls, output_type, value):
     # Validate result type.
     if isinstance(output_type, Exactly):

src/python/pants/init/BUILD

@@ -16,6 +16,7 @@ python_library(
     'src/python/pants/core_tasks',
     'src/python/pants/engine/legacy:address_mapper',
     'src/python/pants/engine/legacy:graph',
+    'src/python/pants/engine/legacy:options_parsing',
     'src/python/pants/engine/legacy:parser',
     'src/python/pants/engine/legacy:source_mapper',
     'src/python/pants/engine/legacy:structs',
@@ -26,7 +27,6 @@ python_library(
     'src/python/pants/engine:scheduler',
     'src/python/pants/goal',
     'src/python/pants/goal:run_tracker',
-    'src/python/pants/logging',
     'src/python/pants/option',
     'src/python/pants/process',
     'src/python/pants/python',

src/python/pants/init/engine_initializer.py

@@ -16,6 +16,7 @@
 from pants.engine.legacy.address_mapper import LegacyAddressMapper
 from pants.engine.legacy.graph import (LegacyBuildGraph, TransitiveHydratedTargets,
                                        create_legacy_graph_tasks)
+from pants.engine.legacy.options_parsing import create_options_parsing_rules
 from pants.engine.legacy.parser import LegacyPythonCallbacksParser
 from pants.engine.legacy.structs import (AppAdaptor, GoTargetAdaptor, JavaLibraryAdaptor,
                                          JunitTestsAdaptor, PythonLibraryAdaptor,
@@ -26,7 +27,9 @@
 from pants.engine.parser import SymbolTable
 from pants.engine.rules import SingletonRule
 from pants.engine.scheduler import Scheduler
+from pants.init.options_initializer import BuildConfigInitializer
 from pants.option.global_options import GlobMatchErrorBehavior
+from pants.option.options_bootstrapper import OptionsBootstrapper
 from pants.util.objects import datatype
 
 
@@ -117,16 +120,16 @@ class EngineInitializer(object):
   """Constructs the components necessary to run the v2 engine with v1 BuildGraph compatibility."""
 
   @staticmethod
-  def setup_legacy_graph(native, bootstrap_options, build_config):
+  def setup_legacy_graph(native, bootstrap_options, build_configuration):
     """Construct and return the components necessary for LegacyBuildGraph construction."""
     return EngineInitializer.setup_legacy_graph_extended(
       bootstrap_options.pants_ignore,
       bootstrap_options.pants_workdir,
       bootstrap_options.build_file_imports,
-      build_config.registered_aliases(),
+      build_configuration,
       native=native,
       glob_match_error_behavior=bootstrap_options.glob_expansion_failure,
-      rules=build_config.rules(),
+      rules=build_configuration.rules(),
       build_ignore_patterns=bootstrap_options.build_ignore,
       exclude_target_regexps=bootstrap_options.exclude_target_regexp,
       subproject_roots=bootstrap_options.subproject_roots,
@@ -140,7 +143,7 @@ def setup_legacy_graph_extended(
     pants_ignore_patterns,
     workdir,
     build_file_imports_behavior,
-    build_file_aliases,
+    build_configuration,
     build_root=None,
     native=None,
     glob_match_error_behavior=None,
@@ -162,8 +165,8 @@ def setup_legacy_graph_extended(
     :type build_file_imports_behavior: string
     :param str build_root: A path to be used as the build root. If None, then default is used.
     :param Native native: An instance of the native-engine subsystem.
-    :param build_file_aliases: BuildFileAliases to register.
-    :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
+    :param build_configuration: The `BuildConfiguration` object to get build file aliases from.
+    :type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration`
     :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
                                       bundles does not expand to anything.
     :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
@@ -178,9 +181,9 @@ def setup_legacy_graph_extended(
     """
 
     build_root = build_root or get_buildroot()
-
-    if not rules:
-      rules = []
+    build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper())
+    build_file_aliases = build_configuration.registered_aliases()
+    rules = rules or build_configuration.rules() or []
 
     symbol_table = LegacySymbolTable(build_file_aliases)
 
@@ -203,12 +206,15 @@ def setup_legacy_graph_extended(
     # Create a Scheduler containing graph and filesystem rules, with no installed goals. The
     # LegacyBuildGraph will explicitly request the products it needs.
     rules = (
+      [
+        SingletonRule.from_instance(GlobMatchErrorBehavior.create(glob_match_error_behavior)),
+        SingletonRule.from_instance(build_configuration),
+      ] +
       create_legacy_graph_tasks(symbol_table) +
       create_fs_rules() +
       create_process_rules() +
       create_graph_rules(address_mapper, symbol_table) +
-      [SingletonRule(GlobMatchErrorBehavior,
-                     GlobMatchErrorBehavior.create(glob_match_error_behavior))] +
+      create_options_parsing_rules() +
       rules
     )
 

src/python/pants/init/global_subsystems.py

@@ -0,0 +1,29 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.binaries.binary_util import BinaryUtil
+from pants.goal.run_tracker import RunTracker
+from pants.init.repro import Reproducer
+from pants.init.subprocess import Subprocess
+from pants.reporting.reporting import Reporting
+from pants.scm.subsystems.changed import Changed
+from pants.source.source_root import SourceRootConfig
+
+
+class GlobalSubsystems(object):
+  @classmethod
+  def get(cls):
+    """Subsystems used outside of any task."""
+    return {
+      SourceRootConfig,
+      Reporting,
+      Reproducer,
+      RunTracker,
+      Changed,
+      BinaryUtil.Factory,
+      Subprocess.Factory
+    }

src/python/pants/logging/setup.py → src/python/pants/init/logging.py

@@ -1,12 +1,13 @@
 # coding=utf-8
-# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
 import logging
 import os
+import sys
 import time
 from collections import namedtuple
 from logging import FileHandler, Formatter, StreamHandler
@@ -18,8 +19,12 @@ class LoggingSetupResult(namedtuple('LoggingSetupResult', ['log_filename', 'log_
   """A structured result for logging setup."""
 
 
-# TODO: Once pantsd had a separate launcher entry point, and so no longer needs to call this
-# function, move this into the pants.init package, and remove the pants.logging package.
+def setup_logging_from_options(bootstrap_options):
+  # N.B. quiet help says 'Squelches all console output apart from errors'.
+  level = 'ERROR' if bootstrap_options.quiet else bootstrap_options.level.upper()
+  return setup_logging(level, console_stream=sys.stderr, log_dir=bootstrap_options.logdir)
+
+
 def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name=None):
   """Configures logging for a given scope, by default the global scope.