Skip to content

Commit

Permalink
Initial @rules for Options computation via the engine. (#5889)
Browse files Browse the repository at this point in the history
Closes #5831
  • Loading branch information
kwlzn authored Jun 8, 2018
1 parent 7f27a74 commit 11723e0
Show file tree
Hide file tree
Showing 37 changed files with 497 additions and 265 deletions.
2 changes: 1 addition & 1 deletion src/python/pants/backend/python/pex_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,4 @@ def get_local_platform():
# TODO(John Sirois): Kill some or all usages when https://github.com/pantsbuild/pex/issues/511
# is fixed.
current_platform = Platform.current()
return current_platform.platform
return current_platform.platform
19 changes: 0 additions & 19 deletions src/python/pants/bin/goal_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@

from pants.base.cmd_line_spec_parser import CmdLineSpecParser
from pants.base.workunit import WorkUnit, WorkUnitLabel
from pants.bin.repro import Reproducer
from pants.binaries.binary_util import BinaryUtil
from pants.build_graph.build_file_parser import BuildFileParser
from pants.engine.native import Native
from pants.engine.round_engine import RoundEngine
Expand All @@ -20,13 +18,9 @@
from pants.goal.run_tracker import RunTracker
from pants.help.help_printer import HelpPrinter
from pants.init.engine_initializer import EngineInitializer
from pants.init.subprocess import Subprocess
from pants.init.target_roots_calculator import TargetRootsCalculator
from pants.java.nailgun_executor import NailgunProcessGroup
from pants.option.ranked_value import RankedValue
from pants.reporting.reporting import Reporting
from pants.scm.subsystems.changed import Changed
from pants.source.source_root import SourceRootConfig
from pants.task.task import QuietTaskMixin
from pants.util.filtering import create_filters, wrap_filters

Expand Down Expand Up @@ -200,19 +194,6 @@ def __init__(self, context, goals, run_tracker, kill_nailguns, exiter=sys.exit):
self._kill_nailguns = kill_nailguns
self._exiter = exiter

@classmethod
def subsystems(cls):
"""Subsystems used outside of any task."""
return {
SourceRootConfig,
Reporting,
Reproducer,
RunTracker,
Changed,
BinaryUtil.Factory,
Subprocess.Factory
}

def _execute_engine(self):
workdir = self._context.options.for_global_scope().pants_workdir
if not workdir.endswith('.pants.d'):
Expand Down
10 changes: 7 additions & 3 deletions src/python/pants/bin/local_pants_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@

from pants.base.build_environment import get_buildroot
from pants.bin.goal_runner import GoalRunner
from pants.bin.repro import Reproducer
from pants.goal.run_tracker import RunTracker
from pants.init.options_initializer import OptionsInitializer
from pants.init.logging import setup_logging_from_options
from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer
from pants.init.repro import Reproducer
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.reporting.reporting import Reporting
from pants.util.contextutil import hard_exit_handler, maybe_profiled
Expand Down Expand Up @@ -48,7 +49,10 @@ def _run(self):
# Bootstrap options and logging.
options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
args=self._args)
options, build_config = OptionsInitializer(options_bootstrapper, exiter=self._exiter).setup()
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
setup_logging_from_options(bootstrap_options)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
global_options = options.for_global_scope()

# Apply exiter options.
Expand Down
3 changes: 2 additions & 1 deletion src/python/pants/build_graph/build_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ def registered_aliases(self):
return BuildFileAliases(
targets=target_factories_by_alias,
objects=self._exposed_object_by_alias.copy(),
context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy())
context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy()
)

def register_aliases(self, aliases):
"""Registers the given aliases to be exposed in parsed BUILD files.
Expand Down
11 changes: 11 additions & 0 deletions src/python/pants/engine/legacy/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,17 @@ python_library(
],
)

python_library(
name='options_parsing',
sources=['options_parsing.py'],
dependencies=[
'src/python/pants/build_graph',
'src/python/pants/engine:fs',
'src/python/pants/engine:objects',
'src/python/pants/option',
],
)

python_library(
name='structs',
sources=['structs.py'],
Expand Down
44 changes: 44 additions & 0 deletions src/python/pants/engine/legacy/options_parsing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# coding=utf-8
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

from pants.build_graph.build_configuration import BuildConfiguration
from pants.engine.rules import RootRule, rule
from pants.engine.selectors import Select
from pants.init.options_initializer import OptionsInitializer
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.util.objects import datatype


class OptionsParseRequest(datatype(['args', 'env'])):
"""Represents a request for Options computation."""

@classmethod
def create(cls, args, env):
assert isinstance(args, (list, tuple))
return cls(
tuple(args),
tuple(sorted(env.items() if isinstance(env, dict) else env))
)


class Options(datatype(['options', 'build_config'])):
"""Represents the result of an Options computation."""


# TODO: Accommodate file_option, dir_option, etc.
@rule(Options, [Select(OptionsBootstrapper), Select(BuildConfiguration)])
def parse_options(options_bootstrapper, build_config):
options = OptionsInitializer.create(options_bootstrapper, build_config)
options.freeze()
return Options(options, build_config)


def create_options_parsing_rules():
return [
parse_options,
RootRule(OptionsBootstrapper),
]
4 changes: 4 additions & 0 deletions src/python/pants/engine/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ def __str__(self):
class SingletonRule(datatype(['output_constraint', 'value']), Rule):
"""A default rule for a product, which is thus a singleton for that product."""

@classmethod
def from_instance(cls, obj):
return cls(type(obj), obj)

def __new__(cls, output_type, value):
# Validate result type.
if isinstance(output_type, Exactly):
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/init/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ python_library(
'src/python/pants/core_tasks',
'src/python/pants/engine/legacy:address_mapper',
'src/python/pants/engine/legacy:graph',
'src/python/pants/engine/legacy:options_parsing',
'src/python/pants/engine/legacy:parser',
'src/python/pants/engine/legacy:source_mapper',
'src/python/pants/engine/legacy:structs',
Expand All @@ -26,7 +27,6 @@ python_library(
'src/python/pants/engine:scheduler',
'src/python/pants/goal',
'src/python/pants/goal:run_tracker',
'src/python/pants/logging',
'src/python/pants/option',
'src/python/pants/process',
'src/python/pants/python',
Expand Down
28 changes: 17 additions & 11 deletions src/python/pants/init/engine_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from pants.engine.legacy.address_mapper import LegacyAddressMapper
from pants.engine.legacy.graph import (LegacyBuildGraph, TransitiveHydratedTargets,
create_legacy_graph_tasks)
from pants.engine.legacy.options_parsing import create_options_parsing_rules
from pants.engine.legacy.parser import LegacyPythonCallbacksParser
from pants.engine.legacy.structs import (AppAdaptor, GoTargetAdaptor, JavaLibraryAdaptor,
JunitTestsAdaptor, PythonLibraryAdaptor,
Expand All @@ -26,7 +27,9 @@
from pants.engine.parser import SymbolTable
from pants.engine.rules import SingletonRule
from pants.engine.scheduler import Scheduler
from pants.init.options_initializer import BuildConfigInitializer
from pants.option.global_options import GlobMatchErrorBehavior
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.util.objects import datatype


Expand Down Expand Up @@ -117,16 +120,16 @@ class EngineInitializer(object):
"""Constructs the components necessary to run the v2 engine with v1 BuildGraph compatibility."""

@staticmethod
def setup_legacy_graph(native, bootstrap_options, build_config):
def setup_legacy_graph(native, bootstrap_options, build_configuration):
"""Construct and return the components necessary for LegacyBuildGraph construction."""
return EngineInitializer.setup_legacy_graph_extended(
bootstrap_options.pants_ignore,
bootstrap_options.pants_workdir,
bootstrap_options.build_file_imports,
build_config.registered_aliases(),
build_configuration,
native=native,
glob_match_error_behavior=bootstrap_options.glob_expansion_failure,
rules=build_config.rules(),
rules=build_configuration.rules(),
build_ignore_patterns=bootstrap_options.build_ignore,
exclude_target_regexps=bootstrap_options.exclude_target_regexp,
subproject_roots=bootstrap_options.subproject_roots,
Expand All @@ -140,7 +143,7 @@ def setup_legacy_graph_extended(
pants_ignore_patterns,
workdir,
build_file_imports_behavior,
build_file_aliases,
build_configuration,
build_root=None,
native=None,
glob_match_error_behavior=None,
Expand All @@ -162,8 +165,8 @@ def setup_legacy_graph_extended(
:type build_file_imports_behavior: string
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param build_file_aliases: BuildFileAliases to register.
:type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:type build_configuration: :class:`pants.build_graph.build_configuration.BuildConfiguration`
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
Expand All @@ -178,9 +181,9 @@ def setup_legacy_graph_extended(
"""

build_root = build_root or get_buildroot()

if not rules:
rules = []
build_configuration = build_configuration or BuildConfigInitializer.get(OptionsBootstrapper())
build_file_aliases = build_configuration.registered_aliases()
rules = rules or build_configuration.rules() or []

symbol_table = LegacySymbolTable(build_file_aliases)

Expand All @@ -203,12 +206,15 @@ def setup_legacy_graph_extended(
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
[
SingletonRule.from_instance(GlobMatchErrorBehavior.create(glob_match_error_behavior)),
SingletonRule.from_instance(build_configuration),
] +
create_legacy_graph_tasks(symbol_table) +
create_fs_rules() +
create_process_rules() +
create_graph_rules(address_mapper, symbol_table) +
[SingletonRule(GlobMatchErrorBehavior,
GlobMatchErrorBehavior.create(glob_match_error_behavior))] +
create_options_parsing_rules() +
rules
)

Expand Down
29 changes: 29 additions & 0 deletions src/python/pants/init/global_subsystems.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# coding=utf-8
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

from pants.binaries.binary_util import BinaryUtil
from pants.goal.run_tracker import RunTracker
from pants.init.repro import Reproducer
from pants.init.subprocess import Subprocess
from pants.reporting.reporting import Reporting
from pants.scm.subsystems.changed import Changed
from pants.source.source_root import SourceRootConfig


class GlobalSubsystems(object):
@classmethod
def get(cls):
"""Subsystems used outside of any task."""
return {
SourceRootConfig,
Reporting,
Reproducer,
RunTracker,
Changed,
BinaryUtil.Factory,
Subprocess.Factory
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# coding=utf-8
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

import logging
import os
import sys
import time
from collections import namedtuple
from logging import FileHandler, Formatter, StreamHandler
Expand All @@ -18,8 +19,12 @@ class LoggingSetupResult(namedtuple('LoggingSetupResult', ['log_filename', 'log_
"""A structured result for logging setup."""


# TODO: Once pantsd had a separate launcher entry point, and so no longer needs to call this
# function, move this into the pants.init package, and remove the pants.logging package.
def setup_logging_from_options(bootstrap_options):
# N.B. quiet help says 'Squelches all console output apart from errors'.
level = 'ERROR' if bootstrap_options.quiet else bootstrap_options.level.upper()
return setup_logging(level, console_stream=sys.stderr, log_dir=bootstrap_options.logdir)


def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name=None):
"""Configures logging for a given scope, by default the global scope.
Expand Down
Loading

0 comments on commit 11723e0

Please sign in to comment.