Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add UI to engine execution #6647

Merged
merged 45 commits into from
Oct 25, 2018
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
8da78d7
initial
wisechengyi Oct 16, 2018
b287219
initial
wisechengyi Oct 16, 2018
f22b2a3
single worker works
wisechengyi Oct 16, 2018
605b788
basic works
wisechengyi Oct 17, 2018
cb99460
fmt
wisechengyi Oct 17, 2018
9a8d71b
comments
wisechengyi Oct 17, 2018
9c288cf
fmt
wisechengyi Oct 17, 2018
df8849a
cmt
wisechengyi Oct 17, 2018
60700e8
cmt
wisechengyi Oct 17, 2018
c2b7e2a
noop instead of panic for non-tty
wisechengyi Oct 17, 2018
2fcfd7d
option plumbing
wisechengyi Oct 17, 2018
a65e348
optional compiles
wisechengyi Oct 18, 2018
c67490a
fmt
wisechengyi Oct 18, 2018
298d821
plumb render option
wisechengyi Oct 18, 2018
956464f
fmt
wisechengyi Oct 18, 2018
8ac61be
plubm through run_console_rules codepath
wisechengyi Oct 18, 2018
bff4738
doc
wisechengyi Oct 18, 2018
3f3caf1
Merge branch 'master' into 6571
wisechengyi Oct 18, 2018
fb09b8a
fix tests
wisechengyi Oct 18, 2018
3ec95d2
correct doc
wisechengyi Oct 18, 2018
75a7474
clippy if let
wisechengyi Oct 18, 2018
2a3a45f
clippy if let
wisechengyi Oct 18, 2018
07d4eb5
clippy
wisechengyi Oct 19, 2018
32547fb
Merge branch '6571' of https://github.com/wisechengyi/pants into 6571
wisechengyi Oct 19, 2018
df64776
fmt
wisechengyi Oct 19, 2018
61a3a76
address comments on option
wisechengyi Oct 19, 2018
d9be623
--amend
wisechengyi Oct 19, 2018
96c39bc
plumb down worker count
wisechengyi Oct 19, 2018
0c95fd9
fmt
wisechengyi Oct 19, 2018
a89dabc
link improvement ticket
wisechengyi Oct 20, 2018
ed7d4d0
Merge branch 'master' into 6571
wisechengyi Oct 20, 2018
1b77e56
new method for console rules
wisechengyi Oct 21, 2018
d4057e9
doc
wisechengyi Oct 21, 2018
73679c6
extract method
wisechengyi Oct 21, 2018
ac9741d
fix dep and fmt
wisechengyi Oct 22, 2018
bb1839a
adjust interface
wisechengyi Oct 23, 2018
a66be04
refactor exception handling for console rules
wisechengyi Oct 23, 2018
446370b
--v2-ui to global opts + validation
wisechengyi Oct 23, 2018
7739761
dangling parenthesis
wisechengyi Oct 23, 2018
a7aa106
make static
wisechengyi Oct 23, 2018
bc87c72
display create and initialize works
wisechengyi Oct 24, 2018
3c09ec8
simplify display logic
wisechengyi Oct 25, 2018
a3500fe
remove non tty case for engine display
wisechengyi Oct 25, 2018
dda77fd
fix doc
wisechengyi Oct 25, 2018
725cf8f
fmt
wisechengyi Oct 25, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/python/pants/bin/local_pants_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,8 @@ def _maybe_run_v2(self):
try:
self._graph_session.run_console_rules(
self._options.goals_and_possible_v2_goals,
self._target_roots
self._target_roots,
self._global_options.render_v2_engine_ui
)
except Exception as e:
logger.warn('Encountered unhandled exception {!r} during rule execution!'
Expand Down
10 changes: 6 additions & 4 deletions src/python/pants/engine/native.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@
Session* session_create(Scheduler*);
void session_destroy(Session*);

ExecutionRequest* execution_request_create(void);
ExecutionRequest* execution_request_create(_Bool);
void execution_request_destroy(ExecutionRequest*);

uint64_t graph_len(Scheduler*);
Expand Down Expand Up @@ -781,8 +781,10 @@ def to_ids_buf(self, types):
def new_tasks(self):
return self.gc(self.lib.tasks_create(), self.lib.tasks_destroy)

def new_execution_request(self):
return self.gc(self.lib.execution_request_create(), self.lib.execution_request_destroy)
def new_execution_request(self, render_v2_engine_ui):
return self.gc(
self.lib.execution_request_create(render_v2_engine_ui),
self.lib.execution_request_destroy)

def new_session(self, scheduler):
return self.gc(self.lib.session_create(scheduler), self.lib.session_destroy)
Expand Down Expand Up @@ -866,7 +868,7 @@ def tc(constraint):
execution_options.remote_store_chunk_bytes,
execution_options.remote_store_chunk_upload_timeout_seconds,
execution_options.process_execution_parallelism,
execution_options.process_execution_cleanup_local_dirs
execution_options.process_execution_cleanup_local_dirs,
)
return self.gc(scheduler, self.lib.scheduler_destroy)

Expand Down
20 changes: 11 additions & 9 deletions src/python/pants/engine/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ def __init__(

self._native = native
self.include_trace_on_error = include_trace_on_error

# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(list(rules))
self._root_subject_types = sorted(rule_index.roots, key=repr)
Expand Down Expand Up @@ -421,13 +420,13 @@ def visualize_graph_to_file(self, filename):
def visualize_rule_graph_to_file(self, filename):
self._scheduler.visualize_rule_graph_to_file(filename)

def execution_request_literal(self, request_specs):
native_execution_request = self._scheduler._native.new_execution_request()
def execution_request_literal(self, request_specs, render_v2_engine_ui):
native_execution_request = self._scheduler._native.new_execution_request(render_v2_engine_ui)
for subject, product in request_specs:
self._scheduler.add_root_selection(native_execution_request, subject, product)
return ExecutionRequest(request_specs, native_execution_request)

def execution_request(self, products, subjects):
def execution_request(self, products, subjects, render_v2_engine_ui=False):
"""Create and return an ExecutionRequest for the given products and subjects.

The resulting ExecutionRequest object will contain keys tied to this scheduler's product Graph,
Expand All @@ -441,10 +440,11 @@ def execution_request(self, products, subjects):
:param subjects: A list of Spec and/or PathGlobs objects.
:type subject: list of :class:`pants.base.specs.Spec`, `pants.build_graph.Address`, and/or
:class:`pants.engine.fs.PathGlobs` objects.
:param bool render_v2_engine_ui: whether to render the v2 engine UI
:returns: An ExecutionRequest for the given products and subjects.
"""
roots = (tuple((s, p) for s in subjects for p in products))
return self.execution_request_literal(roots)
return self.execution_request_literal(roots, render_v2_engine_ui)

def invalidate_files(self, direct_filenames):
"""Invalidates the given filenames in an internal product Graph instance."""
Expand Down Expand Up @@ -512,14 +512,15 @@ def execute(self, execution_request):
except TaskError as e:
return ExecutionResult.failure(e)

def products_request(self, products, subjects):
def products_request(self, products, subjects, render_v2_engine_ui):
"""Executes a request for multiple products for some subjects, and returns the products.

:param list products: A list of product type for the request.
:param list subjects: A list of subjects for the request.
:param bool render_v2_engine_ui: whether to render the v2 engine UI
:returns: A dict from product type to lists of products each with length matching len(subjects).
"""
request = self.execution_request(products, subjects)
request = self.execution_request(products, subjects, render_v2_engine_ui)
result = self.execute(request)
if result.error:
raise result.error
Expand Down Expand Up @@ -559,14 +560,15 @@ def products_request(self, products, subjects):
product_results[product].append(state.value)
return product_results

def product_request(self, product, subjects):
def product_request(self, product, subjects, render_v2_engine_ui=False):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed on #6654, I think that you should add a @console_rule specific method here, and remove the TODO that Daniel added:

# TODO: consider adding a new top-level function adjacent to products_request used for running console tasks,
# so that this code doesn't need to exist in this form.

The idea would be that: a call to product_request(s) would never render the UI, but a call to this new method would, and would also address that TODO. It would also not support batching like product_requests (plural) does.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Made a new method specifically for console rules. However it is not clear to me whether the entire if statement was specifically for console rules, thus can be removed?

if throw_root_states:
unique_exceptions = tuple({t.exc for t in throw_root_states})
# TODO: consider adding a new top-level function adjacent to products_request used for running console tasks,
# so that this code doesn't need to exist in this form.
if len(unique_exceptions) == 1 and isinstance(unique_exceptions[0], GracefulTerminationException):
raise unique_exceptions[0]
exception_noun = pluralize(len(unique_exceptions), 'Exception')
if self._scheduler.include_trace_on_error:
cumulative_trace = '\n'.join(self.trace(request))
raise ExecutionError(
'{} encountered:\n{}'.format(exception_noun, cumulative_trace),
unique_exceptions,
)
else:
raise ExecutionError(
'{} encountered:\n {}'.format(
exception_noun,
'\n '.join('{}: {}'.format(type(t).__name__, str(t)) for t in unique_exceptions)),
unique_exceptions
)

"""Executes a request for a single product for some subjects, and returns the products.

:param class product: A product type for the request.
:param list subjects: A list of subjects for the request.
:param bool render_v2_engine_ui: whether to render the v2 engine UI
:returns: A list of the requested products, with length match len(subjects).
"""
return self.products_request([product], subjects)[product]
return self.products_request([product], subjects, render_v2_engine_ui)[product]

def capture_snapshots(self, path_globs_and_roots):
"""Synchronously captures Snapshots for each matching PathGlobs rooted at a its root directory.
Expand Down
5 changes: 3 additions & 2 deletions src/python/pants/init/engine_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,21 +192,22 @@ def validate_goals(self, goals):
if invalid_goals:
raise self.InvalidGoals(invalid_goals)

def run_console_rules(self, goals, target_roots):
def run_console_rules(self, goals, target_roots, render_v2_engine_ui):
"""Runs @console_rules sequentially and interactively by requesting their implicit Goal products.

For retryable failures, raises scheduler.ExecutionError.

:param list goals: The list of requested goal names as passed on the commandline.
:param TargetRoots target_roots: The targets root of the request.
:param bool render_v2_engine_ui: whether to render the v2 engine UI
"""
# Reduce to only applicable goals - with validation happening by way of `validate_goals()`.
goals = [goal for goal in goals if goal in self.goal_map]
subjects = self._determine_subjects(target_roots)
for goal in goals:
goal_product = self.goal_map[goal]
logger.debug('requesting {} to satisfy execution of `{}` goal'.format(goal_product, goal))
self.scheduler_session.product_request(goal_product, subjects)
self.scheduler_session.product_request(goal_product, subjects, render_v2_engine_ui)

def create_build_graph(self, target_roots, build_root=None):
"""Construct and return a `BuildGraph` given a set of input specs.
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/option/global_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ def register_bootstrap_options(cls, register):
'of the directory will be overwritten if any filenames collide.')
register('--print-exception-stacktrace', advanced=True, type=bool,
help='Print to console the full exception stack trace if encountered.')
register('--render-v2-engine-ui', default=False, type=bool, daemon=False,
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved
help='whether to show v2 engine execution progress.')
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved

# BinaryUtil options.
register('--binaries-baseurls', type=list, advanced=True,
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/pantsd/service/scheduler_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ def _prefork_body(self, session, options):
session.validate_goals(options.goals_and_possible_v2_goals)

# N.B. @console_rules run pre-fork in order to cache the products they request during execution.
session.run_console_rules(options.goals_and_possible_v2_goals, target_roots)
session.run_console_rules(options.goals_and_possible_v2_goals, target_roots, render_v2_engine_ui=False)
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved

return target_roots

Expand Down
1 change: 1 addition & 0 deletions src/rust/engine/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions src/rust/engine/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,4 @@ resettable = { path = "resettable" }
smallvec = "0.6"
tokio = "0.1"
tempfile = "3"
ui = { path = "ui" }
5 changes: 3 additions & 2 deletions src/rust/engine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ extern crate resettable;
#[macro_use]
extern crate smallvec;
extern crate tokio;
extern crate ui;

use std::ffi::CStr;
use std::fs::File;
Expand Down Expand Up @@ -571,8 +572,8 @@ pub extern "C" fn session_destroy(ptr: *mut Session) {
}

#[no_mangle]
pub extern "C" fn execution_request_create() -> *const ExecutionRequest {
Box::into_raw(Box::new(ExecutionRequest::new()))
pub extern "C" fn execution_request_create(should_render_ui: bool) -> *const ExecutionRequest {
Box::into_raw(Box::new(ExecutionRequest::new(should_render_ui)))
}

#[no_mangle]
Expand Down
52 changes: 50 additions & 2 deletions src/rust/engine/src/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use nodes::{NodeKey, Select, Tracer, TryInto, Visualizer};
use parking_lot::Mutex;
use rule_graph;
use selectors;
use ui::EngineDisplay;

///
/// A Session represents a related series of requests (generally: one run of the pants CLI) on an
Expand Down Expand Up @@ -55,11 +56,16 @@ impl Session {
pub struct ExecutionRequest {
// Set of roots for an execution, in the order they were declared.
pub roots: Vec<Root>,
// Flag used to determine whether to show engine execution progress.
pub should_render_ui: bool,
}

impl ExecutionRequest {
pub fn new() -> ExecutionRequest {
ExecutionRequest { roots: Vec::new() }
pub fn new(should_render_ui: bool) -> ExecutionRequest {
ExecutionRequest {
roots: Vec::new(),
should_render_ui,
}
}

///
Expand Down Expand Up @@ -261,12 +267,54 @@ impl Scheduler {
core: self.core.clone(),
};
let (sender, receiver) = mpsc::channel();

// Setting up display
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved
let display_worker_count = 8;
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved
let mut optional_display = if request.should_render_ui {
Some(EngineDisplay::for_stdout(0))
} else {
None
};
if let Some(display) = optional_display.as_mut() {
display.start();
display.render();
let worker_ids: Vec<String> = (0..display_worker_count)
.map(|s| format!("{}", s))
.collect();
for worker_id in worker_ids {
display.add_worker(worker_id);
display.render();
}
}

Scheduler::execute_helper(context, sender, request.roots.clone(), 8);
let roots: Vec<_> = request
.roots
.clone()
.into_iter()
.map(|s| s.into())
.collect();

let results = loop {
if let Ok(res) = receiver.recv_timeout(Duration::from_millis(100)) {
break res;
} else if let Some(display) = optional_display.as_mut() {
let ongoing_tasks = self.core.graph.heavy_hitters(&roots, display_worker_count);
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved
for (i, task) in ongoing_tasks.iter().enumerate() {
display.update(i.to_string(), format!("{:?}", task));
}
// If the number of ongoing tasks is less than the number of workers,
// fill the rest of the workers with empty string.
wisechengyi marked this conversation as resolved.
Show resolved Hide resolved
for i in ongoing_tasks.len()..display_worker_count {
display.update(i.to_string(), "".to_string());
}
display.render();
}
};
if let Some(display) = optional_display.as_mut() {
display.render();
display.finish();
};

request
.roots
Expand Down
3 changes: 1 addition & 2 deletions src/rust/engine/ui/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@ version = "0.0.1"
authors = [ "Pants Build <pantsbuild@gmail.com>" ]

[lib]
name = "engine_display"
path = "src/display.rs"

[[bin]]
name = "ui"
name = "demo"
path = "src/main.rs"

[dependencies]
Expand Down
6 changes: 5 additions & 1 deletion src/rust/engine/ui/src/display.rs
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ impl EngineDisplay {

fn render_for_pipe(&self) {
// TODO: Handle non-tty output w polling interval adjustment + summary rendering.
panic!("TODO");
// Nothing needs to be printed to pipe by default.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, while this definitely shouldn't panic, I think that rather than disabling this at this level, we should ensure that when we're inspecting the "should we render the UX" global option, we check whether we're in a tty, and then we explicitly disable the UI rendering if we do not have a TTY.

This information is passed from the pantsd client to the pantsd server via: https://github.com/pantsbuild/pants/blob/master/src/python/pants/java/nailgun_protocol.py#L285-L314 ... but I think that the gist is still that looking at isatty() for stdin/stderr/stdout should tell you whether to render.

Having done that, we should maybe just remove this method entirely if possible...?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should've clarify a bit that this is not required for this change. Just that I don't want pants to crash on pipe. There is already logic above it which does not call this function at all if it is tty.

pub fn render(&mut self) {
// TODO: Split this fork out into sub-types of EngineDisplay.
if self.is_tty {
self.render_for_tty()
} else {
self.render_for_pipe()
}
}

Copy link
Member

@stuhood stuhood Oct 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok. But if my understanding is correct, we should just remove support for attempting to render to a pipe entirely? Maybe that can be a followup ticket: to basically just have the EngineDisplay constructor return a Result::Err if you attempt to create a display against a pipe?

}

// Paints one screen of rendering.
Expand Down Expand Up @@ -309,6 +309,10 @@ impl EngineDisplay {

// Terminates the EngineDisplay and returns the cursor to a static position.
pub fn finish(&mut self) {
// Don't do anything if it's not tty.
if !self.is_tty {
return;
}
self.running = false;
let current_pos = self.get_cursor_pos();
let action_count = self.action_map.len() as u16;
Expand Down
4 changes: 2 additions & 2 deletions src/rust/engine/ui/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@
// Arc<Mutex> can be more clear than needing to grok Orderings:
#![cfg_attr(feature = "cargo-clippy", allow(mutex_atomic))]

extern crate engine_display;
extern crate rand;
extern crate ui;

use std::thread;
use std::time::Duration;

use rand::Rng;

use engine_display::EngineDisplay;
use ui::EngineDisplay;

// N.B. This is purely a demo/testing bin target for exercising the library.

Expand Down
2 changes: 1 addition & 1 deletion tests/python/pants_test/engine/test_scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def test_trace_includes_rule_exception_traceback(self):
]

scheduler = create_scheduler(rules)
request = scheduler._native.new_execution_request()
request = scheduler._native.new_execution_request(render_v2_engine_ui=False)
subject = B()
scheduler.add_root_selection(request, subject, A)
session = scheduler.new_session()
Expand Down