-
-
Notifications
You must be signed in to change notification settings - Fork 636
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add an API to coarsen/partition Targets by their cycles #12251
Changes from all commits
15b2869
207104b
653911a
a24f4dc
9a927bd
f33419d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -66,6 +66,7 @@ use hashing::Digest; | |
use log::{self, debug, error, warn, Log}; | ||
use logging::logger::PANTS_LOGGER; | ||
use logging::{Logger, PythonLogLevel}; | ||
use petgraph::graph::{DiGraph, Graph}; | ||
use process_execution::RemoteCacheWarningsBehavior; | ||
use regex::Regex; | ||
use rule_graph::{self, RuleGraph}; | ||
|
@@ -77,7 +78,7 @@ use workunit_store::{ | |
|
||
use crate::{ | ||
externs, nodes, Core, ExecutionRequest, ExecutionStrategyOptions, ExecutionTermination, Failure, | ||
Function, Intrinsics, LocalStoreOptions, Params, RemotingOptions, Rule, Scheduler, Session, | ||
Function, Intrinsics, Key, LocalStoreOptions, Params, RemotingOptions, Rule, Scheduler, Session, | ||
Tasks, Types, Value, | ||
}; | ||
|
||
|
@@ -396,6 +397,15 @@ py_module_initializer!(native_engine, |py, m| { | |
py_fn!(py, ensure_remote_has_recursive(a: PyScheduler, b: PyList)), | ||
)?; | ||
|
||
m.add( | ||
py, | ||
"strongly_connected_components", | ||
py_fn!( | ||
py, | ||
strongly_connected_components(a: Vec<(PyObject, Vec<PyObject>)>) | ||
), | ||
)?; | ||
|
||
m.add_class::<PyExecutionRequest>(py)?; | ||
m.add_class::<PyExecutionStrategyOptions>(py)?; | ||
m.add_class::<PyExecutor>(py)?; | ||
|
@@ -830,6 +840,40 @@ fn nailgun_server_await_shutdown(py: Python, nailgun_server_ptr: PyNailgunServer | |
}) | ||
} | ||
|
||
fn strongly_connected_components( | ||
py: Python, | ||
adjacency_lists: Vec<(PyObject, Vec<PyObject>)>, | ||
) -> CPyResult<Vec<Vec<PyObject>>> { | ||
let mut graph: DiGraph<Key, (), u32> = Graph::new(); | ||
let mut node_ids: HashMap<Key, _> = HashMap::new(); | ||
|
||
for (node, adjacency_list) in adjacency_lists { | ||
let node_key = externs::key_for(node.clone_ref(py).into())?; | ||
let node_id = *node_ids | ||
.entry(node_key) | ||
.or_insert_with(|| graph.add_node(node_key)); | ||
for dependency in adjacency_list { | ||
let dependency_key = externs::key_for(dependency.clone_ref(py).into())?; | ||
let dependency_id = node_ids | ||
.entry(dependency_key) | ||
.or_insert_with(|| graph.add_node(dependency_key)); | ||
graph.add_edge(node_id, *dependency_id, ()); | ||
} | ||
} | ||
|
||
Ok( | ||
petgraph::algo::tarjan_scc(&graph) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I love great libraries like this :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe add a brief comment motivating why we use Tarjan instead of Kosaraju, given that both are offered: https://www.geeksforgeeks.org/comparision-between-tarjans-and-kosarajus-algorithm/ |
||
.into_iter() | ||
.map(|component| { | ||
component | ||
.into_iter() | ||
.map(|node_id| externs::val_for(&graph[node_id]).consume_into_py_object(py)) | ||
.collect::<Vec<_>>() | ||
}) | ||
.collect(), | ||
) | ||
} | ||
|
||
/// | ||
/// Given a set of Tasks and type information, creates a Scheduler. | ||
/// | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It'd be great to do this w/ PyO3, but I see it depends on
key_for
which hasn't been ported yet. I can port this in a followup - it's more important we land this to unblock Patrick.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yea. You can see in an earlier commit where I was attempting to do that too: the issue was that without hash/eq, I'd need to use PyDict rather than the rust native hashmap (not much of a hardship...maybe even more performant, but awkward).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What do you mean without hash/eq?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
PyObject
itself does not have Eq/Hash implementations, so you need wrapper types around it to add those (we useValue
andKey
to do that).