Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[engine] rm python graphmaker; create dot formatted display #4295

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/python/pants/engine/addressable.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ class Exactly(TypeConstraint):
def satisfied_by_type(self, obj_type):
return obj_type in self._types

def graph_str(self):
if len(self.types) == 1:
return self.types[0].__name__
else:
return repr(self)


class SubclassesOf(TypeConstraint):
"""Objects of the exact type as well as any sub-types are allowed."""
Expand Down
603 changes: 3 additions & 600 deletions src/python/pants/engine/rules.py

Large diffs are not rendered by default.

34 changes: 32 additions & 2 deletions src/python/pants/engine/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,17 @@ def graph_trace(self):
yield line.rstrip()

def assert_ruleset_valid(self):
listed = list(TypeId(self._to_id(t)) for t in self.root_subject_types)
root_type_ids = self._root_type_ids()

raw_value = self._native.lib.validator_run(self._scheduler, listed, len(listed))
raw_value = self._native.lib.validator_run(self._scheduler, root_type_ids, len(root_type_ids))
value = self._from_value(raw_value)

if isinstance(value, Exception):
raise ValueError(str(value))

def _root_type_ids(self):
return list(TypeId(self._to_id(t)) for t in sorted(self.root_subject_types))

def _to_value(self, obj):
return self._native.context.to_value(obj)

Expand Down Expand Up @@ -177,6 +180,33 @@ def _register_tasks(self, tasks):
def visualize_graph_to_file(self, filename):
self._native.lib.graph_visualize(self._scheduler, bytes(filename))

def rule_graph_visualization(self):
root_type_ids = self._root_type_ids()

with temporary_file_path() as path:
self._native.lib.rule_graph_visualize(
self._scheduler,
root_type_ids,
len(root_type_ids),
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()

def rule_subgraph_visualization(self, root_subject_type, product_type):
root_type_id = TypeId(self._to_id(root_subject_type))

product_type_id = TypeConstraint(self._to_id(constraint_for(product_type)))
with temporary_file_path() as path:
self._native.lib.rule_subgraph_visualize(
self._scheduler,
root_type_id,
product_type_id,
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()

def invalidate_via_keys(self, subject_keys):
return self._native.lib.graph_invalidate(self._scheduler,
subject_keys,
Expand Down
4 changes: 4 additions & 0 deletions src/python/pants/engine/subsystem/native.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@

typedef void ExternContext;

// On the rust side the integration is defined in externs.rs
typedef void (*extern_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
typedef Key (*extern_key_for)(ExternContext*, Value*);
typedef Value (*extern_val_for)(ExternContext*, Key*);
Expand Down Expand Up @@ -172,6 +173,9 @@

Value validator_run(RawScheduler*, TypeId*, uint64_t);

void rule_graph_visualize(RawScheduler*, TypeId*, uint64_t, char*);
void rule_subgraph_visualize(RawScheduler*, TypeId, TypeConstraint, char*);

void nodes_destroy(RawNodes*);
'''
)
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/engine/subsystem/native_engine_version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
f3ae10a39f2b893026b4ebfa20c2e45dc8b293ee
33d19203e5948e2c510aa872df99e2053ad44976
4 changes: 4 additions & 0 deletions src/rust/engine/src/externs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@ pub fn project_str(value: &Value, field: &str) -> String {
val_to_str(&name_val)
}

pub fn key_to_str(key: &Key) -> String {
val_to_str(&val_for(key))
}

pub fn id_to_str(digest: Id) -> String {
with_externs(|e| {
(e.id_to_str)(e.context, digest).to_string().unwrap_or_else(|e| {
Expand Down
69 changes: 68 additions & 1 deletion src/rust/engine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ use std::os::raw;
use std::path::Path;
use std::sync::Arc;

use std::fs::OpenOptions;
use std::io::{BufWriter, Write};
use std::io;

use core::{Function, Key, TypeConstraint, TypeId, Value};
use externs::{
Buffer,
Expand Down Expand Up @@ -51,7 +55,7 @@ use graph::Graph;
use nodes::Failure;
use scheduler::{RootResult, Scheduler, ExecutionStat};
use tasks::Tasks;
use rule_graph::{GraphMaker, RootSubjectTypes};
use rule_graph::{GraphMaker, RootSubjectTypes, RuleGraph};

pub struct RawScheduler {
scheduler: Scheduler,
Expand Down Expand Up @@ -449,6 +453,69 @@ pub extern fn validator_run(
})
}

#[no_mangle]
pub extern fn rule_graph_visualize(
scheduler_ptr: *mut RawScheduler,
subject_types_ptr: *mut TypeId,
subject_types_len: u64,
path_ptr: *const raw::c_char
) {
with_scheduler(scheduler_ptr, |raw| {
with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| {
let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() };
let path = Path::new(path_str.as_str());

let graph = graph_full(raw, subject_types);
write_to_file(path, &graph).unwrap_or_else(|e| {
println!("Failed to visualize to {}: {:?}", path.display(), e);
});
})
})
}

#[no_mangle]
pub extern fn rule_subgraph_visualize(
scheduler_ptr: *mut RawScheduler,
subject_type: TypeId,
product_type: TypeConstraint,
path_ptr: *const raw::c_char
) {
with_scheduler(scheduler_ptr, |raw| {
let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() };
let path = Path::new(path_str.as_str());

let graph = graph_sub(raw, subject_type, product_type);
write_to_file(path, &graph).unwrap_or_else(|e| {
println!("Failed to visualize to {}: {:?}", path.display(), e);
});
})
}


fn graph_full(raw: &mut RawScheduler, subject_types: &Vec<TypeId>) -> RuleGraph {
let graph_maker = GraphMaker::new(&raw.scheduler.tasks,
RootSubjectTypes { subject_types: subject_types.clone() });
graph_maker.full_graph()
}

fn graph_sub(
raw: &mut RawScheduler,
subject_type: TypeId,
product_type: TypeConstraint
) -> RuleGraph {
let graph_maker = GraphMaker::new(&raw.scheduler.tasks,
RootSubjectTypes { subject_types: vec![subject_type.clone()] });
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This struct doesn't appear to be carrying its weight. Could either make it a tuple struct so that you can construct it more easily (RootSubjectTypes(vec![subject_type.clone())) or just remove it, probably.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah. I added it thinking it might accrete more behavior, but then it didn't. xxing

graph_maker.sub_graph(&subject_type, &product_type)
}

fn write_to_file(path: &Path, graph: &RuleGraph) -> io::Result<()> {
let file = try!(OpenOptions::new().append(true).open(path));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Append doesn't seem like a good default... I don't want a graph appended to an existing file, since dot isn't appendable afaik?

Copy link
Contributor Author

@baroquebobcat baroquebobcat Mar 2, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. This was cargo culted from one of the other file creating fns. -- trace in graph.rs

let mut f = BufWriter::new(file);

try!(write!(&mut f, "{}\n", format!("{}", graph)));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We've mostly switched to ? rather than try!... will make a pass over the old code at some point, but might as well use ? in new code.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updating to it.

Ok(())
}

fn with_scheduler<F, T>(scheduler_ptr: *mut RawScheduler, f: F) -> T
where F: FnOnce(&mut RawScheduler)->T {
let mut scheduler = unsafe { Box::from_raw(scheduler_ptr) };
Expand Down
Loading