[engine] rm python graphmaker; create dot formatted display (#4295)

### Problem

Printing rule graphs has not been converted to Rust. This can make it hard to visualize what the rule sets look like. Also, the Python rule graph construction is effectively dead code already.

### Solution

Port rule graph visualization and visualization based testing to the rust side of the engine implementation.

### Result

Now we're setup to use the RuleGraph from the Rust side for functions as described in #4303 and #4304.

src/python/pants/build_graph/build_file_address_mapper.py

@@ -23,7 +23,7 @@
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.address_mapper import AddressMapper
 from pants.build_graph.build_file_parser import BuildFileParser
-from pants.util.dirutil import fast_relpath, longest_dir_prefix, join_specs
+from pants.util.dirutil import fast_relpath, join_specs, longest_dir_prefix
 
 
 logger = logging.getLogger(__name__)

src/python/pants/engine/addressable.py

@@ -137,6 +137,12 @@ class Exactly(TypeConstraint):
   def satisfied_by_type(self, obj_type):
     return obj_type in self._types
 
+  def graph_str(self):
+    if len(self.types) == 1:
+      return self.types[0].__name__
+    else:
+      return repr(self)
+
 
 class SubclassesOf(TypeConstraint):
   """Objects of the exact type as well as any sub-types are allowed."""

src/python/pants/engine/scheduler.py

@@ -17,7 +17,7 @@
 from pants.build_graph.address import Address, BuildFileAddress
 from pants.engine.addressable import SubclassesOf
 from pants.engine.fs import FileContent, FilesContent, Path, PathGlobs, Snapshot
-from pants.engine.isolated_process import create_snapshot_singletons, _Snapshots
+from pants.engine.isolated_process import _Snapshots, create_snapshot_singletons
 from pants.engine.nodes import Return, Throw
 from pants.engine.rules import RuleIndex
 from pants.engine.selectors import (Select, SelectDependencies, SelectLiteral, SelectProjection,
@@ -84,14 +84,17 @@ def graph_trace(self):
           yield line.rstrip()
 
   def assert_ruleset_valid(self):
-    listed = list(TypeId(self._to_id(t)) for t in self.root_subject_types)
+    root_type_ids = self._root_type_ids()
 
-    raw_value = self._native.lib.validator_run(self._scheduler, listed, len(listed))
+    raw_value = self._native.lib.validator_run(self._scheduler, root_type_ids, len(root_type_ids))
     value = self._from_value(raw_value)
 
     if isinstance(value, Exception):
       raise ValueError(str(value))
 
+  def _root_type_ids(self):
+    return list(TypeId(self._to_id(t)) for t in sorted(self.root_subject_types))
+
   def _to_value(self, obj):
     return self._native.context.to_value(obj)
 
@@ -198,6 +201,33 @@ def _register_tasks(self, tasks):
   def visualize_graph_to_file(self, filename):
     self._native.lib.graph_visualize(self._scheduler, bytes(filename))
 
+  def rule_graph_visualization(self):
+    root_type_ids = self._root_type_ids()
+
+    with temporary_file_path() as path:
+      self._native.lib.rule_graph_visualize(
+        self._scheduler,
+        root_type_ids,
+        len(root_type_ids),
+        bytes(path))
+      with open(path) as fd:
+        for line in fd.readlines():
+          yield line.rstrip()
+
+  def rule_subgraph_visualization(self, root_subject_type, product_type):
+    root_type_id = TypeId(self._to_id(root_subject_type))
+
+    product_type_id = TypeConstraint(self._to_id(constraint_for(product_type)))
+    with temporary_file_path() as path:
+      self._native.lib.rule_subgraph_visualize(
+        self._scheduler,
+        root_type_id,
+        product_type_id,
+        bytes(path))
+      with open(path) as fd:
+        for line in fd.readlines():
+          yield line.rstrip()
+
   def invalidate(self, filenames):
     filenames_buf = self._native.context.utf8_buf_buf(filenames)
     return self._native.lib.graph_invalidate(self._scheduler, filenames_buf)

src/python/pants/engine/subsystem/native.py

@@ -83,6 +83,7 @@
 
     typedef void ExternContext;
 
+    // On the rust side the integration is defined in externs.rs
     typedef void             (*extern_log)(ExternContext*, uint8_t, uint8_t*, uint64_t);
     typedef Key              (*extern_key_for)(ExternContext*, Value*);
     typedef Value            (*extern_val_for)(ExternContext*, Key*);
@@ -196,6 +197,9 @@
 
     Value validator_run(RawScheduler*, TypeId*, uint64_t);
 
+    void rule_graph_visualize(RawScheduler*, TypeId*, uint64_t, char*);
+    void rule_subgraph_visualize(RawScheduler*, TypeId, TypeConstraint, char*);
+
     void nodes_destroy(RawNodes*);
     '''
   )

src/python/pants/engine/subsystem/native_engine_version

@@ -1 +1 @@
-c99656dc56811ab5e99f1bbc312950d2c6a2432e
+00bd3b7dfb8532b371806c6c38c52338eae0d82a

src/rust/engine/src/externs.rs

@@ -114,6 +114,10 @@ pub fn project_str(value: &Value, field: &str) -> String {
   val_to_str(&name_val)
 }
 
+pub fn key_to_str(key: &Key) -> String {
+  val_to_str(&val_for(key))
+}
+
 pub fn id_to_str(digest: Id) -> String {
   with_externs(|e| {
     (e.id_to_str)(e.context, digest).to_string().unwrap_or_else(|e| {

src/rust/engine/src/lib.rs

@@ -28,11 +28,14 @@ extern crate tar;
 extern crate tempdir;
 
 use std::ffi::CStr;
+use std::fs::File;
+use std::io;
 use std::mem;
 use std::os::raw;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
+
 use context::Core;
 use core::{Function, Key, TypeConstraint, TypeId, Value};
 use externs::{
@@ -60,7 +63,7 @@ use externs::{
   with_vec,
 };
 use nodes::Failure;
-use rule_graph::{GraphMaker, RootSubjectTypes};
+use rule_graph::{GraphMaker, RuleGraph};
 use scheduler::{RootResult, Scheduler, ExecutionStat};
 use tasks::Tasks;
 use types::Types;
@@ -495,7 +498,7 @@ pub extern fn validator_run(
   with_scheduler(scheduler_ptr, |raw| {
     with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| {
       let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks,
-                                        RootSubjectTypes { subject_types: subject_types.clone() });
+                                        subject_types.clone());
       let graph = graph_maker.full_graph();
 
       match graph.validate() {
@@ -510,6 +513,67 @@ pub extern fn validator_run(
   })
 }
 
+#[no_mangle]
+pub extern fn rule_graph_visualize(
+  scheduler_ptr: *mut RawScheduler,
+  subject_types_ptr: *mut TypeId,
+  subject_types_len: u64,
+  path_ptr: *const raw::c_char
+) {
+  with_scheduler(scheduler_ptr, |raw| {
+    with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| {
+      let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() };
+      let path = PathBuf::from(path_str);
+
+      let graph = graph_full(raw, subject_types);
+      write_to_file(path.as_path(), &graph).unwrap_or_else(|e| {
+        println!("Failed to visualize to {}: {:?}", path.display(), e);
+      });
+    })
+  })
+}
+
+#[no_mangle]
+pub extern fn rule_subgraph_visualize(
+  scheduler_ptr: *mut RawScheduler,
+  subject_type: TypeId,
+  product_type: TypeConstraint,
+  path_ptr: *const raw::c_char
+) {
+  with_scheduler(scheduler_ptr, |raw| {
+    let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() };
+    let path = PathBuf::from(path_str);
+
+    let graph = graph_sub(raw, subject_type, product_type);
+    write_to_file(path.as_path(), &graph).unwrap_or_else(|e| {
+      println!("Failed to visualize to {}: {:?}", path.display(), e);
+    });
+  })
+}
+
+
+fn graph_full(raw: &mut RawScheduler, subject_types: &Vec<TypeId>) -> RuleGraph {
+  let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks,
+                                    subject_types.clone());
+  graph_maker.full_graph()
+}
+
+fn graph_sub(
+  raw: &mut RawScheduler,
+  subject_type: TypeId,
+  product_type: TypeConstraint
+) -> RuleGraph {
+  let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks,
+                                    vec![subject_type.clone()]);
+  graph_maker.sub_graph(&subject_type, &product_type)
+}
+
+fn write_to_file(path: &Path, graph: &RuleGraph) -> io::Result<()> {
+  let file = File::create(path)?;
+  let mut f = io::BufWriter::new(file);
+  graph.visualize(&mut f)
+}
+
 fn with_scheduler<F, T>(scheduler_ptr: *mut RawScheduler, f: F) -> T
     where F: FnOnce(&mut RawScheduler)->T {
   let mut scheduler = unsafe { Box::from_raw(scheduler_ptr) };