[RFC] Allow dune to treat targets opaquely.

Summary: Some targets will be generated by external build commands (such
as static resource managers, documentation generators, etc). Often,
those resources must be generated into a directory and the external tool
will manage the contents of that directory.

Dune can properly determine when to invoke the external build tool to
regenerate that directory content, and the external build tool will be
responsible for doing so.

In this sense, Dune can treat directories generated by build commands as
"opaque" resources, almost as if the directory was the tar equivalent of
the directory.

This diff attempts to implement this, but I'd like some feedback.

There is one issue with this implementation which I could use some help
resolving. Even if the mtime does not change for the file, Dune decides
to regenerate the target - merely because it is a directory, and I
cannot find the place in Dune where it decides to do that.

src/build_system.ml

@@ -705,21 +705,21 @@ let remove_old_artifacts t ~dir ~subdirs_to_keep =
     | files ->
       List.iter files ~f:(fun fn ->
         let path = Path.relative dir fn in
-        match Unix.lstat (Path.to_string path) with
-        | { st_kind = S_DIR; _ } -> begin
-            match subdirs_to_keep with
-            | All -> ()
-            | These set ->
-              if String.Set.mem set fn ||
-                 Path.Set.mem t.build_dirs_to_keep path then
-                ()
-              else
-                Path.rm_rf path
-          end
-        | exception _ ->
-          if not (Path.Table.mem t.files path) then Path.unlink path
-        | _ ->
-          if not (Path.Table.mem t.files path) then Path.unlink path)
+        let path_is_a_target = Path.Table.mem t.files path in
+        if path_is_a_target then ()
+        else
+          match Unix.lstat (Path.to_string path) with
+          | { st_kind = S_DIR; _ } -> begin
+              match subdirs_to_keep with
+              | All -> ()
+              | These set ->
+                if String.Set.mem set fn ||
+                  Path.Set.mem t.build_dirs_to_keep path then ()
+                else
+                  Path.rm_rf path
+            end
+          | exception _ -> Path.unlink path
+          | _ -> Path.unlink path)
 
 let no_rule_found =
   let fail fn ~loc =

src/utils.ml

@@ -203,13 +203,24 @@ module Cached_digest = struct
     ; table       = Hashtbl.create 1024
     }
 
+  let dir_digest stat =
+    let ints = stat.Unix.st_size + stat.st_perm in
+    (string_of_int ints ^
+     string_of_float stat.st_mtime ^
+     string_of_float stat.st_ctime)
   let refresh fn =
-    let digest = Digest.file (Path.to_string fn) in
+    let path = Path.to_string fn in
+    let stat = Unix.stat path in
+    let digest =
+      if stat.st_kind == Unix.S_DIR then
+        dir_digest stat
+      else Digest.file (Path.to_string fn)
+    in
     Hashtbl.replace cache.table ~key:fn
       ~data:{ digest
-            ; timestamp = (Unix.stat (Path.to_string fn)).st_mtime
+            ; timestamp = stat.st_mtime
             ; timestamp_checked = cache.checked_key
-            };
+      };
     digest
 
   let file fn =
@@ -218,9 +229,15 @@ module Cached_digest = struct
       if x.timestamp_checked = cache.checked_key then
         x.digest
       else begin
-        let mtime = (Unix.stat (Path.to_string fn)).st_mtime in
+        let stat = Unix.stat (Path.to_string fn) in
+        let mtime = stat.st_mtime in
         if mtime <> x.timestamp then begin
-          let digest = Digest.file (Path.to_string fn) in
+          let digest =
+            if stat.st_kind == Unix.S_DIR then
+              dir_digest stat
+            else
+              Digest.file (Path.to_string fn)
+          in
           x.digest    <- digest;
           x.timestamp <- mtime;
         end;